+ source /admin/home/loubna/.bashrc ++ HISTCONTROL=ignoreboth ++ shopt -s histappend ++ HISTSIZE=1000 ++ HISTFILESIZE=2000 ++ shopt -s checkwinsize ++ '[' -x /usr/bin/lesspipe ']' +++ SHELL=/bin/sh +++ lesspipe ++ eval 'export LESSOPEN="| /usr/bin/lesspipe %s"; export LESSCLOSE="/usr/bin/lesspipe %s %s";' +++ export 'LESSOPEN=| /usr/bin/lesspipe %s' +++ LESSOPEN='| /usr/bin/lesspipe %s' +++ export 'LESSCLOSE=/usr/bin/lesspipe %s %s' +++ LESSCLOSE='/usr/bin/lesspipe %s %s' ++ '[' -z '' ']' ++ '[' -r /etc/debian_chroot ']' ++ case "$TERM" in ++ color_prompt=yes ++ '[' -n '' ']' ++ '[' yes = yes ']' ++ PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' ++ unset color_prompt force_color_prompt ++ case "$TERM" in ++ PS1='\[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' ++ '[' -x /usr/bin/dircolors ']' ++ test -r /admin/home/loubna/.dircolors +++ dircolors -b ++ eval 'LS_COLORS='\''rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:'\''; export LS_COLORS' +++ LS_COLORS='rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:' +++ export LS_COLORS ++ alias 'ls=ls --color=auto' ++ alias 'grep=grep --color=auto' ++ alias 'fgrep=fgrep --color=auto' ++ alias 'egrep=egrep --color=auto' ++ alias 'll=ls -alF' ++ alias 'la=ls -A' ++ alias 'l=ls -CF' ++ alias 'alert=notify-send --urgency=low -i "$([ $? = 0 ] && echo terminal || echo error)" "$(history|tail -n1|sed -e '\''s/^\s*[0-9]\+\s*//;s/[;&|]\s*alert$//'\'')"' ++ '[' -f /admin/home/loubna/.bash_aliases ']' ++ shopt -oq posix ++ '[' -f /usr/share/bash-completion/bash_completion ']' ++ . /usr/share/bash-completion/bash_completion +++ BASH_COMPLETION_VERSINFO=(2 10) +++ [[ ehxB == *v* ]] +++ BASH_COMPLETION_ORIGINAL_V_VALUE=+v +++ [[ -n '' ]] +++ set +v +++ _blacklist_glob='@(acroread.sh)' +++ shopt -s extglob progcomp +++ complete -u groups slay w sux +++ complete -A stopped -P '"%' -S '"' bg +++ complete -j -P '"%' -S '"' fg jobs disown +++ complete -v readonly unset +++ complete -A setopt set +++ complete -A shopt shopt +++ complete -A helptopic help +++ complete -a unalias +++ complete -c command type which +++ complete -b builtin +++ [[ linux-gnu == *@(solaris|aix)* ]] +++ [[ linux-gnu == *@(solaris|aix)* ]] +++ [[ linux-gnu == *@(solaris|aix)* ]] +++ _backup_glob='@(#*#|*@(~|.@(bak|orig|rej|swp|dpkg*|rpm@(orig|new|save))))' +++ complete -F _service service +++ _sysvdirs +++ sysvdirs=() +++ [[ -d /etc/rc.d/init.d ]] +++ [[ -d /etc/init.d ]] +++ sysvdirs+=(/etc/init.d) +++ [[ -f /etc/slackware-version ]] +++ return 0 +++ for svcdir in "${sysvdirs[@]}" +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/acpid ]] +++ complete -F _service /etc/init.d/acpid +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/apparmor ]] +++ complete -F _service /etc/init.d/apparmor +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/apport ]] +++ complete -F _service /etc/init.d/apport +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/atd ]] +++ complete -F _service /etc/init.d/atd +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/chrony ]] +++ complete -F _service /etc/init.d/chrony +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/console-setup.sh ]] +++ complete -F _service /etc/init.d/console-setup.sh +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/cron ]] +++ complete -F _service /etc/init.d/cron +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/cryptdisks ]] +++ complete -F _service /etc/init.d/cryptdisks +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/cryptdisks-early ]] +++ complete -F _service /etc/init.d/cryptdisks-early +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/datadog-agent ]] +++ complete -F _service /etc/init.d/datadog-agent +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/datadog-agent-process ]] +++ complete -F _service /etc/init.d/datadog-agent-process +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/datadog-agent-security ]] +++ complete -F _service /etc/init.d/datadog-agent-security +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/datadog-agent-trace ]] +++ complete -F _service /etc/init.d/datadog-agent-trace +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/dbus ]] +++ complete -F _service /etc/init.d/dbus +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/docker ]] +++ complete -F _service /etc/init.d/docker +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/gdrdrv ]] +++ complete -F _service /etc/init.d/gdrdrv +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/grub-common ]] +++ complete -F _service /etc/init.d/grub-common +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/hwclock.sh ]] +++ complete -F _service /etc/init.d/hwclock.sh +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/irqbalance ]] +++ complete -F _service /etc/init.d/irqbalance +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/iscsid ]] +++ complete -F _service /etc/init.d/iscsid +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/iwpmd ]] +++ complete -F _service /etc/init.d/iwpmd +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/keyboard-setup.sh ]] +++ complete -F _service /etc/init.d/keyboard-setup.sh +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/kmod ]] +++ complete -F _service /etc/init.d/kmod +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/lvm2 ]] +++ complete -F _service /etc/init.d/lvm2 +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/lvm2-lvmpolld ]] +++ complete -F _service /etc/init.d/lvm2-lvmpolld +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/multipath-tools ]] +++ complete -F _service /etc/init.d/multipath-tools +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/munge ]] +++ complete -F _service /etc/init.d/munge +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/nfs-common ]] +++ complete -F _service /etc/init.d/nfs-common +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/open-iscsi ]] +++ complete -F _service /etc/init.d/open-iscsi +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/open-vm-tools ]] +++ complete -F _service /etc/init.d/open-vm-tools +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/plymouth ]] +++ complete -F _service /etc/init.d/plymouth +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/plymouth-log ]] +++ complete -F _service /etc/init.d/plymouth-log +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/procps ]] +++ complete -F _service /etc/init.d/procps +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/rpcbind ]] +++ complete -F _service /etc/init.d/rpcbind +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/rsync ]] +++ complete -F _service /etc/init.d/rsync +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/rsyslog ]] +++ complete -F _service /etc/init.d/rsyslog +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/screen-cleanup ]] +++ complete -F _service /etc/init.d/screen-cleanup +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/sendmail ]] +++ complete -F _service /etc/init.d/sendmail +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/ssh ]] +++ complete -F _service /etc/init.d/ssh +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/sysstat ]] +++ complete -F _service /etc/init.d/sysstat +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/udev ]] +++ complete -F _service /etc/init.d/udev +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/ufw ]] +++ complete -F _service /etc/init.d/ufw +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/unattended-upgrades ]] +++ complete -F _service /etc/init.d/unattended-upgrades +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/uuidd ]] +++ complete -F _service /etc/init.d/uuidd +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/x11-common ]] +++ complete -F _service /etc/init.d/x11-common +++ unset svc svcdir sysvdirs +++ [[ linux-gnu == *freebsd* ]] +++ shopt -u hostcomplete +++ complete -F _user_at_host talk ytalk finger +++ complete -F _known_hosts traceroute traceroute6 fping fping6 telnet rsh rlogin ftp dig mtr ssh-installkeys showmount +++ shopt -q cdable_vars +++ complete -F _cd -o nospace cd pushd +++ complete -F _command aoss command do else eval exec ltrace nice nohup padsp then time tsocks vsound xargs +++ complete -F _root_command fakeroot gksu gksudo kdesudo really +++ complete -F _longopt a2ps awk base64 bash bc bison cat chroot colordiff cp csplit cut date df diff dir du enscript env expand fmt fold gperf grep grub head irb ld ldd less ln ls m4 md5sum mkdir mkfifo mknod mv netstat nl nm objcopy objdump od paste pr ptx readelf rm rmdir sed seq shasum sha1sum sha224sum sha256sum sha384sum sha512sum shar sort split strip sum tac tail tee texindex touch tr uname unexpand uniq units vdir wc who +++ [[ 5 -gt 4 ]] +++ declare -Ag _xspecs +++ _install_xspec '!*.?(t)bz?(2)' bunzip2 bzcat pbunzip2 pbzcat lbunzip2 lbzcat +++ local 'xspec=!*.?(t)bz?(2)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.?(t)bz?(2)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.?(t)bz?(2)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.?(t)bz?(2)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.?(t)bz?(2)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.?(t)bz?(2)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.?(t)bz?(2)' +++ _install_xspec '!*.@(zip|[egjswx]ar|exe|pk3|wsz|zargo|xpi|s[tx][cdiw]|sx[gm]|o[dt][tspgfc]|od[bm]|oxt|epub|apk|aab|ipa|do[ct][xm]|p[op]t[mx]|xl[st][xm]|pyz|whl)' unzip zipinfo +++ local 'xspec=!*.@(zip|[egjswx]ar|exe|pk3|wsz|zargo|xpi|s[tx][cdiw]|sx[gm]|o[dt][tspgfc]|od[bm]|oxt|epub|apk|aab|ipa|do[ct][xm]|p[op]t[mx]|xl[st][xm]|pyz|whl)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(zip|[egjswx]ar|exe|pk3|wsz|zargo|xpi|s[tx][cdiw]|sx[gm]|o[dt][tspgfc]|od[bm]|oxt|epub|apk|aab|ipa|do[ct][xm]|p[op]t[mx]|xl[st][xm]|pyz|whl)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(zip|[egjswx]ar|exe|pk3|wsz|zargo|xpi|s[tx][cdiw]|sx[gm]|o[dt][tspgfc]|od[bm]|oxt|epub|apk|aab|ipa|do[ct][xm]|p[op]t[mx]|xl[st][xm]|pyz|whl)' +++ _install_xspec '*.Z' compress znew +++ local 'xspec=*.Z' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='*.Z' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.Z' +++ _install_xspec '!*.@(Z|[gGd]z|t[ag]z)' gunzip zcat +++ local 'xspec=!*.@(Z|[gGd]z|t[ag]z)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(Z|[gGd]z|t[ag]z)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(Z|[gGd]z|t[ag]z)' +++ _install_xspec '!*.@(Z|[gGdz]z|t[ag]z)' unpigz +++ local 'xspec=!*.@(Z|[gGdz]z|t[ag]z)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(Z|[gGdz]z|t[ag]z)' +++ _install_xspec '!*.Z' uncompress +++ local 'xspec=!*.Z' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.Z' +++ _install_xspec '!*.@(tlz|lzma)' lzcat lzegrep lzfgrep lzgrep lzless lzmore unlzma +++ local 'xspec=!*.@(tlz|lzma)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(tlz|lzma)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(tlz|lzma)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(tlz|lzma)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(tlz|lzma)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(tlz|lzma)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(tlz|lzma)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(tlz|lzma)' +++ _install_xspec '!*.@(?(t)xz|tlz|lzma)' unxz xzcat +++ local 'xspec=!*.@(?(t)xz|tlz|lzma)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(t)xz|tlz|lzma)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(t)xz|tlz|lzma)' +++ _install_xspec '!*.lrz' lrunzip +++ local 'xspec=!*.lrz' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.lrz' +++ _install_xspec '!*.@(gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx)' ee +++ local 'xspec=!*.@(gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx)' +++ _install_xspec '!*.@(gif|jp?(e)g|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|svg)' qiv +++ local 'xspec=!*.@(gif|jp?(e)g|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|svg)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(gif|jp?(e)g|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|svg)' +++ _install_xspec '!*.@(gif|jp?(e)g?(2)|j2[ck]|jp[2f]|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|?(e)ps)' xv +++ local 'xspec=!*.@(gif|jp?(e)g?(2)|j2[ck]|jp[2f]|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|?(e)ps)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(gif|jp?(e)g?(2)|j2[ck]|jp[2f]|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|?(e)ps)' +++ _install_xspec '!*.@(@(?(e)ps|?(E)PS|pdf|PDF)?(.gz|.GZ|.bz2|.BZ2|.Z))' gv ggv kghostview +++ local 'xspec=!*.@(@(?(e)ps|?(E)PS|pdf|PDF)?(.gz|.GZ|.bz2|.BZ2|.Z))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(@(?(e)ps|?(E)PS|pdf|PDF)?(.gz|.GZ|.bz2|.BZ2|.Z))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(@(?(e)ps|?(E)PS|pdf|PDF)?(.gz|.GZ|.bz2|.BZ2|.Z))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(@(?(e)ps|?(E)PS|pdf|PDF)?(.gz|.GZ|.bz2|.BZ2|.Z))' +++ _install_xspec '!*.@(dvi|DVI)?(.@(gz|Z|bz2))' xdvi kdvi +++ local 'xspec=!*.@(dvi|DVI)?(.@(gz|Z|bz2))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(dvi|DVI)?(.@(gz|Z|bz2))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(dvi|DVI)?(.@(gz|Z|bz2))' +++ _install_xspec '!*.dvi' dvips dviselect dvitype dvipdf advi dvipdfm dvipdfmx +++ local 'xspec=!*.dvi' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.dvi' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.dvi' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.dvi' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.dvi' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.dvi' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.dvi' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.dvi' +++ _install_xspec '!*.[pf]df' acroread gpdf +++ local 'xspec=!*.[pf]df' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.[pf]df' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.[pf]df' +++ _install_xspec '!*.@(pdf|fdf)?(.@(gz|GZ|bz2|BZ2|Z))' xpdf +++ local 'xspec=!*.@(pdf|fdf)?(.@(gz|GZ|bz2|BZ2|Z))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(pdf|fdf)?(.@(gz|GZ|bz2|BZ2|Z))' +++ _install_xspec '!*.@(?(e)ps|pdf)' kpdf +++ local 'xspec=!*.@(?(e)ps|pdf)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' +++ _install_xspec '!*.@(okular|@(?(e|x)ps|?(E|X)PS|[pf]df|[PF]DF|dvi|DVI|cb[rz]|CB[RZ]|djv?(u)|DJV?(U)|dvi|DVI|gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx|GIF|JP?(E)G|MIFF|TIF?(F)|PN[GM]|P[BGP]M|BMP|XPM|ICO|XWD|TGA|PCX|epub|EPUB|odt|ODT|fb?(2)|FB?(2)|mobi|MOBI|g3|G3|chm|CHM)?(.?(gz|GZ|bz2|BZ2|xz|XZ)))' okular +++ local 'xspec=!*.@(okular|@(?(e|x)ps|?(E|X)PS|[pf]df|[PF]DF|dvi|DVI|cb[rz]|CB[RZ]|djv?(u)|DJV?(U)|dvi|DVI|gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx|GIF|JP?(E)G|MIFF|TIF?(F)|PN[GM]|P[BGP]M|BMP|XPM|ICO|XWD|TGA|PCX|epub|EPUB|odt|ODT|fb?(2)|FB?(2)|mobi|MOBI|g3|G3|chm|CHM)?(.?(gz|GZ|bz2|BZ2|xz|XZ)))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(okular|@(?(e|x)ps|?(E|X)PS|[pf]df|[PF]DF|dvi|DVI|cb[rz]|CB[RZ]|djv?(u)|DJV?(U)|dvi|DVI|gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx|GIF|JP?(E)G|MIFF|TIF?(F)|PN[GM]|P[BGP]M|BMP|XPM|ICO|XWD|TGA|PCX|epub|EPUB|odt|ODT|fb?(2)|FB?(2)|mobi|MOBI|g3|G3|chm|CHM)?(.?(gz|GZ|bz2|BZ2|xz|XZ)))' +++ _install_xspec '!*.pdf' epdfview pdfunite +++ local 'xspec=!*.pdf' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.pdf' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.pdf' +++ _install_xspec '!*.@(cb[rz7t]|djv?(u)|?(e)ps|pdf)' zathura +++ local 'xspec=!*.@(cb[rz7t]|djv?(u)|?(e)ps|pdf)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(cb[rz7t]|djv?(u)|?(e)ps|pdf)' +++ _install_xspec '!*.@(?(e)ps|pdf)' ps2pdf ps2pdf12 ps2pdf13 ps2pdf14 ps2pdfwr +++ local 'xspec=!*.@(?(e)ps|pdf)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' +++ _install_xspec '!*.texi*' makeinfo texi2html +++ local 'xspec=!*.texi*' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.texi*' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.texi*' +++ _install_xspec '!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' tex latex slitex jadetex pdfjadetex pdftex pdflatex texi2dvi xetex xelatex luatex lualatex +++ local 'xspec=!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ _install_xspec '!*.mp3' mpg123 mpg321 madplay +++ local 'xspec=!*.mp3' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.mp3' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.mp3' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.mp3' +++ _install_xspec '!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' xine aaxine fbxine +++ local 'xspec=!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' +++ _install_xspec '!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM|iso|ISO)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' kaffeine dragon +++ local 'xspec=!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM|iso|ISO)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM|iso|ISO)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM|iso|ISO)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' +++ _install_xspec '!*.@(avi|asf|wmv)' aviplay +++ local 'xspec=!*.@(avi|asf|wmv)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(avi|asf|wmv)' +++ _install_xspec '!*.@(rm?(j)|ra?(m)|smi?(l))' realplay +++ local 'xspec=!*.@(rm?(j)|ra?(m)|smi?(l))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(rm?(j)|ra?(m)|smi?(l))' +++ _install_xspec '!*.@(mpg|mpeg|avi|mov|qt)' xanim +++ local 'xspec=!*.@(mpg|mpeg|avi|mov|qt)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(mpg|mpeg|avi|mov|qt)' +++ _install_xspec '!*.@(og[ag]|m3u|flac|spx)' ogg123 +++ local 'xspec=!*.@(og[ag]|m3u|flac|spx)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(og[ag]|m3u|flac|spx)' +++ _install_xspec '!*.@(mp3|og[ag]|pls|m3u)' gqmpeg freeamp +++ local 'xspec=!*.@(mp3|og[ag]|pls|m3u)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(mp3|og[ag]|pls|m3u)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(mp3|og[ag]|pls|m3u)' +++ _install_xspec '!*.fig' xfig +++ local 'xspec=!*.fig' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.fig' +++ _install_xspec '!*.@(mid?(i)|cmf)' playmidi +++ local 'xspec=!*.@(mid?(i)|cmf)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(mid?(i)|cmf)' +++ _install_xspec '!*.@(mid?(i)|rmi|rcp|[gr]36|g18|mod|xm|it|x3m|s[3t]m|kar)' timidity +++ local 'xspec=!*.@(mid?(i)|rmi|rcp|[gr]36|g18|mod|xm|it|x3m|s[3t]m|kar)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(mid?(i)|rmi|rcp|[gr]36|g18|mod|xm|it|x3m|s[3t]m|kar)' +++ _install_xspec '!*.@(669|abc|am[fs]|d[bs]m|dmf|far|it|mdl|m[eo]d|mid?(i)|mt[2m]|oct|okt?(a)|p[st]m|s[3t]m|ult|umx|wav|xm)' modplugplay modplug123 +++ local 'xspec=!*.@(669|abc|am[fs]|d[bs]m|dmf|far|it|mdl|m[eo]d|mid?(i)|mt[2m]|oct|okt?(a)|p[st]m|s[3t]m|ult|umx|wav|xm)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(669|abc|am[fs]|d[bs]m|dmf|far|it|mdl|m[eo]d|mid?(i)|mt[2m]|oct|okt?(a)|p[st]m|s[3t]m|ult|umx|wav|xm)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(669|abc|am[fs]|d[bs]m|dmf|far|it|mdl|m[eo]d|mid?(i)|mt[2m]|oct|okt?(a)|p[st]m|s[3t]m|ult|umx|wav|xm)' +++ _install_xspec '*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' vi vim gvim rvim view rview rgvim rgview gview emacs xemacs sxemacs kate kwrite +++ local 'xspec=*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ _install_xspec '!*.@(zip|z|gz|tgz)' bzme +++ local 'xspec=!*.@(zip|z|gz|tgz)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(zip|z|gz|tgz)' +++ _install_xspec '!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' netscape mozilla lynx galeon dillo elinks amaya epiphany +++ local 'xspec=!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' +++ _install_xspec '!*.@(sxw|stw|sxg|sgl|doc?([mx])|dot?([mx])|rtf|txt|htm|html|?(f)odt|ott|odm|pdf)' oowriter lowriter +++ local 'xspec=!*.@(sxw|stw|sxg|sgl|doc?([mx])|dot?([mx])|rtf|txt|htm|html|?(f)odt|ott|odm|pdf)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxw|stw|sxg|sgl|doc?([mx])|dot?([mx])|rtf|txt|htm|html|?(f)odt|ott|odm|pdf)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxw|stw|sxg|sgl|doc?([mx])|dot?([mx])|rtf|txt|htm|html|?(f)odt|ott|odm|pdf)' +++ _install_xspec '!*.@(sxi|sti|pps?(x)|ppt?([mx])|pot?([mx])|?(f)odp|otp)' ooimpress loimpress +++ local 'xspec=!*.@(sxi|sti|pps?(x)|ppt?([mx])|pot?([mx])|?(f)odp|otp)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxi|sti|pps?(x)|ppt?([mx])|pot?([mx])|?(f)odp|otp)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxi|sti|pps?(x)|ppt?([mx])|pot?([mx])|?(f)odp|otp)' +++ _install_xspec '!*.@(sxc|stc|xls?([bmx])|xlw|xlt?([mx])|[ct]sv|?(f)ods|ots)' oocalc localc +++ local 'xspec=!*.@(sxc|stc|xls?([bmx])|xlw|xlt?([mx])|[ct]sv|?(f)ods|ots)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxc|stc|xls?([bmx])|xlw|xlt?([mx])|[ct]sv|?(f)ods|ots)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxc|stc|xls?([bmx])|xlw|xlt?([mx])|[ct]sv|?(f)ods|ots)' +++ _install_xspec '!*.@(sxd|std|sda|sdd|?(f)odg|otg)' oodraw lodraw +++ local 'xspec=!*.@(sxd|std|sda|sdd|?(f)odg|otg)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxd|std|sda|sdd|?(f)odg|otg)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxd|std|sda|sdd|?(f)odg|otg)' +++ _install_xspec '!*.@(sxm|smf|mml|odf)' oomath lomath +++ local 'xspec=!*.@(sxm|smf|mml|odf)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxm|smf|mml|odf)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxm|smf|mml|odf)' +++ _install_xspec '!*.odb' oobase lobase +++ local 'xspec=!*.odb' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.odb' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.odb' +++ _install_xspec '!*.[rs]pm' rpm2cpio +++ local 'xspec=!*.[rs]pm' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.[rs]pm' +++ _install_xspec '!*.aux' bibtex +++ local 'xspec=!*.aux' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.aux' +++ _install_xspec '!*.po' poedit gtranslator kbabel lokalize +++ local 'xspec=!*.po' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.po' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.po' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.po' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.po' +++ _install_xspec '!*.@([Pp][Rr][Gg]|[Cc][Ll][Pp])' harbour gharbour hbpp +++ local 'xspec=!*.@([Pp][Rr][Gg]|[Cc][Ll][Pp])' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@([Pp][Rr][Gg]|[Cc][Ll][Pp])' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@([Pp][Rr][Gg]|[Cc][Ll][Pp])' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@([Pp][Rr][Gg]|[Cc][Ll][Pp])' +++ _install_xspec '!*.[Hh][Rr][Bb]' hbrun +++ local 'xspec=!*.[Hh][Rr][Bb]' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.[Hh][Rr][Bb]' +++ _install_xspec '!*.ly' lilypond ly2dvi +++ local 'xspec=!*.ly' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.ly' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.ly' +++ _install_xspec '!*.@(dif?(f)|?(d)patch)?(.@([gx]z|bz2|lzma))' cdiff +++ local 'xspec=!*.@(dif?(f)|?(d)patch)?(.@([gx]z|bz2|lzma))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(dif?(f)|?(d)patch)?(.@([gx]z|bz2|lzma))' +++ _install_xspec '!@(*.@(ks|jks|jceks|p12|pfx|bks|ubr|gkr|cer|crt|cert|p7b|pkipath|pem|p10|csr|crl)|cacerts)' portecle +++ local 'xspec=!@(*.@(ks|jks|jceks|p12|pfx|bks|ubr|gkr|cer|crt|cert|p7b|pkipath|pem|p10|csr|crl)|cacerts)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!@(*.@(ks|jks|jceks|p12|pfx|bks|ubr|gkr|cer|crt|cert|p7b|pkipath|pem|p10|csr|crl)|cacerts)' +++ _install_xspec '!*.@(mp[234c]|og[ag]|@(fl|a)ac|m4[abp]|spx|tta|w?(a)v|wma|aif?(f)|asf|ape)' kid3 kid3-qt +++ local 'xspec=!*.@(mp[234c]|og[ag]|@(fl|a)ac|m4[abp]|spx|tta|w?(a)v|wma|aif?(f)|asf|ape)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(mp[234c]|og[ag]|@(fl|a)ac|m4[abp]|spx|tta|w?(a)v|wma|aif?(f)|asf|ape)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(mp[234c]|og[ag]|@(fl|a)ac|m4[abp]|spx|tta|w?(a)v|wma|aif?(f)|asf|ape)' +++ unset -f _install_xspec +++ complete -F _minimal '' +++ complete -D -F _completion_loader +++ compat_dir=/etc/bash_completion.d +++ [[ -d /etc/bash_completion.d ]] +++ [[ -r /etc/bash_completion.d ]] +++ [[ -x /etc/bash_completion.d ]] +++ for i in "$compat_dir"/* +++ [[ apport_completion != @(@(#*#|*@(~|.@(bak|orig|rej|swp|dpkg*|rpm@(orig|new|save))))|Makefile*|@(acroread.sh)) ]] +++ [[ -f /etc/bash_completion.d/apport_completion ]] +++ [[ -r /etc/bash_completion.d/apport_completion ]] +++ . /etc/bash_completion.d/apport_completion ++++ complete -F _apport-bug -o filenames -o dirnames ubuntu-bug ++++ complete -F _apport-bug -o filenames -o dirnames apport-bug ++++ complete -F _apport-cli -o filenames -o dirnames apport-cli ++++ complete -F _apport-unpack -o filenames -o dirnames apport-unpack ++++ complete -F _apport-collect apport-collect +++ for i in "$compat_dir"/* +++ [[ git-prompt != @(@(#*#|*@(~|.@(bak|orig|rej|swp|dpkg*|rpm@(orig|new|save))))|Makefile*|@(acroread.sh)) ]] +++ [[ -f /etc/bash_completion.d/git-prompt ]] +++ [[ -r /etc/bash_completion.d/git-prompt ]] +++ . /etc/bash_completion.d/git-prompt ++++ [[ -e /usr/lib/git-core/git-sh-prompt ]] ++++ . /usr/lib/git-core/git-sh-prompt +++++ __git_printf_supports_v= +++++ printf -v __git_printf_supports_v -- %s yes +++ unset compat_dir i _blacklist_glob +++ user_completion=/admin/home/loubna/.bash_completion +++ [[ /usr/share/bash-completion/bash_completion != /admin/home/loubna/.bash_completion ]] +++ [[ -r /admin/home/loubna/.bash_completion ]] +++ unset user_completion +++ unset -f have +++ unset have +++ set +v +++ unset BASH_COMPLETION_ORIGINAL_V_VALUE ++ export PATH=/opt/slurm/bin:/opt/slurm/sbin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/fsx/loubna/google-cloud-sdk/bin:/fsx/loubna/miniconda3/envs/eval-harness/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin ++ PATH=/opt/slurm/bin:/opt/slurm/sbin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/fsx/loubna/google-cloud-sdk/bin:/fsx/loubna/miniconda3/envs/eval-harness/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin +++ /fsx/loubna/miniconda3/bin/conda shell.bash hook ++ __conda_setup='export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' export _CE_M='\'''\'' export _CE_CONDA='\'''\'' export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\'' # Copyright (C) 2012 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause __conda_exe() ( "$CONDA_EXE" $_CE_M $_CE_CONDA "$@" ) __conda_hashr() { if [ -n "${ZSH_VERSION:+x}" ]; then \rehash elif [ -n "${POSH_VERSION:+x}" ]; then : # pass else \hash -r fi } __conda_activate() { if [ -n "${CONDA_PS1_BACKUP:+x}" ]; then # Handle transition from shell activated with conda <= 4.3 to a subsequent activation # after conda updated to >= 4.4. See issue #6173. PS1="$CONDA_PS1_BACKUP" \unset CONDA_PS1_BACKUP fi \local ask_conda ask_conda="$(PS1="${PS1:-}" __conda_exe shell.posix "$@")" || \return \eval "$ask_conda" __conda_hashr } __conda_reactivate() { \local ask_conda ask_conda="$(PS1="${PS1:-}" __conda_exe shell.posix reactivate)" || \return \eval "$ask_conda" __conda_hashr } conda() { \local cmd="${1-__missing__}" case "$cmd" in activate|deactivate) __conda_activate "$@" ;; install|update|upgrade|remove|uninstall) __conda_exe "$@" || \return __conda_reactivate ;; *) __conda_exe "$@" ;; esac } if [ -z "${CONDA_SHLVL+x}" ]; then \export CONDA_SHLVL=0 # In dev-mode CONDA_EXE is python.exe and on Windows # it is in a different relative location to condabin. if [ -n "${_CE_CONDA:+x}" ] && [ -n "${WINDIR+x}" ]; then PATH="$(\dirname "$CONDA_EXE")/condabin${PATH:+":${PATH}"}" else PATH="$(\dirname "$(\dirname "$CONDA_EXE")")/condabin${PATH:+":${PATH}"}" fi \export PATH # We'\''re not allowing PS1 to be unbound. It must at least be set. # However, we'\''re not exporting it, which can cause problems when starting a second shell # via a first shell (i.e. starting zsh from bash). if [ -z "${PS1+x}" ]; then PS1= fi fi conda activate base' ++ '[' 0 -eq 0 ']' ++ eval 'export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' export _CE_M='\'''\'' export _CE_CONDA='\'''\'' export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\'' # Copyright (C) 2012 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause __conda_exe() ( "$CONDA_EXE" $_CE_M $_CE_CONDA "$@" ) __conda_hashr() { if [ -n "${ZSH_VERSION:+x}" ]; then \rehash elif [ -n "${POSH_VERSION:+x}" ]; then : # pass else \hash -r fi } __conda_activate() { if [ -n "${CONDA_PS1_BACKUP:+x}" ]; then # Handle transition from shell activated with conda <= 4.3 to a subsequent activation # after conda updated to >= 4.4. See issue #6173. PS1="$CONDA_PS1_BACKUP" \unset CONDA_PS1_BACKUP fi \local ask_conda ask_conda="$(PS1="${PS1:-}" __conda_exe shell.posix "$@")" || \return \eval "$ask_conda" __conda_hashr } __conda_reactivate() { \local ask_conda ask_conda="$(PS1="${PS1:-}" __conda_exe shell.posix reactivate)" || \return \eval "$ask_conda" __conda_hashr } conda() { \local cmd="${1-__missing__}" case "$cmd" in activate|deactivate) __conda_activate "$@" ;; install|update|upgrade|remove|uninstall) __conda_exe "$@" || \return __conda_reactivate ;; *) __conda_exe "$@" ;; esac } if [ -z "${CONDA_SHLVL+x}" ]; then \export CONDA_SHLVL=0 # In dev-mode CONDA_EXE is python.exe and on Windows # it is in a different relative location to condabin. if [ -n "${_CE_CONDA:+x}" ] && [ -n "${WINDIR+x}" ]; then PATH="$(\dirname "$CONDA_EXE")/condabin${PATH:+":${PATH}"}" else PATH="$(\dirname "$(\dirname "$CONDA_EXE")")/condabin${PATH:+":${PATH}"}" fi \export PATH # We'\''re not allowing PS1 to be unbound. It must at least be set. # However, we'\''re not exporting it, which can cause problems when starting a second shell # via a first shell (i.e. starting zsh from bash). if [ -z "${PS1+x}" ]; then PS1= fi fi conda activate base' +++ export CONDA_EXE=/fsx/loubna/miniconda3/bin/conda +++ CONDA_EXE=/fsx/loubna/miniconda3/bin/conda +++ export _CE_M= +++ _CE_M= +++ export _CE_CONDA= +++ _CE_CONDA= +++ export CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python +++ CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python +++ '[' -z x ']' +++ conda activate base +++ local cmd=activate +++ case "$cmd" in +++ __conda_activate activate base +++ '[' -n '' ']' +++ local ask_conda ++++ PS1='\[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' ++++ __conda_exe shell.posix activate base ++++ /fsx/loubna/miniconda3/bin/conda shell.posix activate base +++ ask_conda='PS1='\''(base) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '\'' export PATH='\''/opt/slurm/bin:/opt/slurm/sbin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/fsx/loubna/google-cloud-sdk/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin'\'' export CONDA_PREFIX='\''/fsx/loubna/miniconda3'\'' export CONDA_SHLVL='\''3'\'' export CONDA_DEFAULT_ENV='\''base'\'' export CONDA_PROMPT_MODIFIER='\''(base) '\'' export CONDA_PREFIX_2='\''/fsx/loubna/miniconda3/envs/eval-harness'\'' export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' export _CE_M='\'''\'' export _CE_CONDA='\'''\'' export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\''' +++ eval 'PS1='\''(base) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '\'' export PATH='\''/opt/slurm/bin:/opt/slurm/sbin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/fsx/loubna/google-cloud-sdk/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin'\'' export CONDA_PREFIX='\''/fsx/loubna/miniconda3'\'' export CONDA_SHLVL='\''3'\'' export CONDA_DEFAULT_ENV='\''base'\'' export CONDA_PROMPT_MODIFIER='\''(base) '\'' export CONDA_PREFIX_2='\''/fsx/loubna/miniconda3/envs/eval-harness'\'' export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' export _CE_M='\'''\'' export _CE_CONDA='\'''\'' export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\''' ++++ PS1='(base) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' ++++ export PATH=/opt/slurm/bin:/opt/slurm/sbin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/fsx/loubna/google-cloud-sdk/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin ++++ PATH=/opt/slurm/bin:/opt/slurm/sbin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/fsx/loubna/google-cloud-sdk/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin ++++ export CONDA_PREFIX=/fsx/loubna/miniconda3 ++++ CONDA_PREFIX=/fsx/loubna/miniconda3 ++++ export CONDA_SHLVL=3 ++++ CONDA_SHLVL=3 ++++ export CONDA_DEFAULT_ENV=base ++++ CONDA_DEFAULT_ENV=base ++++ export 'CONDA_PROMPT_MODIFIER=(base) ' ++++ CONDA_PROMPT_MODIFIER='(base) ' ++++ export CONDA_PREFIX_2=/fsx/loubna/miniconda3/envs/eval-harness ++++ CONDA_PREFIX_2=/fsx/loubna/miniconda3/envs/eval-harness ++++ export CONDA_EXE=/fsx/loubna/miniconda3/bin/conda ++++ CONDA_EXE=/fsx/loubna/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python +++ __conda_hashr +++ '[' -n '' ']' +++ '[' -n '' ']' +++ hash -r ++ unset __conda_setup ++ export WANDB_CACHE_DIR=/fsx/loubna/.tmp/wandb ++ WANDB_CACHE_DIR=/fsx/loubna/.tmp/wandb ++ export TMPDIR=/fsx/loubna/.tmp ++ TMPDIR=/fsx/loubna/.tmp ++ export HUGGINGFACE_HUB_CACHE=/fsx/loubna/.cache ++ HUGGINGFACE_HUB_CACHE=/fsx/loubna/.cache ++ export HF_DATASETS_CACHE=/fsx/loubna/.cache ++ HF_DATASETS_CACHE=/fsx/loubna/.cache ++ '[' -f /fsx/loubna/google-cloud-sdk/path.bash.inc ']' ++ . /fsx/loubna/google-cloud-sdk/path.bash.inc ++++ command readlink /fsx/loubna/google-cloud-sdk/path.bash.inc ++++ readlink /fsx/loubna/google-cloud-sdk/path.bash.inc +++ script_link= +++ script_link=/fsx/loubna/google-cloud-sdk/path.bash.inc +++ apparent_sdk_dir=/fsx/loubna/google-cloud-sdk +++ '[' /fsx/loubna/google-cloud-sdk == /fsx/loubna/google-cloud-sdk/path.bash.inc ']' ++++ command cd -P /fsx/loubna/google-cloud-sdk ++++ cd -P /fsx/loubna/google-cloud-sdk ++++ command pwd -P ++++ pwd -P +++ sdk_dir=/fsx/loubna/google-cloud-sdk +++ bin_path=/fsx/loubna/google-cloud-sdk/bin +++ [[ :/opt/slurm/bin:/opt/slurm/sbin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/fsx/loubna/google-cloud-sdk/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin: != *\:\/\f\s\x\/\l\o\u\b\n\a\/\g\o\o\g\l\e\-\c\l\o\u\d\-\s\d\k\/\b\i\n\:* ]] ++ '[' -f /fsx/loubna/google-cloud-sdk/completion.bash.inc ']' ++ . /fsx/loubna/google-cloud-sdk/completion.bash.inc +++ complete -o nospace -F _python_argcomplete gcloud +++ unset bq_COMMANDS +++ complete -F _bq_completer bq +++ complete -o nospace -F _python_argcomplete gsutil + conda activate brr4 + local cmd=activate + case "$cmd" in + __conda_activate activate brr4 + '[' -n '' ']' + local ask_conda ++ PS1='(base) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' ++ __conda_exe shell.posix activate brr4 ++ /fsx/loubna/miniconda3/bin/conda shell.posix activate brr4 + ask_conda='PS1='\''(brr4) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '\'' export PATH='\''/opt/slurm/bin:/opt/slurm/sbin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/fsx/loubna/google-cloud-sdk/bin:/fsx/loubna/miniconda3/envs/brr4/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin'\'' export CONDA_PREFIX='\''/fsx/loubna/miniconda3/envs/brr4'\'' export CONDA_SHLVL='\''4'\'' export CONDA_DEFAULT_ENV='\''brr4'\'' export CONDA_PROMPT_MODIFIER='\''(brr4) '\'' export CONDA_PREFIX_3='\''/fsx/loubna/miniconda3'\'' export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' export _CE_M='\'''\'' export _CE_CONDA='\'''\'' export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\''' + eval 'PS1='\''(brr4) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '\'' export PATH='\''/opt/slurm/bin:/opt/slurm/sbin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/fsx/loubna/google-cloud-sdk/bin:/fsx/loubna/miniconda3/envs/brr4/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin'\'' export CONDA_PREFIX='\''/fsx/loubna/miniconda3/envs/brr4'\'' export CONDA_SHLVL='\''4'\'' export CONDA_DEFAULT_ENV='\''brr4'\'' export CONDA_PROMPT_MODIFIER='\''(brr4) '\'' export CONDA_PREFIX_3='\''/fsx/loubna/miniconda3'\'' export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' export _CE_M='\'''\'' export _CE_CONDA='\'''\'' export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\''' ++ PS1='(brr4) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' ++ export PATH=/opt/slurm/bin:/opt/slurm/sbin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/fsx/loubna/google-cloud-sdk/bin:/fsx/loubna/miniconda3/envs/brr4/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin ++ PATH=/opt/slurm/bin:/opt/slurm/sbin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/fsx/loubna/google-cloud-sdk/bin:/fsx/loubna/miniconda3/envs/brr4/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin ++ export CONDA_PREFIX=/fsx/loubna/miniconda3/envs/brr4 ++ CONDA_PREFIX=/fsx/loubna/miniconda3/envs/brr4 ++ export CONDA_SHLVL=4 ++ CONDA_SHLVL=4 ++ export CONDA_DEFAULT_ENV=brr4 ++ CONDA_DEFAULT_ENV=brr4 ++ export 'CONDA_PROMPT_MODIFIER=(brr4) ' ++ CONDA_PROMPT_MODIFIER='(brr4) ' ++ export CONDA_PREFIX_3=/fsx/loubna/miniconda3 ++ CONDA_PREFIX_3=/fsx/loubna/miniconda3 ++ export CONDA_EXE=/fsx/loubna/miniconda3/bin/conda ++ CONDA_EXE=/fsx/loubna/miniconda3/bin/conda ++ export _CE_M= ++ _CE_M= ++ export _CE_CONDA= ++ _CE_CONDA= ++ export CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python ++ CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python + __conda_hashr + '[' -n '' ']' + '[' -n '' ']' + hash -r ++ date + echo 'START TIME: Wed Jun 21 17:11:39 UTC 2023' START TIME: Wed Jun 21 17:11:39 UTC 2023 + BRRR_REPO=/fsx/loubna/code/fork/brrr + SCRIPT_REPO=/fsx/loubna/code/fork/brrr/examples/gpt2_mqa + pushd /fsx/loubna/code/fork/brrr/examples/gpt2_mqa /fsx/loubna/code/fork/brrr/examples/gpt2_mqa /fsx/loubna/code/fork/brrr/examples/gpt2_mqa/hub_logs + GPUS_PER_NODE=8 + NNODES=8 ++ head -n 1 ++ scontrol show hostnames 'ip-26-0-154-[80,121,138,144,155,185,245],ip-26-0-155-33' + MASTER_ADDR=ip-26-0-154-80 + MASTER_PORT=6000 + CONFIG_FILE=/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/configs/config_5b_explicit_model_conf.yaml + export USE_FAST=1 + USE_FAST=1 + export CUDA_DEVICE_MAX_CONNECTIONS=1 + CUDA_DEVICE_MAX_CONNECTIONS=1 + CMD=' main.py --config-file /fsx/loubna/code/fork/brrr/examples/gpt2_mqa/configs/config_5b_explicit_model_conf.yaml ' + export 'LAUNCHER=python -u -m torch.distributed.run --nproc_per_node 8 --nnodes 8 --rdzv_endpoint ip-26-0-154-80:6000 --rdzv_backend c10d --max_restarts 0 --tee 3 ' + LAUNCHER='python -u -m torch.distributed.run --nproc_per_node 8 --nnodes 8 --rdzv_endpoint ip-26-0-154-80:6000 --rdzv_backend c10d --max_restarts 0 --tee 3 ' + echo main.py --config-file /fsx/loubna/code/fork/brrr/examples/gpt2_mqa/configs/config_5b_explicit_model_conf.yaml main.py --config-file /fsx/loubna/code/fork/brrr/examples/gpt2_mqa/configs/config_5b_explicit_model_conf.yaml + export NCCL_ASYNC_ERROR_HANDLING=1 + NCCL_ASYNC_ERROR_HANDLING=1 + export NCCL_PROTO=simple + NCCL_PROTO=simple + export RDMAV_FORK_SAFE=1 + RDMAV_FORK_SAFE=1 + export FI_EFA_FORK_SAFE=1 + FI_EFA_FORK_SAFE=1 + export FI_EFA_USE_DEVICE_RDMA=1 + FI_EFA_USE_DEVICE_RDMA=1 + export FI_PROVIDER=efa + FI_PROVIDER=efa + export FI_LOG_LEVEL=1 + FI_LOG_LEVEL=1 + export NCCL_IB_DISABLE=1 + NCCL_IB_DISABLE=1 + export NCCL_SOCKET_IFNAME=ens + NCCL_SOCKET_IFNAME=ens + SRUN_ARGS=' --wait=60 --kill-on-bad-exit=1 ' + SLURM_JOB_ID=161646 + srun --wait=60 --kill-on-bad-exit=1 --jobid 161646 -u bash -c 'python -u -m torch.distributed.run --nproc_per_node 8 --nnodes 8 --rdzv_endpoint ip-26-0-154-80:6000 --rdzv_backend c10d --max_restarts 0 --tee 3 --node_rank $SLURM_PROCID --role $SLURMD_NODENAME: main.py --config-file /fsx/loubna/code/fork/brrr/examples/gpt2_mqa/configs/config_5b_explicit_model_conf.yaml ' master_addr is only used for static rdzv_backend and when rdzv_endpoint is not specified. WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** master_addr is only used for static rdzv_backend and when rdzv_endpoint is not specified. WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** master_addr is only used for static rdzv_backend and when rdzv_endpoint is not specified. WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** master_addr is only used for static rdzv_backend and when rdzv_endpoint is not specified. WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** master_addr is only used for static rdzv_backend and when rdzv_endpoint is not specified. WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** master_addr is only used for static rdzv_backend and when rdzv_endpoint is not specified. WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** master_addr is only used for static rdzv_backend and when rdzv_endpoint is not specified. WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** master_addr is only used for static rdzv_backend and when rdzv_endpoint is not specified. WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: Config(general=GeneralArgs(name='1b_starcoderdata_lr3', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: kill_switch_path=PosixPath('/fsx/loubna/br4-experiments/kill_loubna_starcoder'), [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: ignore_sanity_checks=True), [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: profile=None, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: checkpoints=CheckpointsArgs(checkpoints_path=PosixPath('/fsx/loubna/br4-experiments/checkpoints/debug/1b_star'), [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: checkpoint_interval=10000, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: load_from_specific_checkpoint=None, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: upload_s3_path=None, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: upload_s3_num_workers=None), [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: parallelism=ParallelismArgs(dp=64, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: pp=1, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: tp=1, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: pp_engine=, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: tp_mode=, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: recompute_granularity=None, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: tp_column_linear_async_communication=True), [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: model=ModelArgs(hf_gpt2_model_name='/fsx/loubna/starcoder-tokenizer/15b', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: hidden_size=2048, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: num_attention_heads=16, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: n_inner=8192, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: n_layer=24, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: max_position_embeddings=8192, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: vocab_size=49152, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: layer_norm_epsilon=1e-05, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: scale_attn_weights=True, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: activation_function='gelu', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: attention_softmax_in_fp32=True, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: resid_pdrop=0.1, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: attn_pdrop=0.1, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: embd_pdrop=0.1, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: pad_key_length=True, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: make_vocab_size_divisible_by=128, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: dtype=torch.bfloat16, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: init_method=RandomInit(std=0.02209), [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: seed=42), [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: logging=LoggingArgs(log_level='info', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: log_level_replica='info', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: iteration_step_info_interval=1, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: tensorboard_logger=TensorboardLoggerConfig(tensorboard_dir=PosixPath('/fsx/loubna/br4-experiments/tensorboard/debug'))), [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: tokens=TokensArgs(sequence_length=8192, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: train_steps=150000, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: micro_batch_size=1, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: batch_accumulation_per_replica=1, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: val_check_interval=2500, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: limit_val_batches=2, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: limit_test_batches=0), [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: optimizer=OptimizerArgs(zero_stage=0, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: weight_decay=0.1, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: clip_grad=1.0, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: accumulate_grad_in_fp32=True, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: adam_eps=1e-08, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: adam_beta1=0.9, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: adam_beta2=0.95, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: learning_rate=0.0003), [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: learning_rate_scheduler=LRSchedulerArgs(lr_warmup_steps=2000, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: lr_warmup_style='linear', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: lr_decay_style='cosine', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: lr_decay_steps=150000, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: min_decay_lr=3e-05), [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: data=DataArgs(seed=1234, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: num_loading_workers=2, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: dataset=PretrainNemoArgs(data_prefix=[3.0, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.01, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 53.89, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 1.78, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.85, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 5.68, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.01, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 1.31, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.98, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.08, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.03, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.09, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 1.12, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 23.78, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.7, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.61, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.26, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 1.68, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 2.23, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.3, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.31, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.45, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.12, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 6.81, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 9.11, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.06, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 44.66, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.58, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 2.23, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.01, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 1.25, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 1.03, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 1.31, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 2.87, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.01, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.05, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 3.32, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.03, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.19, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.39, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 5.2, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.02, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 1.56, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.01, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.07, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.41, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 3.66, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.56, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.03, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.001, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.23, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.02, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.01, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 4.69, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.35, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.33, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.01, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 3.09, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.46, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.2, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.05, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.04, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 11.09, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.4, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.3, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.42, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 48.92, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.64, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 1.4, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.71, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.91, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 29.36, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 86.94, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 64.71, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 74.93, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 60.89, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 60.4, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 26.52, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.001, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 1.42, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.94, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.01, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.0002, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.11, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.18, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 0.05, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 1.0, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 1.0, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 54.4, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 32.0, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 7.12, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: 6.0, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document'], [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: index_mapping_dir=None, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: splits_string='0.969,0.999,1', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: skip_warmup=True, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: dataloader_type='single', [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: validation_drop_last=True, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: eod_mask_loss=False, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: no_seqlen_plus_one_input_tokens=False, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: pad_samples_to_global_batch_size=False))) [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: GPTBigCodeConfig { [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "activation_function": "gelu", [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "attention_softmax_in_fp32": true, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "attn_pdrop": 0.1, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "bos_token_id": 50256, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "embd_pdrop": 0.1, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "eos_token_id": 50256, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "hf_gpt2_model_name": "/fsx/loubna/starcoder-tokenizer/15b", [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "initializer_range": 0.02, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "layer_norm_epsilon": 1e-05, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "make_vocab_size_divisible_by": 128, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "model_type": "gpt_bigcode", [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "multi_query": true, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "n_embd": 2048, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "n_head": 16, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "n_inner": 8192, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "n_layer": 24, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "n_positions": 8192, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "pad_key_length": true, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "resid_pdrop": 0.1, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "scale_attention_softmax_in_fp32": true, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "scale_attn_weights": true, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "seed": 42, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "transformers_version": "4.30.2", [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "use_cache": true, [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: "vocab_size": 49152 [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: } [ip-26-0-154-121:0]:06/21/2023 17:12:07 [INFO|DP=0|PP=0|TP=0]: [ip-26-0-154-121:0]:06/21/2023 17:12:14 [INFO|DP=0|PP=0|TP=0]: Number of parameters: 1137207296 (2169.05MB). Expecting peak 4*param_size=8676.20MB with grads and Adam optim states (w/o memory optims) [ip-26-0-154-121:0]:06/21/2023 17:12:14 [INFO|DP=0|PP=0|TP=0]: [After model building] Memory usage: 2169.06MB. Peak reserved memory: 2434.00MB [ip-26-0-154-80:2]:06/21/2023 17:12:17 [INFO|DP=50|PP=0|TP=0]: Global rank: 50/64 | PP: 0/1 | DP: 50/64 | TP: 0/1 [ip-26-0-154-80:4]:06/21/2023 17:12:17 [INFO|DP=52|PP=0|TP=0]: Global rank: 52/64 | PP: 0/1 | DP: 52/64 | TP: 0/1 [ip-26-0-154-80:7]:06/21/2023 17:12:17 [INFO|DP=55|PP=0|TP=0]: Global rank: 55/64 | PP: 0/1 | DP: 55/64 | TP: 0/1 [ip-26-0-154-80:5]:06/21/2023 17:12:17 [INFO|DP=53|PP=0|TP=0]: Global rank: 53/64 | PP: 0/1 | DP: 53/64 | TP: 0/1 [ip-26-0-154-80:3]:06/21/2023 17:12:17 [INFO|DP=51|PP=0|TP=0]: Global rank: 51/64 | PP: 0/1 | DP: 51/64 | TP: 0/1 [ip-26-0-154-80:1]:06/21/2023 17:12:17 [INFO|DP=49|PP=0|TP=0]: Global rank: 49/64 | PP: 0/1 | DP: 49/64 | TP: 0/1 [ip-26-0-154-155:5]:06/21/2023 17:12:17 [INFO|DP=29|PP=0|TP=0]: Global rank: 29/64 | PP: 0/1 | DP: 29/64 | TP: 0/1 [ip-26-0-154-155:3]:06/21/2023 17:12:17 [INFO|DP=27|PP=0|TP=0]: Global rank: 27/64 | PP: 0/1 | DP: 27/64 | TP: 0/1 [ip-26-0-154-80:6]:06/21/2023 17:12:17 [INFO|DP=54|PP=0|TP=0]: Global rank: 54/64 | PP: 0/1 | DP: 54/64 | TP: 0/1 [ip-26-0-154-144:5]:06/21/2023 17:12:17 [INFO|DP=21|PP=0|TP=0]: Global rank: 21/64 | PP: 0/1 | DP: 21/64 | TP: 0/1 [ip-26-0-154-144:7]:06/21/2023 17:12:17 [INFO|DP=23|PP=0|TP=0]: Global rank: 23/64 | PP: 0/1 | DP: 23/64 | TP: 0/1 [ip-26-0-154-80:0]:06/21/2023 17:12:17 [INFO|DP=48|PP=0|TP=0]: Global rank: 48/64 | PP: 0/1 | DP: 48/64 | TP: 0/1 [ip-26-0-154-155:7]:06/21/2023 17:12:17 [INFO|DP=31|PP=0|TP=0]: Global rank: 31/64 | PP: 0/1 | DP: 31/64 | TP: 0/1 [ip-26-0-154-144:6]:06/21/2023 17:12:17 [INFO|DP=22|PP=0|TP=0]: Global rank: 22/64 | PP: 0/1 | DP: 22/64 | TP: 0/1 [ip-26-0-154-144:2]:06/21/2023 17:12:17 [INFO|DP=18|PP=0|TP=0]: Global rank: 18/64 | PP: 0/1 | DP: 18/64 | TP: 0/1 [ip-26-0-154-155:1]:06/21/2023 17:12:17 [INFO|DP=25|PP=0|TP=0]: Global rank: 25/64 | PP: 0/1 | DP: 25/64 | TP: 0/1 [ip-26-0-154-144:1]:06/21/2023 17:12:17 [INFO|DP=17|PP=0|TP=0]: Global rank: 17/64 | PP: 0/1 | DP: 17/64 | TP: 0/1 [ip-26-0-154-155:0]:06/21/2023 17:12:17 [INFO|DP=24|PP=0|TP=0]: Global rank: 24/64 | PP: 0/1 | DP: 24/64 | TP: 0/1 [ip-26-0-154-144:4]:06/21/2023 17:12:17 [INFO|DP=20|PP=0|TP=0]: Global rank: 20/64 | PP: 0/1 | DP: 20/64 | TP: 0/1 [ip-26-0-154-155:6]:06/21/2023 17:12:17 [INFO|DP=30|PP=0|TP=0]: Global rank: 30/64 | PP: 0/1 | DP: 30/64 | TP: 0/1 [ip-26-0-154-144:3]:06/21/2023 17:12:17 [INFO|DP=19|PP=0|TP=0]: Global rank: 19/64 | PP: 0/1 | DP: 19/64 | TP: 0/1 [ip-26-0-154-155:2]:06/21/2023 17:12:17 [INFO|DP=26|PP=0|TP=0]: Global rank: 26/64 | PP: 0/1 | DP: 26/64 | TP: 0/1 [ip-26-0-154-144:0]:06/21/2023 17:12:17 [INFO|DP=16|PP=0|TP=0]: Global rank: 16/64 | PP: 0/1 | DP: 16/64 | TP: 0/1 [ip-26-0-154-155:4]:06/21/2023 17:12:17 [INFO|DP=28|PP=0|TP=0]: Global rank: 28/64 | PP: 0/1 | DP: 28/64 | TP: 0/1 [ip-26-0-155-33:1]:06/21/2023 17:12:17 [INFO|DP=57|PP=0|TP=0]: Global rank: 57/64 | PP: 0/1 | DP: 57/64 | TP: 0/1 [ip-26-0-154-245:0]:06/21/2023 17:12:17 [INFO|DP=40|PP=0|TP=0]: Global rank: 40/64 | PP: 0/1 | DP: 40/64 | TP: 0/1 [ip-26-0-154-245:6]:06/21/2023 17:12:17 [INFO|DP=46|PP=0|TP=0]: Global rank: 46/64 | PP: 0/1 | DP: 46/64 | TP: 0/1 [ip-26-0-154-245:4]:06/21/2023 17:12:17 [INFO|DP=44|PP=0|TP=0]: Global rank: 44/64 | PP: 0/1 | DP: 44/64 | TP: 0/1 [ip-26-0-154-245:1]:06/21/2023 17:12:17 [INFO|DP=41|PP=0|TP=0]: Global rank: 41/64 | PP: 0/1 | DP: 41/64 | TP: 0/1 [ip-26-0-154-245:3]:06/21/2023 17:12:17 [INFO|DP=43|PP=0|TP=0]: Global rank: 43/64 | PP: 0/1 | DP: 43/64 | TP: 0/1 [ip-26-0-154-138:0]:06/21/2023 17:12:17 [INFO|DP=8|PP=0|TP=0]: Global rank: 8/64 | PP: 0/1 | DP: 8/64 | TP: 0/1 [ip-26-0-154-245:2]:06/21/2023 17:12:17 [INFO|DP=42|PP=0|TP=0]: Global rank: 42/64 | PP: 0/1 | DP: 42/64 | TP: 0/1 [ip-26-0-154-245:7]:06/21/2023 17:12:17 [INFO|DP=47|PP=0|TP=0]: Global rank: 47/64 | PP: 0/1 | DP: 47/64 | TP: 0/1 [ip-26-0-154-245:5]:06/21/2023 17:12:17 [INFO|DP=45|PP=0|TP=0]: Global rank: 45/64 | PP: 0/1 | DP: 45/64 | TP: 0/1 [ip-26-0-154-138:2]:06/21/2023 17:12:17 [INFO|DP=10|PP=0|TP=0]: Global rank: 10/64 | PP: 0/1 | DP: 10/64 | TP: 0/1 [ip-26-0-155-33:4]:06/21/2023 17:12:17 [INFO|DP=60|PP=0|TP=0]: Global rank: 60/64 | PP: 0/1 | DP: 60/64 | TP: 0/1 [ip-26-0-154-138:4]:06/21/2023 17:12:17 [INFO|DP=12|PP=0|TP=0]: Global rank: 12/64 | PP: 0/1 | DP: 12/64 | TP: 0/1 [ip-26-0-154-138:5]:06/21/2023 17:12:17 [INFO|DP=13|PP=0|TP=0]: Global rank: 13/64 | PP: 0/1 | DP: 13/64 | TP: 0/1 [ip-26-0-155-33:5]:06/21/2023 17:12:17 [INFO|DP=61|PP=0|TP=0]: Global rank: 61/64 | PP: 0/1 | DP: 61/64 | TP: 0/1 [ip-26-0-154-138:3]:06/21/2023 17:12:17 [INFO|DP=11|PP=0|TP=0]: Global rank: 11/64 | PP: 0/1 | DP: 11/64 | TP: 0/1 [ip-26-0-154-138:6]:06/21/2023 17:12:17 [INFO|DP=14|PP=0|TP=0]: Global rank: 14/64 | PP: 0/1 | DP: 14/64 | TP: 0/1 [ip-26-0-154-138:1]:06/21/2023 17:12:17 [INFO|DP=9|PP=0|TP=0]: Global rank: 9/64 | PP: 0/1 | DP: 9/64 | TP: 0/1 [ip-26-0-154-138:7]:06/21/2023 17:12:17 [INFO|DP=15|PP=0|TP=0]: Global rank: 15/64 | PP: 0/1 | DP: 15/64 | TP: 0/1 [ip-26-0-154-185:0]:06/21/2023 17:12:17 [INFO|DP=32|PP=0|TP=0]: Global rank: 32/64 | PP: 0/1 | DP: 32/64 | TP: 0/1 [ip-26-0-154-185:5]:06/21/2023 17:12:17 [INFO|DP=37|PP=0|TP=0]: Global rank: 37/64 | PP: 0/1 | DP: 37/64 | TP: 0/1 [ip-26-0-155-33:2]:06/21/2023 17:12:17 [INFO|DP=58|PP=0|TP=0]: Global rank: 58/64 | PP: 0/1 | DP: 58/64 | TP: 0/1 [ip-26-0-154-185:1]:06/21/2023 17:12:17 [INFO|DP=33|PP=0|TP=0]: Global rank: 33/64 | PP: 0/1 | DP: 33/64 | TP: 0/1 [ip-26-0-154-185:2]:06/21/2023 17:12:17 [INFO|DP=34|PP=0|TP=0]: Global rank: 34/64 | PP: 0/1 | DP: 34/64 | TP: 0/1 [ip-26-0-154-185:7]:06/21/2023 17:12:17 [INFO|DP=39|PP=0|TP=0]: Global rank: 39/64 | PP: 0/1 | DP: 39/64 | TP: 0/1 [ip-26-0-154-185:4]:06/21/2023 17:12:17 [INFO|DP=36|PP=0|TP=0]: Global rank: 36/64 | PP: 0/1 | DP: 36/64 | TP: 0/1 [ip-26-0-154-185:3]:06/21/2023 17:12:17 [INFO|DP=35|PP=0|TP=0]: Global rank: 35/64 | PP: 0/1 | DP: 35/64 | TP: 0/1 [ip-26-0-155-33:3]:06/21/2023 17:12:17 [INFO|DP=59|PP=0|TP=0]: Global rank: 59/64 | PP: 0/1 | DP: 59/64 | TP: 0/1 [ip-26-0-155-33:6]:06/21/2023 17:12:17 [INFO|DP=62|PP=0|TP=0]: Global rank: 62/64 | PP: 0/1 | DP: 62/64 | TP: 0/1 [ip-26-0-154-185:6]:06/21/2023 17:12:17 [INFO|DP=38|PP=0|TP=0]: Global rank: 38/64 | PP: 0/1 | DP: 38/64 | TP: 0/1 [ip-26-0-155-33:7]:06/21/2023 17:12:17 [INFO|DP=63|PP=0|TP=0]: Global rank: 63/64 | PP: 0/1 | DP: 63/64 | TP: 0/1 [ip-26-0-154-121:6]:06/21/2023 17:12:17 [INFO|DP=6|PP=0|TP=0]: Global rank: 6/64 | PP: 0/1 | DP: 6/64 | TP: 0/1 [ip-26-0-154-121:7]:06/21/2023 17:12:17 [INFO|DP=7|PP=0|TP=0]: Global rank: 7/64 | PP: 0/1 | DP: 7/64 | TP: 0/1 [ip-26-0-154-121:3]:06/21/2023 17:12:17 [INFO|DP=3|PP=0|TP=0]: Global rank: 3/64 | PP: 0/1 | DP: 3/64 | TP: 0/1 [ip-26-0-155-33:0]:06/21/2023 17:12:17 [INFO|DP=56|PP=0|TP=0]: Global rank: 56/64 | PP: 0/1 | DP: 56/64 | TP: 0/1 [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: Global rank: 0/64 | PP: 0/1 | DP: 0/64 | TP: 0/1 [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: Using Nemo Dataloader [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: Building GPT datasets. [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.019344 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: number of documents: 2721616 [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 888559) total of 888559 documents [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: document indices in [888559, 1804629) total of 916070 documents [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: document indices in [1804629, 2721616) total of 916987 documents [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:2]:06/21/2023 17:12:17 [INFO|DP=2|PP=0|TP=0]: Global rank: 2/64 | PP: 0/1 | DP: 2/64 | TP: 0/1 [ip-26-0-154-121:4]:06/21/2023 17:12:17 [INFO|DP=4|PP=0|TP=0]: Global rank: 4/64 | PP: 0/1 | DP: 4/64 | TP: 0/1 [ip-26-0-154-121:1]:06/21/2023 17:12:17 [INFO|DP=1|PP=0|TP=0]: Global rank: 1/64 | PP: 0/1 | DP: 1/64 | TP: 0/1 [ip-26-0-154-121:5]:06/21/2023 17:12:17 [INFO|DP=5|PP=0|TP=0]: Global rank: 5/64 | PP: 0/1 | DP: 5/64 | TP: 0/1 [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.040789 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 888559 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 192825 [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.074311 [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 192825) and [192825, 192825) ... [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.007099 [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_train_indexmap_37739ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_train_indexmap_37739ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_train_indexmap_37739ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: total number of samples: 192826 [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.036178 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 916070 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 197783 [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.049154 [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 197783) and [197783, 197783) ... [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.010081 [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_valid_indexmap_31ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_valid_indexmap_31ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_valid_indexmap_31ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: total number of samples: 197784 [ip-26-0-154-121:0]:06/21/2023 17:12:17 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.027 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: total number of samples: 197931 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005553 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: number of documents: 968 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 316) total of 316 documents [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: document indices in [316, 642) total of 326 documents [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: document indices in [642, 968) total of 326 documents [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002982 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 316 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 167 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.042372 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 167) and [167, 167) ... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002017 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: total number of samples: 168 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: total number of samples: 135 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: total number of samples: 157 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.015965 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: number of documents: 8536791 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2787113) total of 2787113 documents [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: document indices in [2787113, 5660514) total of 2873401 documents [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: document indices in [5660514, 8536791) total of 2876277 documents [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.120176 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 2787113 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 809627 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.065953 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 809627) and [809627, 809627) ... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.024930 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_train_indexmap_677919ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_train_indexmap_677919ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_train_indexmap_677919ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: total number of samples: 809628 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.118043 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 2873401 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 833053 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.072435 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 833053) and [833053, 833053) ... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.027259 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_valid_indexmap_552ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_valid_indexmap_552ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_valid_indexmap_552ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: total number of samples: 833054 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.035 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: total number of samples: 843540 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.014100 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: number of documents: 158792 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 51843) total of 51843 documents [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: document indices in [51843, 105291) total of 53448 documents [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: document indices in [105291, 158792) total of 53501 documents [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.005362 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 51843 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 26874 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040846 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 26874) and [26874, 26874) ... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003032 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_train_indexmap_22392ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_train_indexmap_22392ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_train_indexmap_22392ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: total number of samples: 26875 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:18 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.004622 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 53448 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 27837 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040357 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 27837) and [27837, 27837) ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003572 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_valid_indexmap_19ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_valid_indexmap_19ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_valid_indexmap_19ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of samples: 27838 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.014 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of samples: 27546 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.013027 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: number of documents: 153194 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 50015) total of 50015 documents [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: document indices in [50015, 101579) total of 51564 documents [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: document indices in [101579, 153194) total of 51615 documents [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.005175 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 50015 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 11352 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040000 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 11352) and [11352, 11352) ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002614 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_train_indexmap_10693ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_train_indexmap_10693ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_train_indexmap_10693ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of samples: 11353 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003594 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 51564 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 11372 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.042131 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 11372) and [11372, 11372) ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002669 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_valid_indexmap_9ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_valid_indexmap_9ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_valid_indexmap_9ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of samples: 11373 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.014 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of samples: 12126 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.016184 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: number of documents: 2239354 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 731110) total of 731110 documents [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: document indices in [731110, 1484855) total of 753745 documents [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: document indices in [1484855, 2239354) total of 754499 documents [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (13675) is smaller than 80% of number of samples per epoch (57778), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.055070 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 731110 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 115557 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.046816 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 57778) and [57778, 115557) ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.005101 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_train_indexmap_71453ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_train_indexmap_71453ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_train_indexmap_71453ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of samples: 115558 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.027755 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 753745 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 59021 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.043461 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 59021) and [59021, 59021) ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003085 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_valid_indexmap_59ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_valid_indexmap_59ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_valid_indexmap_59ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of samples: 59022 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.021 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of samples: 59162 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.007206 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: number of documents: 523 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 171) total of 171 documents [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: document indices in [171, 347) total of 176 documents [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: document indices in [347, 523) total of 176 documents [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (42) is smaller than 80% of number of samples per epoch (84), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002534 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 171 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 169 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.038796 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 84) and [84, 169) ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001551 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of samples: 170 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of samples: 82 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of samples: 75 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.015987 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: number of documents: 295364 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 96431) total of 96431 documents [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: document indices in [96431, 195848) total of 99417 documents [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: document indices in [195848, 295364) total of 99516 documents [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.005373 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 96431 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 19290 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039761 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 19290) and [19290, 19290) ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002961 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_train_indexmap_16480ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_train_indexmap_16480ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_train_indexmap_16480ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of samples: 19291 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.004926 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 99417 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 19482 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.038933 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 19482) and [19482, 19482) ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003592 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_valid_indexmap_14ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_valid_indexmap_14ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_valid_indexmap_14ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of samples: 19483 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of samples: 19724 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.016289 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: number of documents: 210816 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 68827) total of 68827 documents [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: document indices in [68827, 139786) total of 70959 documents [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: document indices in [139786, 210816) total of 71030 documents [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (740) is smaller than 80% of number of samples per epoch (11589), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.007237 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 68827 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 23178 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040140 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 11589) and [11589, 23178) ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002931 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_train_indexmap_12329ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_train_indexmap_12329ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_train_indexmap_12329ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of samples: 23179 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:19 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.004489 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 70959 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 11791 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039277 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 11791) and [11791, 11791) ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002164 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of samples: 11792 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.014 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of samples: 11881 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.008863 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: number of documents: 5001 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1633) total of 1633 documents [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: document indices in [1633, 3316) total of 1683 documents [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: document indices in [3316, 5001) total of 1685 documents [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002855 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1633 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 1282 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.038609 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 1282) and [1282, 1282) ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002757 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_train_indexmap_1007ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_train_indexmap_1007ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_train_indexmap_1007ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1283 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.014 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1255 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1256 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.008362 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: number of documents: 8042 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2625) total of 2625 documents [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: document indices in [2625, 5332) total of 2707 documents [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: document indices in [5332, 8042) total of 2710 documents [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002700 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 2625 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 398 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.038350 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 398) and [398, 398) ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002150 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_train_indexmap_378ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_train_indexmap_378ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_train_indexmap_378ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of samples: 399 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of samples: 397 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of samples: 395 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.009227 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: number of documents: 16870 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 5508) total of 5508 documents [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: document indices in [5508, 11186) total of 5678 documents [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: document indices in [11186, 16870) total of 5684 documents [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003033 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 5508 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 1570 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039729 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 1570) and [1570, 1570) ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001839 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_train_indexmap_1133ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_train_indexmap_1133ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_train_indexmap_1133ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1571 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1568 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1522 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.015213 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: number of documents: 267627 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 87375) total of 87375 documents [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: document indices in [87375, 177456) total of 90081 documents [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: document indices in [177456, 267627) total of 90171 documents [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (2800) is smaller than 80% of number of samples per epoch (11290), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.007729 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 87375 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 22580 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040785 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 11290) and [11290, 22580) ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002498 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_train_indexmap_14090ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_train_indexmap_14090ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_train_indexmap_14090ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of samples: 22581 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.004585 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 90081 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 12029 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039526 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 12029) and [12029, 12029) ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001880 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_valid_indexmap_12ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_valid_indexmap_12ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_valid_indexmap_12ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of samples: 12030 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of samples: 11686 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.015449 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: number of documents: 4700526 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1534640) total of 1534640 documents [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: document indices in [1534640, 3116791) total of 1582151 documents [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: document indices in [3116791, 4700526) total of 1583735 documents [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.055524 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1534640 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 339913 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.054533 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 339913) and [339913, 339913) ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.010395 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_train_indexmap_299145ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_train_indexmap_299145ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_train_indexmap_299145ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of samples: 339914 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.059681 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1582151 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 350498 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.051186 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 350498) and [350498, 350498) ... [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.010320 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_valid_indexmap_244ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_valid_indexmap_244ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_valid_indexmap_244ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of samples: 350499 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:20 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.029 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of samples: 349999 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.010305 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: number of documents: 98447 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 32142) total of 32142 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [32142, 65278) total of 33136 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [65278, 98447) total of 33169 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003455 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 32142 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 8896 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041036 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 8896) and [8896, 8896) ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001787 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_train_indexmap_8806ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_train_indexmap_8806ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_train_indexmap_8806ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.043 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of samples: 8897 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003296 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 33136 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 9454 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.038220 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 9454) and [9454, 9454) ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001789 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9455 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9199 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.011075 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: number of documents: 124066 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 40506) total of 40506 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [40506, 82265) total of 41759 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [82265, 124066) total of 41801 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (1196) is smaller than 80% of number of samples per epoch (6478), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.005106 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 40506 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 12956 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041826 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 6478) and [6478, 12956) ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002413 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_train_indexmap_7674ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_train_indexmap_7674ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_train_indexmap_7674ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of samples: 12957 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003097 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 41759 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 6942 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.038313 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 6942) and [6942, 6942) ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001834 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6943 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6521 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.010775 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: number of documents: 30934 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 10099) total of 10099 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [10099, 20511) total of 10412 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [20511, 30934) total of 10423 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (473) is smaller than 80% of number of samples per epoch (2798), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003340 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 10099 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 5596 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039083 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 2798) and [2798, 5596) ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002939 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_train_indexmap_3271ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_train_indexmap_3271ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_train_indexmap_3271ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5597 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002064 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 10412 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 2938 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.038679 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 2938) and [2938, 2938) ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001752 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_valid_indexmap_3ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_valid_indexmap_3ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_valid_indexmap_3ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2939 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2818 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.013994 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: number of documents: 110981 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 36233) total of 36233 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [36233, 73588) total of 37355 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [73588, 110981) total of 37393 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003364 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 36233 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 26802 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039934 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 26802) and [26802, 26802) ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003137 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_train_indexmap_21134ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_train_indexmap_21134ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_train_indexmap_21134ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of samples: 26803 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002844 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 37355 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 29259 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.038964 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 29259) and [29259, 29259) ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003608 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_valid_indexmap_18ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_valid_indexmap_18ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_valid_indexmap_18ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of samples: 29260 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.015 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of samples: 27777 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.015504 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: number of documents: 365491 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 119326) total of 119326 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [119326, 242347) total of 123021 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [242347, 365491) total of 123144 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.006073 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 119326 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 32165 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039686 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 32165) and [32165, 32165) ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003060 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_train_indexmap_28053ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_train_indexmap_28053ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_train_indexmap_28053ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of samples: 32166 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.005684 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 123021 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 33790 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039822 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 33790) and [33790, 33790) ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002354 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of samples: 33791 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.015 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of samples: 33144 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.011200 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: number of documents: 39042 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 12747) total of 12747 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [12747, 25888) total of 13141 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: document indices in [25888, 39042) total of 13154 documents [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003367 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 12747 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 4178 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039485 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 4178) and [4178, 4178) ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001824 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_train_indexmap_3774ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_train_indexmap_3774ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_train_indexmap_3774ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4179 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002196 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 13141 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 4309 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.038429 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 4309) and [4309, 4309) ... [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001697 [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:21 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4310 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4206 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.013542 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: number of documents: 97167 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 31724) total of 31724 documents [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: document indices in [31724, 64429) total of 32705 documents [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: document indices in [64429, 97167) total of 32738 documents [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003548 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 31724 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 3987 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039047 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 3987) and [3987, 3987) ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002063 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_train_indexmap_3900ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_train_indexmap_3900ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_train_indexmap_3900ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of samples: 3988 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003350 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 32705 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 4020 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.038234 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 4020) and [4020, 4020) ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001688 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4021 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4218 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.014498 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: number of documents: 186375 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 60848) total of 60848 documents [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: document indices in [60848, 123580) total of 62732 documents [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: document indices in [123580, 186375) total of 62795 documents [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.004812 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 60848 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 6094 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039533 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 6094) and [6094, 6094) ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001903 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_train_indexmap_5661ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_train_indexmap_5661ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_train_indexmap_5661ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6095 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003603 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 62732 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 6103 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.037956 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 6103) and [6103, 6103) ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001750 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6104 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.014 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6249 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.006883 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: number of documents: 9226 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 3013) total of 3013 documents [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: document indices in [3013, 6118) total of 3105 documents [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: document indices in [6118, 9226) total of 3108 documents [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003075 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 3013 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 2234 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039052 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 2234) and [2234, 2234) ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002837 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_train_indexmap_1510ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_train_indexmap_1510ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_train_indexmap_1510ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2235 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1942 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2227 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.014775 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: number of documents: 3390320 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1106880) total of 1106880 documents [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: document indices in [1106880, 2248029) total of 1141149 documents [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: document indices in [2248029, 3390320) total of 1142291 documents [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (4719) is smaller than 80% of number of samples per epoch (80949), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.084155 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1106880 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 161898 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.051641 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 80949) and [80949, 161898) ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.006199 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_train_indexmap_85668ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_train_indexmap_85668ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_train_indexmap_85668ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of samples: 161899 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.039818 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1141149 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 81429 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.046223 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 81429) and [81429, 81429) ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.004169 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_valid_indexmap_70ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_valid_indexmap_70ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_valid_indexmap_70ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of samples: 81430 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.024 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of samples: 82121 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.015425 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: number of documents: 1380468 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 450699) total of 450699 documents [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: document indices in [450699, 915351) total of 464652 documents [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: document indices in [915351, 1380468) total of 465117 documents [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (7508) is smaller than 80% of number of samples per epoch (107093), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.031644 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 450699 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 214186 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.046808 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 107093) and [107093, 214186) ... [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.007294 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_train_indexmap_114601ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_train_indexmap_114601ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_train_indexmap_114601ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of samples: 214187 [ip-26-0-154-121:0]:06/21/2023 17:12:22 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.016549 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 464652 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 110421 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.043341 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 110421) and [110421, 110421) ... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.004548 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_valid_indexmap_94ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_valid_indexmap_94ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_valid_indexmap_94ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: total number of samples: 110422 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.032 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: total number of samples: 110680 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.009880 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: number of documents: 5386 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1758) total of 1758 documents [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: document indices in [1758, 3571) total of 1813 documents [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: document indices in [3571, 5386) total of 1815 documents [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002834 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1758 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 791 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039597 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 791) and [791, 791) ... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001604 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_train_indexmap_755ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_train_indexmap_755ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_train_indexmap_755ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: total number of samples: 792 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: total number of samples: 816 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: total number of samples: 776 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.017622 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: number of documents: 10801285 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 3526430) total of 3526430 documents [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: document indices in [3526430, 7162038) total of 3635608 documents [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: document indices in [7162038, 10801285) total of 3639247 documents [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (143800) is smaller than 80% of number of samples per epoch (418008), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.308230 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 3526430 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 836016 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.097770 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 418008) and [418008, 836016) ... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.023783 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_train_indexmap_561808ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_train_indexmap_561808ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_train_indexmap_561808ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: total number of samples: 836017 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:23 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.157491 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 3635608 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 430671 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.070204 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 430671) and [430671, 430671) ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.013533 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_valid_indexmap_457ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_valid_indexmap_457ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_valid_indexmap_457ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of samples: 430672 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.032 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of samples: 429969 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.015377 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: number of documents: 587748 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 191890) total of 191890 documents [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: document indices in [191890, 389720) total of 197830 documents [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: document indices in [389720, 587748) total of 198028 documents [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.009643 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 191890 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 7892 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040668 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 7892) and [7892, 7892) ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002042 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_train_indexmap_7297ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_train_indexmap_7297ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_train_indexmap_7297ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7893 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.008148 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 197830 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 7944 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040548 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 7944) and [7944, 7944) ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002059 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7945 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.016 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of samples: 8334 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.016107 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: number of documents: 541454 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 176775) total of 176775 documents [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: document indices in [176775, 359023) total of 182248 documents [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: document indices in [359023, 541454) total of 182431 documents [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (1844) is smaller than 80% of number of samples per epoch (26209), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.014054 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 176775 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 52418 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041968 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 26209) and [26209, 52418) ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003102 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_train_indexmap_28053ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_train_indexmap_28053ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_train_indexmap_28053ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of samples: 52419 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.007956 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 182248 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 26826 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039850 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 26826) and [26826, 26826) ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002904 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of samples: 26827 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.015 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of samples: 26582 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005343 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: number of documents: 1152 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 376) total of 376 documents [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: document indices in [376, 764) total of 388 documents [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: document indices in [764, 1152) total of 388 documents [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (57) is larger than 80% of number of samples per epoch (69), setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002414 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 376 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 138 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039026 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 138) and [138, 138) ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002030 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of samples: 139 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of samples: 71 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of samples: 58 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.008517 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: number of documents: 22653 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 7396) total of 7396 documents [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: document indices in [7396, 15021) total of 7625 documents [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: document indices in [15021, 22653) total of 7632 documents [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002807 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 7396 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 19788 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.038891 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 19788) and [19788, 19788) ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002362 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_train_indexmap_15725ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_train_indexmap_15725ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_train_indexmap_15725ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of samples: 19789 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002370 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 7625 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 21127 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039249 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 21127) and [21127, 21127) ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002065 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_valid_indexmap_13ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_valid_indexmap_13ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_valid_indexmap_13ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of samples: 21128 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.014 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of samples: 21473 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.015709 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: number of documents: 158356 [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 51701) total of 51701 documents [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: document indices in [51701, 105002) total of 53301 documents [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: document indices in [105002, 158356) total of 53354 documents [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:24 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.004129 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 51701 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 13633 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041548 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 13633) and [13633, 13633) ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002445 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_train_indexmap_12958ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_train_indexmap_12958ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_train_indexmap_12958ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 13634 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.004094 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 53301 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 13486 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040429 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 13486) and [13486, 13486) ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002313 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 13487 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.014 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 13675 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.016197 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: number of documents: 657349 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 214613) total of 214613 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [214613, 435870) total of 221257 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [435870, 657349) total of 221479 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.009485 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 214613 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 19837 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.042067 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 19837) and [19837, 19837) ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003331 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_train_indexmap_16480ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_train_indexmap_16480ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_train_indexmap_16480ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 19838 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.009559 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 221257 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 20451 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040956 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 20451) and [20451, 20451) ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002328 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_valid_indexmap_14ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_valid_indexmap_14ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_valid_indexmap_14ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 20452 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.015 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 20653 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.014066 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: number of documents: 549459 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 179388) total of 179388 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [179388, 364331) total of 184943 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [364331, 549459) total of 185128 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.008433 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 179388 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 40967 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.042248 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 40967) and [40967, 40967) ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003399 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_train_indexmap_36104ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_train_indexmap_36104ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_train_indexmap_36104ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 40968 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.007507 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 184943 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 41599 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041215 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 41599) and [41599, 41599) ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002726 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_valid_indexmap_30ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_valid_indexmap_30ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_valid_indexmap_30ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 41600 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.017 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 42207 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.007387 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: number of documents: 1133 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 370) total of 370 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [370, 751) total of 381 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [751, 1133) total of 382 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (23) is smaller than 80% of number of samples per epoch (51), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002979 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 370 [ip-26-0-154-121:0]: number of epochs: 3 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 154 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039402 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 103) and [103, 154) ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001682 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 155 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 3 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 53 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 49 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.008261 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: number of documents: 6104 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1992) total of 1992 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [1992, 4047) total of 2055 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [4047, 6104) total of 2057 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003335 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1992 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 670 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039098 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 670) and [670, 670) ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002324 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_train_indexmap_629ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_train_indexmap_629ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_train_indexmap_629ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 671 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 696 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 660 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.016068 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: number of documents: 896880 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 292816) total of 292816 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [292816, 594697) total of 301881 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [594697, 896880) total of 302183 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.011787 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 292816 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 42155 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.042488 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 42155) and [42155, 42155) ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002971 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_train_indexmap_41765ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_train_indexmap_41765ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_train_indexmap_41765ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 42156 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.011248 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 301881 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 42766 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.042692 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 42766) and [42766, 42766) ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002996 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_valid_indexmap_34ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_valid_indexmap_34ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_valid_indexmap_34ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 42767 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.017 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 42618 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.008788 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: number of documents: 3688 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1204) total of 1204 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [1204, 2445) total of 1241 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: document indices in [2445, 3688) total of 1243 documents [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (82) is smaller than 80% of number of samples per epoch (296), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003132 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1204 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 593 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040969 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 296) and [296, 593) ... [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002573 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_train_indexmap_378ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_train_indexmap_378ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_train_indexmap_378ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of samples: 594 [ip-26-0-154-121:0]:06/21/2023 17:12:25 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 347 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 301 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.012083 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 19630 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 6409) total of 6409 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [6409, 13016) total of 6607 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [13016, 19630) total of 6614 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002471 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 6409 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 2746 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.038588 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 2746) and [2746, 2746) ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001692 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_train_indexmap_2391ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_train_indexmap_2391ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_train_indexmap_2391ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2747 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002105 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 6607 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 2811 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039790 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 2811) and [2811, 2811) ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001846 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2812 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2578 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.009995 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 46270 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 15106) total of 15106 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [15106, 30680) total of 15574 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [30680, 46270) total of 15590 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003096 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 15106 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 5869 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040236 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 5869) and [5869, 5869) ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002154 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_train_indexmap_4907ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_train_indexmap_4907ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_train_indexmap_4907ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5870 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002355 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 15574 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 6105 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.038791 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 6105) and [6105, 6105) ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001879 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6106 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6324 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.017333 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 522778 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 170678) total of 170678 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [170678, 346640) total of 175962 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [346640, 522778) total of 176138 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.008144 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 170678 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 74798 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041935 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 74798) and [74798, 74798) ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.004059 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_train_indexmap_65415ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_train_indexmap_65415ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_train_indexmap_65415ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 74799 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.007333 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 175962 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 78657 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.042574 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 78657) and [78657, 78657) ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.005215 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_valid_indexmap_54ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_valid_indexmap_54ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_valid_indexmap_54ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 78658 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.016 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 77461 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.011625 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 10289 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 3359) total of 3359 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [3359, 6822) total of 3463 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [6822, 10289) total of 3467 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002831 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 3359 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 311 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041198 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 311) and [311, 311) ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001558 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_train_indexmap_252ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_train_indexmap_252ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_train_indexmap_252ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 312 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 377 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 312 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.017630 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 247919 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 80941) total of 80941 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [80941, 164388) total of 83447 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [164388, 247919) total of 83531 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.005183 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 80941 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 32910 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040682 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 32910) and [32910, 32910) ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003234 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_train_indexmap_19625ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_train_indexmap_19625ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_train_indexmap_19625ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 32911 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.005134 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 83447 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 31935 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039552 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 31935) and [31935, 31935) ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003185 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_valid_indexmap_16ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_valid_indexmap_16ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_valid_indexmap_16ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 31936 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.014 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 32667 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.008399 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 5368 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1752) total of 1752 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [1752, 3559) total of 1807 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [3559, 5368) total of 1809 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003073 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1752 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 126 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039170 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 126) and [126, 126) ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002581 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 127 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 137 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 117 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.010865 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: number of documents: 17554 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 5731) total of 5731 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [5731, 11640) total of 5909 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: document indices in [11640, 17554) total of 5914 documents [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002666 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 5731 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 1375 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039516 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 1375) and [1375, 1375) ... [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001982 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_train_indexmap_881ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_train_indexmap_881ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_train_indexmap_881ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1376 [ip-26-0-154-121:0]:06/21/2023 17:12:26 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1328 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1277 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.010235 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 52838 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 17250) total of 17250 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [17250, 35035) total of 17785 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [35035, 52838) total of 17803 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (140) is smaller than 80% of number of samples per epoch (5018), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003367 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 17250 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 10037 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040694 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 5018) and [5018, 10037) ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001871 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_train_indexmap_5158ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_train_indexmap_5158ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_train_indexmap_5158ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 10038 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002447 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 17785 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 5152 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039819 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 5152) and [5152, 5152) ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001970 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5153 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5279 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.014934 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 928415 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 303112) total of 303112 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [303112, 615607) total of 312495 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [615607, 928415) total of 312808 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (8837) is smaller than 80% of number of samples per epoch (37205), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.021418 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 303112 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 74411 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.043542 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 37205) and [37205, 74411) ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003382 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_train_indexmap_46042ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_train_indexmap_46042ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_train_indexmap_46042ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 74412 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.012392 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 312495 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 38814 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041400 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 38814) and [38814, 38814) ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002997 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_valid_indexmap_38ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_valid_indexmap_38ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_valid_indexmap_38ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 38815 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.017 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 38423 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.010838 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 58151 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 18985) total of 18985 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [18985, 38558) total of 19573 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [38558, 58151) total of 19593 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002836 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 18985 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 7622 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041061 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 7622) and [7622, 7622) ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001913 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_train_indexmap_7045ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_train_indexmap_7045ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_train_indexmap_7045ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7623 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002349 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 19573 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 7891 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039272 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 7891) and [7891, 7891) ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002022 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7892 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7791 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.009178 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 5928 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1936) total of 1936 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [1936, 3931) total of 1995 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [3931, 5928) total of 1997 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002590 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1936 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 400 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040502 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 400) and [400, 400) ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001759 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_train_indexmap_378ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_train_indexmap_378ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_train_indexmap_378ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 401 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 370 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 510 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004973 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 180 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 58) total of 58 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [58, 119) total of 61 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [119, 180) total of 61 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (3) is smaller than 80% of number of samples per epoch (5), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002865 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 58 [ip-26-0-154-121:0]: number of epochs: 3 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 15 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040387 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 10) and [10, 15) ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001702 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_train_indexmap_13ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_train_indexmap_13ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_train_indexmap_13ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 16 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 3 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.014900 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 239568 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 78215) total of 78215 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [78215, 158851) total of 80636 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [158851, 239568) total of 80717 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.005106 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 78215 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 3570 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041451 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 3570) and [3570, 3570) ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001891 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_train_indexmap_2894ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_train_indexmap_2894ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_train_indexmap_2894ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 3571 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.004467 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 80636 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 4061 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040533 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 4061) and [4061, 4061) ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002009 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_valid_indexmap_3ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_valid_indexmap_3ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_valid_indexmap_3ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4062 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.015 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4032 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005880 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: number of documents: 4806 [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1569) total of 1569 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [1569, 3187) total of 1618 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: document indices in [3187, 4806) total of 1619 documents [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:27 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003148 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1569 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 261 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040179 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 261) and [261, 261) ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001783 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_train_indexmap_252ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_train_indexmap_252ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_train_indexmap_252ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 262 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 254 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 284 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005867 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 5429 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1773) total of 1773 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [1773, 3600) total of 1827 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [3600, 5429) total of 1829 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002874 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1773 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 197 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039869 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 197) and [197, 197) ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001553 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 198 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 238 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 215 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.015074 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 1355788 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 442641) total of 442641 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [442641, 898986) total of 456345 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [898986, 1355788) total of 456802 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (7316) is smaller than 80% of number of samples per epoch (51683), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.031134 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 442641 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 103366 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.047291 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 51683) and [51683, 103366) ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.004160 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_train_indexmap_58999ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_train_indexmap_58999ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_train_indexmap_58999ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 103367 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.016285 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 456345 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 53518 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.043190 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 53518) and [53518, 53518) ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002942 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_valid_indexmap_48ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_valid_indexmap_48ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_valid_indexmap_48ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 53519 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.025 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 53423 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.011048 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 49335 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 16107) total of 16107 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [16107, 32713) total of 16606 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [32713, 49335) total of 16622 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003083 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 16107 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 4798 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.042773 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 4798) and [4798, 4798) ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002627 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_train_indexmap_4403ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_train_indexmap_4403ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_train_indexmap_4403ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4799 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003406 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 16606 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 5122 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041388 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 5122) and [5122, 5122) ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001848 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5123 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.014 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5055 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.011625 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 24208 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 7904) total of 7904 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [7904, 16052) total of 8148 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [16052, 24208) total of 8156 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003822 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 7904 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 7505 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040780 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 7505) and [7505, 7505) ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002837 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_train_indexmap_4152ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_train_indexmap_4152ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_train_indexmap_4152ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7506 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.001989 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 8148 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 9093 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039331 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 9093) and [9093, 9093) ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002087 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9094 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9893 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.007632 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 4737 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1547) total of 1547 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [1547, 3141) total of 1594 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [3141, 4737) total of 1596 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (16) is smaller than 80% of number of samples per epoch (110), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003398 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1547 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 220 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041368 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 110) and [110, 220) ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001766 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 221 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 103 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 102 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.015878 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: number of documents: 2206327 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 720327) total of 720327 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [720327, 1462955) total of 742628 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: document indices in [1462955, 2206327) total of 743372 documents [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.026777 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 720327 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 43205 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.045207 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 43205) and [43205, 43205) ... [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003285 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_train_indexmap_38872ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_train_indexmap_38872ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_train_indexmap_38872ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of samples: 43206 [ip-26-0-154-121:0]:06/21/2023 17:12:28 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.026753 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 742628 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 44087 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.046084 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 44087) and [44087, 44087) ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003196 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_valid_indexmap_32ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_valid_indexmap_32ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_valid_indexmap_32ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 44088 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.020 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 44638 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.012310 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 125163 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 40863) total of 40863 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [40863, 82992) total of 42129 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [82992, 125163) total of 42171 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (412) is smaller than 80% of number of samples per epoch (5375), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.004939 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 40863 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 10751 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.043095 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 5375) and [5375, 10751) ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002687 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_train_indexmap_5787ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_train_indexmap_5787ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_train_indexmap_5787ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 10752 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003263 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 42129 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 5431 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040259 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 5431) and [5431, 5431) ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002401 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5432 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5607 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.010790 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 41890 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 13676) total of 13676 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [13676, 27776) total of 14100 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [27776, 41890) total of 14114 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003791 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 13676 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 2538 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040752 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 2538) and [2538, 2538) ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002127 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_train_indexmap_2516ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_train_indexmap_2516ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_train_indexmap_2516ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2539 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002204 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 14100 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 2582 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039966 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 2582) and [2582, 2582) ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001821 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_valid_indexmap_3ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_valid_indexmap_3ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_valid_indexmap_3ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2583 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2873 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.008055 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 7917 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2585) total of 2585 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [2585, 5250) total of 2665 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [5250, 7917) total of 2667 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002864 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 2585 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 751 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040136 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 751) and [751, 751) ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001929 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_train_indexmap_629ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_train_indexmap_629ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_train_indexmap_629ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 752 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 692 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.052 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 720 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.000711 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 13716 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 4478) total of 4478 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [4478, 9095) total of 4617 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [9095, 13716) total of 4621 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002825 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 4478 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 641 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040624 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 641) and [641, 641) ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002811 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_train_indexmap_504ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_train_indexmap_504ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_train_indexmap_504ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 642 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 678 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 622 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.015481 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 975420 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 318457) total of 318457 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [318457, 646774) total of 328317 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [646774, 975420) total of 328646 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.012969 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 318457 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 220815 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.045715 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 220815) and [220815, 220815) ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.006944 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_train_indexmap_139509ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_train_indexmap_139509ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_train_indexmap_139509ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 220816 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.012232 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 328317 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 220203 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.045517 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 220203) and [220203, 220203) ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.007220 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_valid_indexmap_114ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_valid_indexmap_114ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_valid_indexmap_114ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 220204 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.021 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of samples: 223151 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.012259 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: number of documents: 167701 [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 54751) total of 54751 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [54751, 111198) total of 56447 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: document indices in [111198, 167701) total of 56503 documents [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:29 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.004064 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 54751 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 6620 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.042403 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 6620) and [6620, 6620) ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003204 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_train_indexmap_5032ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_train_indexmap_5032ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_train_indexmap_5032ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6621 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003951 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 56447 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 7936 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040387 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 7936) and [7936, 7936) ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002399 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7937 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6885 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.012489 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: number of documents: 62033 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 20252) total of 20252 documents [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: document indices in [20252, 41132) total of 20880 documents [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: document indices in [41132, 62033) total of 20901 documents [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (690) is smaller than 80% of number of samples per epoch (3084), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.004588 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 20252 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 6168 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041666 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 3084) and [3084, 6168) ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001871 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_train_indexmap_3774ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_train_indexmap_3774ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_train_indexmap_3774ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6169 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002448 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 20880 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 3167 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040375 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 3167) and [3167, 3167) ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002108 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 3168 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2971 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.014665 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: number of documents: 571506 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 186587) total of 186587 documents [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: document indices in [186587, 378950) total of 192363 documents [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: document indices in [378950, 571506) total of 192556 documents [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.008447 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 186587 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 5871 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.042678 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 5871) and [5871, 5871) ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001743 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_train_indexmap_5284ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_train_indexmap_5284ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_train_indexmap_5284ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5872 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.007725 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 192363 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 6043 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.042800 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 6043) and [6043, 6043) ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001758 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6044 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.015 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6002 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.015919 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: number of documents: 6353527 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2074315) total of 2074315 documents [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: document indices in [2074315, 4212851) total of 2138536 documents [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: document indices in [4212851, 6353527) total of 2140676 documents [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.076111 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 2074315 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 644296 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.062721 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 644296) and [644296, 644296) ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.021374 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_train_indexmap_615398ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_train_indexmap_615398ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_train_indexmap_615398ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 644297 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.080027 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 2138536 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 662446 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.063692 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 662446) and [662446, 662446) ... [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.020606 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_valid_indexmap_501ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_valid_indexmap_501ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_valid_indexmap_501ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of samples: 662447 [ip-26-0-154-121:0]:06/21/2023 17:12:30 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.034 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 667385 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.014689 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 226209 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 73853) total of 73853 documents [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: document indices in [73853, 149993) total of 76140 documents [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: document indices in [149993, 226209) total of 76216 documents [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (704) is smaller than 80% of number of samples per epoch (7347), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.007499 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 73853 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 14694 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.043381 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 7347) and [7347, 14694) ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002776 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_train_indexmap_8051ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_train_indexmap_8051ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_train_indexmap_8051ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 14695 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.004647 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 76140 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 7543 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041095 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 7543) and [7543, 7543) ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002338 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7544 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7711 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.011835 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 98733 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 32234) total of 32234 documents [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: document indices in [32234, 65467) total of 33233 documents [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: document indices in [65467, 98733) total of 33266 documents [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003893 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 32234 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 18948 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.042408 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 18948) and [18948, 18948) ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002983 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_train_indexmap_17612ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_train_indexmap_17612ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_train_indexmap_17612ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 18949 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003084 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 33233 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 20791 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041089 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 20791) and [20791, 20791) ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002250 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_valid_indexmap_15ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_valid_indexmap_15ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_valid_indexmap_15ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 20792 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.014 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 20468 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.016013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 281016 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 91747) total of 91747 documents [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: document indices in [91747, 186334) total of 94587 documents [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: document indices in [186334, 281016) total of 94682 documents [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.005322 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 91747 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 8963 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.042201 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 8963) and [8963, 8963) ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002486 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_train_indexmap_8932ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_train_indexmap_8932ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_train_indexmap_8932ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 8964 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.004210 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 94587 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 8897 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.042801 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 8897) and [8897, 8897) ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003687 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 8898 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.014 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 8929 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.016876 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 250834 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 81893) total of 81893 documents [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: document indices in [81893, 166321) total of 84428 documents [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: document indices in [166321, 250834) total of 84513 documents [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (2243) is smaller than 80% of number of samples per epoch (9205), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.009016 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 81893 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 18410 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.043876 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 9205) and [9205, 18410) ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002345 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_train_indexmap_11448ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_train_indexmap_11448ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_train_indexmap_11448ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 18411 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.004718 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 84428 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 9368 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.042930 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 9368) and [9368, 9368) ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002831 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9369 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9429 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.015947 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: number of documents: 3299965 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1077381) total of 1077381 documents [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: document indices in [1077381, 2188117) total of 1110736 documents [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: document indices in [2188117, 3299965) total of 1111848 documents [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.042604 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1077381 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 391500 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.053249 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 391500) and [391500, 391500) ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.011889 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_train_indexmap_369339ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_train_indexmap_369339ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_train_indexmap_369339ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of samples: 391501 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:31 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.038658 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1110736 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 404399 [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.053697 [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 404399) and [404399, 404399) ... [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.013361 [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_valid_indexmap_301ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_valid_indexmap_301ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_valid_indexmap_301ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: total number of samples: 404400 [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.028 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: total number of samples: 405099 [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.017767 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: number of documents: 20071773 [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 6553082) total of 6553082 documents [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: document indices in [6553082, 13309046) total of 6755964 documents [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: document indices in [13309046, 20071773) total of 6762727 documents [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (190338) is smaller than 80% of number of samples per epoch (903338), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:32 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.656573 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 6553082 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 1806677 [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.170388 [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 903338) and [903338, 1806677) ... [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.051910 [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_train_indexmap_1093676ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_train_indexmap_1093676ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_train_indexmap_1093676ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1806678 [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.320379 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 6755964 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 930740 [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.115268 [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 930740) and [930740, 930740) ... [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.027927 [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_valid_indexmap_890ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_valid_indexmap_890ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_valid_indexmap_890ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: total number of samples: 930741 [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.101 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: total number of samples: 933066 [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.012587 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: number of documents: 19544285 [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 6380866) total of 6380866 documents [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: document indices in [6380866, 12959283) total of 6578417 documents [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:33 [INFO|DP=0|PP=0|TP=0]: document indices in [12959283, 19544285) total of 6585002 documents [ip-26-0-154-121:0]:06/21/2023 17:12:34 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:34 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (58554) is smaller than 80% of number of samples per epoch (755476), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:34 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.629969 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 6380866 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 1510952 [ip-26-0-154-121:0]:06/21/2023 17:12:34 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.155434 [ip-26-0-154-121:0]:06/21/2023 17:12:34 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 755476) and [755476, 1510952) ... [ip-26-0-154-121:0]:06/21/2023 17:12:34 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.043257 [ip-26-0-154-121:0]:06/21/2023 17:12:34 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_train_indexmap_814030ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:34 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_train_indexmap_814030ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:34 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_train_indexmap_814030ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:34 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:34 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1510953 [ip-26-0-154-121:0]:06/21/2023 17:12:34 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.309179 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 6578417 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 776847 [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.106435 [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 776847) and [776847, 776847) ... [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.023495 [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_valid_indexmap_663ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_valid_indexmap_663ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_valid_indexmap_663ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: total number of samples: 776848 [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.036 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: total number of samples: 776230 [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.028339 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: number of documents: 21029287 [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 6865694) total of 6865694 documents [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: document indices in [6865694, 13943948) total of 7078254 documents [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: document indices in [13943948, 21029287) total of 7085339 documents [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:35 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.324109 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 6865694 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 1011160 [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.130824 [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 1011160) and [1011160, 1011160) ... [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.029956 [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_train_indexmap_942595ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_train_indexmap_942595ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_train_indexmap_942595ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1011161 [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.340358 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 7078254 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 1046847 [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.132349 [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 1046847) and [1046847, 1046847) ... [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.030654 [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_valid_indexmap_767ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_valid_indexmap_767ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_valid_indexmap_767ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1046848 [ip-26-0-154-121:0]:06/21/2023 17:12:36 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.038 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1047246 [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.016246 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: number of documents: 15683017 [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 5120231) total of 5120231 documents [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: document indices in [5120231, 10398982) total of 5278751 documents [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: document indices in [10398982, 15683017) total of 5284035 documents [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (95027) is smaller than 80% of number of samples per epoch (670949), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.502856 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 5120231 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 1341899 [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.129272 [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 670949) and [670949, 1341899) ... [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.038730 [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_train_indexmap_765976ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_train_indexmap_765976ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_train_indexmap_765976ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1341900 [ip-26-0-154-121:0]:06/21/2023 17:12:37 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.245445 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 5278751 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 690230 [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.097330 [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 690230) and [690230, 690230) ... [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.020308 [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_valid_indexmap_623ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_valid_indexmap_623ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_valid_indexmap_623ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: total number of samples: 690231 [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.035 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: total number of samples: 692824 [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.015963 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: number of documents: 12866649 [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 4200736) total of 4200736 documents [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: document indices in [4200736, 8531525) total of 4330789 documents [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: document indices in [8531525, 12866649) total of 4335124 documents [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:38 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (56205) is smaller than 80% of number of samples per epoch (703607), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.390392 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 4200736 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 1407215 [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.116239 [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 703607) and [703607, 1407215) ... [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.040663 [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_train_indexmap_759812ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_train_indexmap_759812ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_train_indexmap_759812ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1407216 [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.193010 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 4330789 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 725127 [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.080600 [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 725127) and [725127, 725127) ... [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.021833 [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_valid_indexmap_618ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_valid_indexmap_618ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_valid_indexmap_618ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: total number of samples: 725128 [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.036 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: total number of samples: 723779 [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.015445 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: number of documents: 10547331 [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 3443519) total of 3443519 documents [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: document indices in [3443519, 6993648) total of 3550129 documents [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: document indices in [6993648, 10547331) total of 3553683 documents [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:39 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (37295) is smaller than 80% of number of samples per epoch (296318), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.278516 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 3443519 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 592636 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.088363 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 296318) and [296318, 592636) ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.017553 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_train_indexmap_333613ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_train_indexmap_333613ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_train_indexmap_333613ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of samples: 592637 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.139384 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 3550129 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 303301 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.066728 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 303301) and [303301, 303301) ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.009396 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_valid_indexmap_272ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_valid_indexmap_272ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_valid_indexmap_272ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of samples: 303302 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.028 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of samples: 304761 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.007534 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: number of documents: 75 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 25) total of 25 documents [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: document indices in [25, 50) total of 25 documents [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: document indices in [50, 75) total of 25 documents [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (2) is smaller than 80% of number of samples per epoch (5), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002548 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 25 [ip-26-0-154-121:0]: number of epochs: 3 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 17 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.043423 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 11) and [11, 17) ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001799 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_train_indexmap_13ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_train_indexmap_13ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_train_indexmap_13ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of samples: 18 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 3 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of samples: 8 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.015151 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: number of documents: 161239 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 52642) total of 52642 documents [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: document indices in [52642, 106913) total of 54271 documents [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: document indices in [106913, 161239) total of 54326 documents [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (3006) is smaller than 80% of number of samples per epoch (14858), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.005795 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 52642 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 29717 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041168 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 14858) and [14858, 29717) ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002829 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_train_indexmap_17864ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_train_indexmap_17864ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_train_indexmap_17864ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of samples: 29718 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003351 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 54271 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 15234 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040477 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 15234) and [15234, 15234) ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002611 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_valid_indexmap_15ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_valid_indexmap_15ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_valid_indexmap_15ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of samples: 15235 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.015 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of samples: 15594 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.011644 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: number of documents: 58208 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 19004) total of 19004 documents [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: document indices in [19004, 38596) total of 19592 documents [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: document indices in [38596, 58208) total of 19612 documents [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003520 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 19004 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 14510 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041596 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 14510) and [14510, 14510) ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002191 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_train_indexmap_11825ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_train_indexmap_11825ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_train_indexmap_11825ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of samples: 14511 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002398 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 19592 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 15491 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040944 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 15491) and [15491, 15491) ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002577 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of samples: 15492 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of samples: 16244 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005725 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: number of documents: 4661 [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1522) total of 1522 documents [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: document indices in [1522, 3091) total of 1569 documents [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: document indices in [3091, 4661) total of 1570 documents [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:40 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003694 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1522 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 149 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040775 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 149) and [149, 149) ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001954 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_train_indexmap_126ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 150 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 149 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 138 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005694 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: number of documents: 93 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 31) total of 31 documents [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [31, 62) total of 31 documents [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [62, 93) total of 31 documents [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (1) is larger than 80% of number of samples per epoch (2), setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002541 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 31 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 5 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.041578 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 5) and [5, 5) ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002324 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_train_indexmap_3ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_train_indexmap_3ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_train_indexmap_3ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.006146 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: number of documents: 7451 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2433) total of 2433 documents [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [2433, 4941) total of 2508 documents [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [4941, 7451) total of 2510 documents [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002842 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 2433 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 1490 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.039890 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 1490) and [1490, 1490) ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002255 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_train_indexmap_1384ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_train_indexmap_1384ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_train_indexmap_1384ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1491 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.014 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1463 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1483 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.007428 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: number of documents: 15850 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 5175) total of 5175 documents [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [5175, 10510) total of 5335 documents [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [10510, 15850) total of 5340 documents [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > last epoch number of samples (133) is smaller than 80% of number of samples per epoch (2132), setting separate_last_epoch to True [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.003368 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 5175 [ip-26-0-154-121:0]: number of epochs: 2 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 4265 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040527 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 2132) and [2132, 4265) ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002471 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_train_indexmap_2265ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_train_indexmap_2265ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_train_indexmap_2265ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4266 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.002093 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 5335 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 2604 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040314 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 2604) and [2604, 2604) ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001910 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2605 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2298 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.010990 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: number of documents: 42103 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 13746) total of 13746 documents [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [13746, 27917) total of 14171 documents [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [27917, 42103) total of 14186 documents [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.004156 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 13746 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 5492 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.040831 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 5492) and [5492, 5492) ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.001912 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_train_indexmap_629ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_train_indexmap_629ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_train_indexmap_629ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5493 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5972 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.013 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5680 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.016229 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: number of documents: 4751547 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1551296) total of 1551296 documents [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [1551296, 3150621) total of 1599325 documents [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [3150621, 4751547) total of 1600926 documents [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.055994 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1551296 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 83478 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.049962 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 83478) and [83478, 83478) ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.004313 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_train_indexmap_12580ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_train_indexmap_12580ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_train_indexmap_12580ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 83479 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.057339 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1599325 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 86239 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.051730 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 86239) and [86239, 86239) ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.003931 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 86240 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.025 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of samples: 86171 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.016056 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: number of documents: 3995948 [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1304607) total of 1304607 documents [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [1304607, 2649604) total of 1344997 documents [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:41 [INFO|DP=0|PP=0|TP=0]: document indices in [2649604, 3995948) total of 1346344 documents [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.047018 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1304607 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 47955 [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.053343 [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 47955) and [47955, 47955) ... [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.004151 [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_train_indexmap_12580ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_train_indexmap_12580ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_train_indexmap_12580ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: total number of samples: 47956 [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.047751 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 1344997 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 49436 [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.050371 [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 49436) and [49436, 49436) ... [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.002706 [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: total number of samples: 49437 [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.022 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: total number of samples: 49416 [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.017504 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: number of documents: 30982955 [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 10115392) total of 10115392 documents [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: document indices in [10115392, 20543954) total of 10428562 documents [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: document indices in [20543954, 30982955) total of 10439001 documents [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:42 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.486831 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 10115392 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 718291 [ip-26-0-154-121:0]:06/21/2023 17:12:43 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.146146 [ip-26-0-154-121:0]:06/21/2023 17:12:43 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 718291) and [718291, 718291) ... [ip-26-0-154-121:0]:06/21/2023 17:12:43 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.021556 [ip-26-0-154-121:0]:06/21/2023 17:12:43 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_train_indexmap_684334ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:43 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_train_indexmap_684334ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:43 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_train_indexmap_684334ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:43 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:43 [INFO|DP=0|PP=0|TP=0]: total number of samples: 718292 [ip-26-0-154-121:0]:06/21/2023 17:12:43 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:43 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:43 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:43 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.534360 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 10428562 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 742303 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.154011 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 742303) and [742303, 742303) ... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.025712 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_valid_indexmap_557ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_valid_indexmap_557ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_valid_indexmap_557ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: total number of samples: 742304 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.036 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: total number of samples: 740971 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.018356 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: number of documents: 7634718 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2492602) total of 2492602 documents [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: document indices in [2492602, 5062374) total of 2569772 documents [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: document indices in [5062374, 7634718) total of 2572344 documents [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.101268 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 2492602 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 647820 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.071891 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 647820) and [647820, 647820) ... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.020282 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_train_indexmap_402550ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_train_indexmap_402550ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_train_indexmap_402550ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: total number of samples: 647821 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.099136 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 2569772 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 669747 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.067865 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 669747) and [669747, 669747) ... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.022726 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_valid_indexmap_328ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_valid_indexmap_328ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_valid_indexmap_328ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: total number of samples: 669748 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.036 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: total number of samples: 665861 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.016357 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: number of documents: 914510 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 298572) total of 298572 documents [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: document indices in [298572, 606387) total of 307815 documents [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: document indices in [606387, 914510) total of 308123 documents [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.011961 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 298572 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 100323 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.047166 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 100323) and [100323, 100323) ... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.006011 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_train_indexmap_89568ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_train_indexmap_89568ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_train_indexmap_89568ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.004 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: total number of samples: 100324 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:44 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.011985 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 307815 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 101463 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.045202 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 101463) and [101463, 101463) ... [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.004136 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_valid_indexmap_73ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_valid_indexmap_73ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_valid_indexmap_73ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: total number of samples: 101464 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.019 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: total number of samples: 99559 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.015942 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: number of documents: 668743 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 218333) total of 218333 documents [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: document indices in [218333, 443425) total of 225092 documents [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: document indices in [443425, 668743) total of 225318 documents [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.009126 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 218333 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 80111 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.046582 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 80111) and [80111, 80111) ... [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.004279 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_train_indexmap_75478ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_train_indexmap_75478ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_train_indexmap_75478ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: total number of samples: 80112 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > WARNING: could not find index map files, building the indices on rank 0 ... [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > only one epoch required, setting separate_last_epoch to False [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save doc-idx mapping (seconds): 0.009120 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]: using: [ip-26-0-154-121:0]: number of documents: 225092 [ip-26-0-154-121:0]: number of epochs: 1 [ip-26-0-154-121:0]: sequence length: 8192 [ip-26-0-154-121:0]: total number of samples: 81238 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save sample-idx mapping (seconds): 0.044907 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > building shuffle index with split [0, 81238) and [81238, 81238) ... [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > elasped time to build and save shuffle-idx mapping (seconds): 0.004498 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_valid_indexmap_62ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_valid_indexmap_62ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_valid_indexmap_62ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: total number of samples: 81239 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.017 seconds [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: total number of samples: 81006 [ip-26-0-154-121:0]:06/21/2023 17:12:45 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:> building indices for blendable datasets ... [ip-26-0-154-121:0]: > sample ratios: [ip-26-0-154-121:0]: dataset 0, input: 0.00391159, achieved: 0.00391157 [ip-26-0-154-121:0]: dataset 1, input: 1.30386e-05, achieved: 1.30596e-05 [ip-26-0-154-121:0]: dataset 2, input: 0.0702651, achieved: 0.0702652 [ip-26-0-154-121:0]: dataset 3, input: 0.00232087, achieved: 0.00232089 [ip-26-0-154-121:0]: dataset 4, input: 0.00110828, achieved: 0.00110831 [ip-26-0-154-121:0]: dataset 5, input: 0.00740594, achieved: 0.00740596 [ip-26-0-154-121:0]: dataset 6, input: 1.30386e-05, achieved: 1.30596e-05 [ip-26-0-154-121:0]: dataset 7, input: 0.00170806, achieved: 0.00170802 [ip-26-0-154-121:0]: dataset 8, input: 0.00127778, achieved: 0.00127777 [ip-26-0-154-121:0]: dataset 9, input: 0.000104309, achieved: 0.00010427 [ip-26-0-154-121:0]: dataset 10, input: 3.91159e-05, achieved: 3.90753e-05 [ip-26-0-154-121:0]: dataset 11, input: 0.000117348, achieved: 0.00011733 [ip-26-0-154-121:0]: dataset 12, input: 0.00146033, achieved: 0.0014603 [ip-26-0-154-121:0]: dataset 13, input: 0.0310058, achieved: 0.0310058 [ip-26-0-154-121:0]: dataset 14, input: 0.000912704, achieved: 0.000912724 [ip-26-0-154-121:0]: dataset 15, input: 0.000795356, achieved: 0.000795395 [ip-26-0-154-121:0]: dataset 16, input: 0.000339004, achieved: 0.000339033 [ip-26-0-154-121:0]: dataset 17, input: 0.00219049, achieved: 0.0021905 [ip-26-0-154-121:0]: dataset 18, input: 0.00290761, achieved: 0.00290764 [ip-26-0-154-121:0]: dataset 19, input: 0.000391159, achieved: 0.000391168 [ip-26-0-154-121:0]: dataset 20, input: 0.000404197, achieved: 0.000404227 [ip-26-0-154-121:0]: dataset 21, input: 0.000586738, achieved: 0.000586751 [ip-26-0-154-121:0]: dataset 22, input: 0.000156463, achieved: 0.000156508 [ip-26-0-154-121:0]: dataset 23, input: 0.0088793, achieved: 0.00887932 [ip-26-0-154-121:0]: dataset 24, input: 0.0118782, achieved: 0.0118782 [ip-26-0-154-121:0]: dataset 25, input: 7.82317e-05, achieved: 7.82542e-05 [ip-26-0-154-121:0]: dataset 26, input: 0.0582305, achieved: 0.0582305 [ip-26-0-154-121:0]: dataset 27, input: 0.00075624, achieved: 0.000756216 [ip-26-0-154-121:0]: dataset 28, input: 0.00290761, achieved: 0.00290764 [ip-26-0-154-121:0]: dataset 29, input: 1.30386e-05, achieved: 1.30596e-05 [ip-26-0-154-121:0]: dataset 30, input: 0.00162983, achieved: 0.00162986 [ip-26-0-154-121:0]: dataset 31, input: 0.00134298, achieved: 0.00134297 [ip-26-0-154-121:0]: dataset 32, input: 0.00170806, achieved: 0.00170802 [ip-26-0-154-121:0]: dataset 33, input: 0.00374208, achieved: 0.00374211 [ip-26-0-154-121:0]: dataset 34, input: 1.30386e-05, achieved: 1.30596e-05 [ip-26-0-154-121:0]: dataset 35, input: 6.51931e-05, achieved: 6.51946e-05 [ip-26-0-154-121:0]: dataset 36, input: 0.00432882, achieved: 0.00432886 [ip-26-0-154-121:0]: dataset 37, input: 3.91159e-05, achieved: 3.90753e-05 [ip-26-0-154-121:0]: dataset 38, input: 0.000247734, achieved: 0.000247719 [ip-26-0-154-121:0]: dataset 39, input: 0.000508506, achieved: 0.000508497 [ip-26-0-154-121:0]: dataset 40, input: 0.00678008, achieved: 0.00678003 [ip-26-0-154-121:0]: dataset 41, input: 2.60772e-05, achieved: 2.61193e-05 [ip-26-0-154-121:0]: dataset 42, input: 0.00203403, achieved: 0.00203399 [ip-26-0-154-121:0]: dataset 43, input: 1.30386e-05, achieved: 1.30596e-05 [ip-26-0-154-121:0]: dataset 44, input: 9.12704e-05, achieved: 9.13139e-05 [ip-26-0-154-121:0]: dataset 45, input: 0.000534584, achieved: 0.000534616 [ip-26-0-154-121:0]: dataset 46, input: 0.00477214, achieved: 0.00477216 [ip-26-0-154-121:0]: dataset 47, input: 0.000730163, achieved: 0.0007302 [ip-26-0-154-121:0]: dataset 48, input: 3.91159e-05, achieved: 3.90753e-05 [ip-26-0-154-121:0]: dataset 49, input: 1.30386e-06, achieved: 1.34742e-06 [ip-26-0-154-121:0]: dataset 50, input: 0.000299888, achieved: 0.000299854 [ip-26-0-154-121:0]: dataset 51, input: 2.60772e-05, achieved: 2.61193e-05 [ip-26-0-154-121:0]: dataset 52, input: 1.30386e-05, achieved: 1.30596e-05 [ip-26-0-154-121:0]: dataset 53, input: 0.00611511, achieved: 0.00611513 [ip-26-0-154-121:0]: dataset 54, input: 0.000456352, achieved: 0.000456362 [ip-26-0-154-121:0]: dataset 55, input: 0.000430275, achieved: 0.000430243 [ip-26-0-154-121:0]: dataset 56, input: 1.30386e-05, achieved: 1.30596e-05 [ip-26-0-154-121:0]: dataset 57, input: 0.00402893, achieved: 0.0040289 [ip-26-0-154-121:0]: dataset 58, input: 0.000599777, achieved: 0.000599811 [ip-26-0-154-121:0]: dataset 59, input: 0.000260772, achieved: 0.000260778 [ip-26-0-154-121:0]: dataset 60, input: 6.51931e-05, achieved: 6.51946e-05 [ip-26-0-154-121:0]: dataset 61, input: 5.21545e-05, achieved: 5.21349e-05 [ip-26-0-154-121:0]: dataset 62, input: 0.0144598, achieved: 0.0144598 [ip-26-0-154-121:0]: dataset 63, input: 0.000521545, achieved: 0.000521557 [ip-26-0-154-121:0]: dataset 64, input: 0.000391159, achieved: 0.000391168 [ip-26-0-154-121:0]: dataset 65, input: 0.000547622, achieved: 0.000547572 [ip-26-0-154-121:0]: dataset 66, input: 0.0637849, achieved: 0.0637849 [ip-26-0-154-121:0]: dataset 67, input: 0.000834472, achieved: 0.00083447 [ip-26-0-154-121:0]: dataset 68, input: 0.00182541, achieved: 0.00182535 [ip-26-0-154-121:0]: dataset 69, input: 0.000925742, achieved: 0.000925784 [ip-26-0-154-121:0]: dataset 70, input: 0.00118651, achieved: 0.00118646 [ip-26-0-154-121:0]: dataset 71, input: 0.0382814, achieved: 0.0382814 [ip-26-0-154-121:0]: dataset 72, input: 0.113358, achieved: 0.113358 [ip-26-0-154-121:0]: dataset 73, input: 0.0843729, achieved: 0.0843729 [ip-26-0-154-121:0]: dataset 74, input: 0.0976984, achieved: 0.0976984 [ip-26-0-154-121:0]: dataset 75, input: 0.0793922, achieved: 0.0793922 [ip-26-0-154-121:0]: dataset 76, input: 0.0787533, achieved: 0.0787532 [ip-26-0-154-121:0]: dataset 77, input: 0.0345784, achieved: 0.0345784 [ip-26-0-154-121:0]: dataset 78, input: 1.30386e-06, achieved: 1.34742e-06 [ip-26-0-154-121:0]: dataset 79, input: 0.00185148, achieved: 0.00185146 [ip-26-0-154-121:0]: dataset 80, input: 0.00122563, achieved: 0.00122564 [ip-26-0-154-121:0]: dataset 81, input: 1.30386e-05, achieved: 1.30596e-05 [ip-26-0-154-121:0]: dataset 82, input: 2.60772e-07, achieved: 3.10944e-07 [ip-26-0-154-121:0]: dataset 83, input: 0.000143425, achieved: 0.000143449 [ip-26-0-154-121:0]: dataset 84, input: 0.000234695, achieved: 0.000234659 [ip-26-0-154-121:0]: dataset 85, input: 6.51931e-05, achieved: 6.51946e-05 [ip-26-0-154-121:0]: dataset 86, input: 0.00130386, achieved: 0.00130389 [ip-26-0-154-121:0]: dataset 87, input: 0.00130386, achieved: 0.00130389 [ip-26-0-154-121:0]: dataset 88, input: 0.0709301, achieved: 0.0709301 [ip-26-0-154-121:0]: dataset 89, input: 0.0417236, achieved: 0.0417236 [ip-26-0-154-121:0]: dataset 90, input: 0.0092835, achieved: 0.00928344 [ip-26-0-154-121:0]: dataset 91, input: 0.00782317, achieved: 0.00782314 [ip-26-0-154-121:0]:06/21/2023 17:12:46 [INFO|DP=0|PP=0|TP=0]: > elapsed time for building blendable dataset indices: 1.12 (sec) [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:> building indices for blendable datasets ... [ip-26-0-154-121:0]: > sample ratios: [ip-26-0-154-121:0]: dataset 0, input: 0.00391159, achieved: 0.00392455 [ip-26-0-154-121:0]: dataset 1, input: 1.30386e-05, achieved: 0 [ip-26-0-154-121:0]: dataset 2, input: 0.0702651, achieved: 0.0702621 [ip-26-0-154-121:0]: dataset 3, input: 0.00232087, achieved: 0.00227877 [ip-26-0-154-121:0]: dataset 4, input: 0.00110828, achieved: 0.00113938 [ip-26-0-154-121:0]: dataset 5, input: 0.00740594, achieved: 0.0073427 [ip-26-0-154-121:0]: dataset 6, input: 1.30386e-05, achieved: 0 [ip-26-0-154-121:0]: dataset 7, input: 0.00170806, achieved: 0.00177238 [ip-26-0-154-121:0]: dataset 8, input: 0.00127778, achieved: 0.00126598 [ip-26-0-154-121:0]: dataset 9, input: 0.000104309, achieved: 0.000126598 [ip-26-0-154-121:0]: dataset 10, input: 3.91159e-05, achieved: 0 [ip-26-0-154-121:0]: dataset 11, input: 0.000117348, achieved: 0.000126598 [ip-26-0-154-121:0]: dataset 12, input: 0.00146033, achieved: 0.00151918 [ip-26-0-154-121:0]: dataset 13, input: 0.0310058, achieved: 0.0310166 [ip-26-0-154-121:0]: dataset 14, input: 0.000912704, achieved: 0.000886188 [ip-26-0-154-121:0]: dataset 15, input: 0.000795356, achieved: 0.00075959 [ip-26-0-154-121:0]: dataset 16, input: 0.000339004, achieved: 0.000379795 [ip-26-0-154-121:0]: dataset 17, input: 0.00219049, achieved: 0.00215217 [ip-26-0-154-121:0]: dataset 18, input: 0.00290761, achieved: 0.00291176 [ip-26-0-154-121:0]: dataset 19, input: 0.000391159, achieved: 0.000379795 [ip-26-0-154-121:0]: dataset 20, input: 0.000404197, achieved: 0.000379795 [ip-26-0-154-121:0]: dataset 21, input: 0.000586738, achieved: 0.000632992 [ip-26-0-154-121:0]: dataset 22, input: 0.000156463, achieved: 0.000126598 [ip-26-0-154-121:0]: dataset 23, input: 0.0088793, achieved: 0.00886188 [ip-26-0-154-121:0]: dataset 24, input: 0.0118782, achieved: 0.0119002 [ip-26-0-154-121:0]: dataset 25, input: 7.82317e-05, achieved: 0.000126598 [ip-26-0-154-121:0]: dataset 26, input: 0.0582305, achieved: 0.0582352 [ip-26-0-154-121:0]: dataset 27, input: 0.00075624, achieved: 0.00075959 [ip-26-0-154-121:0]: dataset 28, input: 0.00290761, achieved: 0.00291176 [ip-26-0-154-121:0]: dataset 29, input: 1.30386e-05, achieved: 0 [ip-26-0-154-121:0]: dataset 30, input: 0.00162983, achieved: 0.00164578 [ip-26-0-154-121:0]: dataset 31, input: 0.00134298, achieved: 0.00139258 [ip-26-0-154-121:0]: dataset 32, input: 0.00170806, achieved: 0.00177238 [ip-26-0-154-121:0]: dataset 33, input: 0.00374208, achieved: 0.00379795 [ip-26-0-154-121:0]: dataset 34, input: 1.30386e-05, achieved: 0 [ip-26-0-154-121:0]: dataset 35, input: 6.51931e-05, achieved: 0.000126598 [ip-26-0-154-121:0]: dataset 36, input: 0.00432882, achieved: 0.00430434 [ip-26-0-154-121:0]: dataset 37, input: 3.91159e-05, achieved: 0 [ip-26-0-154-121:0]: dataset 38, input: 0.000247734, achieved: 0.000253197 [ip-26-0-154-121:0]: dataset 39, input: 0.000508506, achieved: 0.000506393 [ip-26-0-154-121:0]: dataset 40, input: 0.00678008, achieved: 0.00683631 [ip-26-0-154-121:0]: dataset 41, input: 2.60772e-05, achieved: 0 [ip-26-0-154-121:0]: dataset 42, input: 0.00203403, achieved: 0.00202557 [ip-26-0-154-121:0]: dataset 43, input: 1.30386e-05, achieved: 0 [ip-26-0-154-121:0]: dataset 44, input: 9.12704e-05, achieved: 0.000126598 [ip-26-0-154-121:0]: dataset 45, input: 0.000534584, achieved: 0.000506393 [ip-26-0-154-121:0]: dataset 46, input: 0.00477214, achieved: 0.00481074 [ip-26-0-154-121:0]: dataset 47, input: 0.000730163, achieved: 0.00075959 [ip-26-0-154-121:0]: dataset 48, input: 3.91159e-05, achieved: 0 [ip-26-0-154-121:0]: dataset 49, input: 1.30386e-06, achieved: 0 [ip-26-0-154-121:0]: dataset 50, input: 0.000299888, achieved: 0.000253197 [ip-26-0-154-121:0]: dataset 51, input: 2.60772e-05, achieved: 0 [ip-26-0-154-121:0]: dataset 52, input: 1.30386e-05, achieved: 0 [ip-26-0-154-121:0]: dataset 53, input: 0.00611511, achieved: 0.00607672 [ip-26-0-154-121:0]: dataset 54, input: 0.000456352, achieved: 0.000506393 [ip-26-0-154-121:0]: dataset 55, input: 0.000430275, achieved: 0.000379795 [ip-26-0-154-121:0]: dataset 56, input: 1.30386e-05, achieved: 0 [ip-26-0-154-121:0]: dataset 57, input: 0.00402893, achieved: 0.00405115 [ip-26-0-154-121:0]: dataset 58, input: 0.000599777, achieved: 0.000632992 [ip-26-0-154-121:0]: dataset 59, input: 0.000260772, achieved: 0.000253197 [ip-26-0-154-121:0]: dataset 60, input: 6.51931e-05, achieved: 0.000126598 [ip-26-0-154-121:0]: dataset 61, input: 5.21545e-05, achieved: 0.000126598 [ip-26-0-154-121:0]: dataset 62, input: 0.0144598, achieved: 0.0144322 [ip-26-0-154-121:0]: dataset 63, input: 0.000521545, achieved: 0.000506393 [ip-26-0-154-121:0]: dataset 64, input: 0.000391159, achieved: 0.000379795 [ip-26-0-154-121:0]: dataset 65, input: 0.000547622, achieved: 0.000506393 [ip-26-0-154-121:0]: dataset 66, input: 0.0637849, achieved: 0.0638055 [ip-26-0-154-121:0]: dataset 67, input: 0.000834472, achieved: 0.000886188 [ip-26-0-154-121:0]: dataset 68, input: 0.00182541, achieved: 0.00177238 [ip-26-0-154-121:0]: dataset 69, input: 0.000925742, achieved: 0.000886188 [ip-26-0-154-121:0]: dataset 70, input: 0.00118651, achieved: 0.00113938 [ip-26-0-154-121:0]: dataset 71, input: 0.0382814, achieved: 0.0382327 [ip-26-0-154-121:0]: dataset 72, input: 0.113358, achieved: 0.113305 [ip-26-0-154-121:0]: dataset 73, input: 0.0843729, achieved: 0.0843145 [ip-26-0-154-121:0]: dataset 74, input: 0.0976984, achieved: 0.0977339 [ip-26-0-154-121:0]: dataset 75, input: 0.0793922, achieved: 0.0793771 [ip-26-0-154-121:0]: dataset 76, input: 0.0787533, achieved: 0.0787441 [ip-26-0-154-121:0]: dataset 77, input: 0.0345784, achieved: 0.0345613 [ip-26-0-154-121:0]: dataset 78, input: 1.30386e-06, achieved: 0 [ip-26-0-154-121:0]: dataset 79, input: 0.00185148, achieved: 0.00189897 [ip-26-0-154-121:0]: dataset 80, input: 0.00122563, achieved: 0.00126598 [ip-26-0-154-121:0]: dataset 81, input: 1.30386e-05, achieved: 0 [ip-26-0-154-121:0]: dataset 82, input: 2.60772e-07, achieved: 0 [ip-26-0-154-121:0]: dataset 83, input: 0.000143425, achieved: 0.000126598 [ip-26-0-154-121:0]: dataset 84, input: 0.000234695, achieved: 0.000253197 [ip-26-0-154-121:0]: dataset 85, input: 6.51931e-05, achieved: 0.000126598 [ip-26-0-154-121:0]: dataset 86, input: 0.00130386, achieved: 0.00126598 [ip-26-0-154-121:0]: dataset 87, input: 0.00130386, achieved: 0.00126598 [ip-26-0-154-121:0]: dataset 88, input: 0.0709301, achieved: 0.0708951 [ip-26-0-154-121:0]: dataset 89, input: 0.0417236, achieved: 0.0417774 [ip-26-0-154-121:0]: dataset 90, input: 0.0092835, achieved: 0.00924168 [ip-26-0-154-121:0]: dataset 91, input: 0.00782317, achieved: 0.00784909 [ip-26-0-154-121:0]:06/21/2023 17:12:46 [INFO|DP=0|PP=0|TP=0]: > elapsed time for building blendable dataset indices: 0.05 (sec) [ip-26-0-154-121:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:make: Nothing to be done for 'default'. [ip-26-0-154-121:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-154-121:0]:> building indices for blendable datasets ... [ip-26-0-154-121:0]: > sample ratios: [ip-26-0-154-121:0]: dataset 0, input: 0.00391159, achieved: -nan [ip-26-0-154-121:0]: dataset 1, input: 1.30386e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 2, input: 0.0702651, achieved: -nan [ip-26-0-154-121:0]: dataset 3, input: 0.00232087, achieved: -nan [ip-26-0-154-121:0]: dataset 4, input: 0.00110828, achieved: -nan [ip-26-0-154-121:0]: dataset 5, input: 0.00740594, achieved: -nan [ip-26-0-154-121:0]: dataset 6, input: 1.30386e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 7, input: 0.00170806, achieved: -nan [ip-26-0-154-121:0]: dataset 8, input: 0.00127778, achieved: -nan [ip-26-0-154-121:0]: dataset 9, input: 0.000104309, achieved: -nan [ip-26-0-154-121:0]: dataset 10, input: 3.91159e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 11, input: 0.000117348, achieved: -nan [ip-26-0-154-121:0]: dataset 12, input: 0.00146033, achieved: -nan [ip-26-0-154-121:0]: dataset 13, input: 0.0310058, achieved: -nan [ip-26-0-154-121:0]: dataset 14, input: 0.000912704, achieved: -nan [ip-26-0-154-121:0]: dataset 15, input: 0.000795356, achieved: -nan [ip-26-0-154-121:0]: dataset 16, input: 0.000339004, achieved: -nan [ip-26-0-154-121:0]: dataset 17, input: 0.00219049, achieved: -nan [ip-26-0-154-121:0]: dataset 18, input: 0.00290761, achieved: -nan [ip-26-0-154-121:0]: dataset 19, input: 0.000391159, achieved: -nan [ip-26-0-154-121:0]: dataset 20, input: 0.000404197, achieved: -nan [ip-26-0-154-121:0]: dataset 21, input: 0.000586738, achieved: -nan [ip-26-0-154-121:0]: dataset 22, input: 0.000156463, achieved: -nan [ip-26-0-154-121:0]: dataset 23, input: 0.0088793, achieved: -nan [ip-26-0-154-121:0]: dataset 24, input: 0.0118782, achieved: -nan [ip-26-0-154-121:0]: dataset 25, input: 7.82317e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 26, input: 0.0582305, achieved: -nan [ip-26-0-154-121:0]: dataset 27, input: 0.00075624, achieved: -nan [ip-26-0-154-121:0]: dataset 28, input: 0.00290761, achieved: -nan [ip-26-0-154-121:0]: dataset 29, input: 1.30386e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 30, input: 0.00162983, achieved: -nan [ip-26-0-154-121:0]: dataset 31, input: 0.00134298, achieved: -nan [ip-26-0-154-121:0]: dataset 32, input: 0.00170806, achieved: -nan [ip-26-0-154-121:0]: dataset 33, input: 0.00374208, achieved: -nan [ip-26-0-154-121:0]: dataset 34, input: 1.30386e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 35, input: 6.51931e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 36, input: 0.00432882, achieved: -nan [ip-26-0-154-121:0]: dataset 37, input: 3.91159e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 38, input: 0.000247734, achieved: -nan [ip-26-0-154-121:0]: dataset 39, input: 0.000508506, achieved: -nan [ip-26-0-154-121:0]: dataset 40, input: 0.00678008, achieved: -nan [ip-26-0-154-121:0]: dataset 41, input: 2.60772e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 42, input: 0.00203403, achieved: -nan [ip-26-0-154-121:0]: dataset 43, input: 1.30386e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 44, input: 9.12704e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 45, input: 0.000534584, achieved: -nan [ip-26-0-154-121:0]: dataset 46, input: 0.00477214, achieved: -nan [ip-26-0-154-121:0]: dataset 47, input: 0.000730163, achieved: -nan [ip-26-0-154-121:0]: dataset 48, input: 3.91159e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 49, input: 1.30386e-06, achieved: -nan [ip-26-0-154-121:0]: dataset 50, input: 0.000299888, achieved: -nan [ip-26-0-154-121:0]: dataset 51, input: 2.60772e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 52, input: 1.30386e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 53, input: 0.00611511, achieved: -nan [ip-26-0-154-121:0]: dataset 54, input: 0.000456352, achieved: -nan [ip-26-0-154-121:0]: dataset 55, input: 0.000430275, achieved: -nan [ip-26-0-154-121:0]: dataset 56, input: 1.30386e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 57, input: 0.00402893, achieved: -nan [ip-26-0-154-121:0]: dataset 58, input: 0.000599777, achieved: -nan [ip-26-0-154-121:0]: dataset 59, input: 0.000260772, achieved: -nan [ip-26-0-154-121:0]: dataset 60, input: 6.51931e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 61, input: 5.21545e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 62, input: 0.0144598, achieved: -nan [ip-26-0-154-121:0]: dataset 63, input: 0.000521545, achieved: -nan [ip-26-0-154-121:0]: dataset 64, input: 0.000391159, achieved: -nan [ip-26-0-154-121:0]: dataset 65, input: 0.000547622, achieved: -nan [ip-26-0-154-121:0]: dataset 66, input: 0.0637849, achieved: -nan [ip-26-0-154-121:0]: dataset 67, input: 0.000834472, achieved: -nan [ip-26-0-154-121:0]: dataset 68, input: 0.00182541, achieved: -nan [ip-26-0-154-121:0]: dataset 69, input: 0.000925742, achieved: -nan [ip-26-0-154-121:0]: dataset 70, input: 0.00118651, achieved: -nan [ip-26-0-154-121:0]: dataset 71, input: 0.0382814, achieved: -nan [ip-26-0-154-121:0]: dataset 72, input: 0.113358, achieved: -nan [ip-26-0-154-121:0]: dataset 73, input: 0.0843729, achieved: -nan [ip-26-0-154-121:0]: dataset 74, input: 0.0976984, achieved: -nan [ip-26-0-154-121:0]: dataset 75, input: 0.0793922, achieved: -nan [ip-26-0-154-121:0]: dataset 76, input: 0.0787533, achieved: -nan [ip-26-0-154-121:0]: dataset 77, input: 0.0345784, achieved: -nan [ip-26-0-154-121:0]: dataset 78, input: 1.30386e-06, achieved: -nan [ip-26-0-154-121:0]: dataset 79, input: 0.00185148, achieved: -nan [ip-26-0-154-121:0]: dataset 80, input: 0.00122563, achieved: -nan [ip-26-0-154-121:0]: dataset 81, input: 1.30386e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 82, input: 2.60772e-07, achieved: -nan [ip-26-0-154-121:0]: dataset 83, input: 0.000143425, achieved: -nan [ip-26-0-154-121:0]: dataset 84, input: 0.000234695, achieved: -nan [ip-26-0-154-121:0]: dataset 85, input: 6.51931e-05, achieved: -nan [ip-26-0-154-121:0]: dataset 86, input: 0.00130386, achieved: -nan [ip-26-0-154-121:0]: dataset 87, input: 0.00130386, achieved: -nan [ip-26-0-154-121:0]: dataset 88, input: 0.0709301, achieved: -nan [ip-26-0-154-121:0]: dataset 89, input: 0.0417236, achieved: -nan [ip-26-0-154-121:0]: dataset 90, input: 0.0092835, achieved: -nan [ip-26-0-154-121:0]: dataset 91, input: 0.00782317, achieved: -nan [ip-26-0-154-121:0]:06/21/2023 17:12:46 [INFO|DP=0|PP=0|TP=0]: > elapsed time for building blendable dataset indices: 0.05 (sec) [ip-26-0-154-121:0]:06/21/2023 17:12:46 [INFO|DP=0|PP=0|TP=0]: Building dataloader with consumed samples: 0 [ip-26-0-154-121:0]:06/21/2023 17:12:46 [INFO|DP=0|PP=0|TP=0]: Instantiating MegatronPretrainingSampler with total_samples: 9648039 and consumed_samples: 0 [ip-26-0-154-121:0]:06/21/2023 17:12:46 [INFO|DP=0|PP=0|TP=0]: [Before the start of training] datetime: 2023-06-21 17:12:46.542651 [ip-26-0-154-121:0]:06/21/2023 17:12:54 [INFO|DP=0|PP=0|TP=0]: iteration: 1 / 150000 | consumed_samples: 64 | elapsed_time_per_iteration_ms: 7942.0 | tokens_per_sec: 6.601476E+04 | tokens_per_sec_per_gpu: 1.031481E+03 | global_batch_size: 64 | lm_loss: 1.131901E+01 | lr: 1.500E-07 | model_tflops_per_gpu: 11.91 | hardware_tflops_per_gpu: 11.91 | grad_norm: 3458662.500 [ip-26-0-154-121:0]:06/21/2023 17:12:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2 / 150000 | consumed_samples: 128 | elapsed_time_per_iteration_ms: 908.2 | tokens_per_sec: 5.772839E+05 | tokens_per_sec_per_gpu: 9.020061E+03 | global_batch_size: 64 | lm_loss: 1.133580E+01 | lr: 3.000E-07 | model_tflops_per_gpu: 104.19 | hardware_tflops_per_gpu: 104.19 | grad_norm: 3456567.750 [ip-26-0-154-121:0]:06/21/2023 17:12:56 [INFO|DP=0|PP=0|TP=0]: iteration: 3 / 150000 | consumed_samples: 192 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.778924E+05 | tokens_per_sec_per_gpu: 9.029568E+03 | global_batch_size: 64 | lm_loss: 1.133109E+01 | lr: 4.500E-07 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3456367.500 [ip-26-0-154-121:0]:06/21/2023 17:12:57 [INFO|DP=0|PP=0|TP=0]: iteration: 4 / 150000 | consumed_samples: 256 | elapsed_time_per_iteration_ms: 908.3 | tokens_per_sec: 5.771974E+05 | tokens_per_sec_per_gpu: 9.018709E+03 | global_batch_size: 64 | lm_loss: 1.129390E+01 | lr: 6.000E-07 | model_tflops_per_gpu: 104.18 | hardware_tflops_per_gpu: 104.18 | grad_norm: 3454064.750 [ip-26-0-154-121:0]:06/21/2023 17:12:58 [INFO|DP=0|PP=0|TP=0]: iteration: 5 / 150000 | consumed_samples: 320 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775488E+05 | tokens_per_sec_per_gpu: 9.024199E+03 | global_batch_size: 64 | lm_loss: 1.128798E+01 | lr: 7.500E-07 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 3459975.000 [ip-26-0-154-121:0]:06/21/2023 17:12:59 [INFO|DP=0|PP=0|TP=0]: iteration: 6 / 150000 | consumed_samples: 384 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780706E+05 | tokens_per_sec_per_gpu: 9.032353E+03 | global_batch_size: 64 | lm_loss: 1.122666E+01 | lr: 9.000E-07 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3434915.750 [ip-26-0-154-121:0]:06/21/2023 17:12:59 [INFO|DP=0|PP=0|TP=0]: iteration: 7 / 150000 | consumed_samples: 448 | elapsed_time_per_iteration_ms: 908.4 | tokens_per_sec: 5.771463E+05 | tokens_per_sec_per_gpu: 9.017911E+03 | global_batch_size: 64 | lm_loss: 1.139375E+01 | lr: 1.050E-06 | model_tflops_per_gpu: 104.17 | hardware_tflops_per_gpu: 104.17 | grad_norm: 3398655.750 [ip-26-0-154-121:0]:06/21/2023 17:13:00 [INFO|DP=0|PP=0|TP=0]: iteration: 8 / 150000 | consumed_samples: 512 | elapsed_time_per_iteration_ms: 909.1 | tokens_per_sec: 5.767299E+05 | tokens_per_sec_per_gpu: 9.011405E+03 | global_batch_size: 64 | lm_loss: 1.132901E+01 | lr: 1.200E-06 | model_tflops_per_gpu: 104.09 | hardware_tflops_per_gpu: 104.09 | grad_norm: 3384247.250 [ip-26-0-154-121:0]:06/21/2023 17:13:01 [INFO|DP=0|PP=0|TP=0]: iteration: 9 / 150000 | consumed_samples: 576 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777205E+05 | tokens_per_sec_per_gpu: 9.026883E+03 | global_batch_size: 64 | lm_loss: 1.142463E+01 | lr: 1.350E-06 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 3228550.250 [ip-26-0-154-121:0]:06/21/2023 17:13:02 [INFO|DP=0|PP=0|TP=0]: iteration: 10 / 150000 | consumed_samples: 640 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783258E+05 | tokens_per_sec_per_gpu: 9.036341E+03 | global_batch_size: 64 | lm_loss: 1.145040E+01 | lr: 1.500E-06 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3201493.000 [ip-26-0-154-121:0]:06/21/2023 17:13:03 [INFO|DP=0|PP=0|TP=0]: iteration: 11 / 150000 | consumed_samples: 704 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.774180E+05 | tokens_per_sec_per_gpu: 9.022157E+03 | global_batch_size: 64 | lm_loss: 1.134010E+01 | lr: 1.650E-06 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 3148579.000 [ip-26-0-154-121:0]:06/21/2023 17:13:04 [INFO|DP=0|PP=0|TP=0]: iteration: 12 / 150000 | consumed_samples: 768 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778565E+05 | tokens_per_sec_per_gpu: 9.029009E+03 | global_batch_size: 64 | lm_loss: 1.141350E+01 | lr: 1.800E-06 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 2896910.500 [ip-26-0-154-121:0]:06/21/2023 17:13:05 [INFO|DP=0|PP=0|TP=0]: iteration: 13 / 150000 | consumed_samples: 832 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784451E+05 | tokens_per_sec_per_gpu: 9.038205E+03 | global_batch_size: 64 | lm_loss: 1.142503E+01 | lr: 1.950E-06 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 2878092.750 [ip-26-0-154-121:0]:06/21/2023 17:13:06 [INFO|DP=0|PP=0|TP=0]: iteration: 14 / 150000 | consumed_samples: 896 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784343E+05 | tokens_per_sec_per_gpu: 9.038036E+03 | global_batch_size: 64 | lm_loss: 1.134461E+01 | lr: 2.100E-06 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 2840895.750 [ip-26-0-154-121:0]:06/21/2023 17:13:07 [INFO|DP=0|PP=0|TP=0]: iteration: 15 / 150000 | consumed_samples: 960 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786306E+05 | tokens_per_sec_per_gpu: 9.041104E+03 | global_batch_size: 64 | lm_loss: 1.135684E+01 | lr: 2.250E-06 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 2821103.250 [ip-26-0-154-121:0]:06/21/2023 17:13:08 [INFO|DP=0|PP=0|TP=0]: iteration: 16 / 150000 | consumed_samples: 1024 | elapsed_time_per_iteration_ms: 1014.8 | tokens_per_sec: 5.166521E+05 | tokens_per_sec_per_gpu: 8.072689E+03 | global_batch_size: 64 | lm_loss: 1.116935E+01 | lr: 2.400E-06 | model_tflops_per_gpu: 93.25 | hardware_tflops_per_gpu: 93.25 | grad_norm: 2785721.250 [ip-26-0-154-121:0]:06/21/2023 17:13:09 [INFO|DP=0|PP=0|TP=0]: iteration: 17 / 150000 | consumed_samples: 1088 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784013E+05 | tokens_per_sec_per_gpu: 9.037520E+03 | global_batch_size: 64 | lm_loss: 1.129455E+01 | lr: 2.550E-06 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 2804017.000 [ip-26-0-154-121:0]:06/21/2023 17:13:10 [INFO|DP=0|PP=0|TP=0]: iteration: 18 / 150000 | consumed_samples: 1152 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786587E+05 | tokens_per_sec_per_gpu: 9.041542E+03 | global_batch_size: 64 | lm_loss: 1.124516E+01 | lr: 2.700E-06 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2788353.250 [ip-26-0-154-121:0]:06/21/2023 17:13:10 [INFO|DP=0|PP=0|TP=0]: iteration: 19 / 150000 | consumed_samples: 1216 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779449E+05 | tokens_per_sec_per_gpu: 9.030390E+03 | global_batch_size: 64 | lm_loss: 1.131146E+01 | lr: 2.850E-06 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 2769269.000 [ip-26-0-154-121:0]:06/21/2023 17:13:11 [INFO|DP=0|PP=0|TP=0]: iteration: 20 / 150000 | consumed_samples: 1280 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784358E+05 | tokens_per_sec_per_gpu: 9.038060E+03 | global_batch_size: 64 | lm_loss: 1.123812E+01 | lr: 3.000E-06 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 2763078.500 [ip-26-0-154-121:0]:06/21/2023 17:13:12 [INFO|DP=0|PP=0|TP=0]: iteration: 21 / 150000 | consumed_samples: 1344 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780051E+05 | tokens_per_sec_per_gpu: 9.031330E+03 | global_batch_size: 64 | lm_loss: 1.132517E+01 | lr: 3.150E-06 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 2754007.500 [ip-26-0-154-121:0]:06/21/2023 17:13:13 [INFO|DP=0|PP=0|TP=0]: iteration: 22 / 150000 | consumed_samples: 1408 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788971E+05 | tokens_per_sec_per_gpu: 9.045267E+03 | global_batch_size: 64 | lm_loss: 1.127459E+01 | lr: 3.300E-06 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 2742163.250 [ip-26-0-154-121:0]:06/21/2023 17:13:14 [INFO|DP=0|PP=0|TP=0]: iteration: 23 / 150000 | consumed_samples: 1472 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784104E+05 | tokens_per_sec_per_gpu: 9.037663E+03 | global_batch_size: 64 | lm_loss: 1.135549E+01 | lr: 3.450E-06 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 2733627.500 [ip-26-0-154-121:0]:06/21/2023 17:13:15 [INFO|DP=0|PP=0|TP=0]: iteration: 24 / 150000 | consumed_samples: 1536 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788751E+05 | tokens_per_sec_per_gpu: 9.044924E+03 | global_batch_size: 64 | lm_loss: 1.127944E+01 | lr: 3.600E-06 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 2729195.000 [ip-26-0-154-121:0]:06/21/2023 17:13:16 [INFO|DP=0|PP=0|TP=0]: iteration: 25 / 150000 | consumed_samples: 1600 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781785E+05 | tokens_per_sec_per_gpu: 9.034039E+03 | global_batch_size: 64 | lm_loss: 1.132686E+01 | lr: 3.750E-06 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 2730155.000 [ip-26-0-154-121:0]:06/21/2023 17:13:17 [INFO|DP=0|PP=0|TP=0]: iteration: 26 / 150000 | consumed_samples: 1664 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784386E+05 | tokens_per_sec_per_gpu: 9.038102E+03 | global_batch_size: 64 | lm_loss: 1.126368E+01 | lr: 3.900E-06 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 2727438.750 [ip-26-0-154-121:0]:06/21/2023 17:13:18 [INFO|DP=0|PP=0|TP=0]: iteration: 27 / 150000 | consumed_samples: 1728 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781555E+05 | tokens_per_sec_per_gpu: 9.033680E+03 | global_batch_size: 64 | lm_loss: 1.130907E+01 | lr: 4.050E-06 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 2725275.750 [ip-26-0-154-121:0]:06/21/2023 17:13:19 [INFO|DP=0|PP=0|TP=0]: iteration: 28 / 150000 | consumed_samples: 1792 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780227E+05 | tokens_per_sec_per_gpu: 9.031605E+03 | global_batch_size: 64 | lm_loss: 1.136758E+01 | lr: 4.200E-06 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 2722488.750 [ip-26-0-154-121:0]:06/21/2023 17:13:20 [INFO|DP=0|PP=0|TP=0]: iteration: 29 / 150000 | consumed_samples: 1856 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783553E+05 | tokens_per_sec_per_gpu: 9.036802E+03 | global_batch_size: 64 | lm_loss: 1.133718E+01 | lr: 4.350E-06 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 2723029.750 [ip-26-0-154-121:0]:06/21/2023 17:13:20 [INFO|DP=0|PP=0|TP=0]: iteration: 30 / 150000 | consumed_samples: 1920 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785828E+05 | tokens_per_sec_per_gpu: 9.040357E+03 | global_batch_size: 64 | lm_loss: 1.130794E+01 | lr: 4.500E-06 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 2718225.750 [ip-26-0-154-121:0]:06/21/2023 17:13:21 [INFO|DP=0|PP=0|TP=0]: iteration: 31 / 150000 | consumed_samples: 1984 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.775708E+05 | tokens_per_sec_per_gpu: 9.024543E+03 | global_batch_size: 64 | lm_loss: 1.133210E+01 | lr: 4.650E-06 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 2718057.000 [ip-26-0-154-121:0]:06/21/2023 17:13:22 [INFO|DP=0|PP=0|TP=0]: iteration: 32 / 150000 | consumed_samples: 2048 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783292E+05 | tokens_per_sec_per_gpu: 9.036393E+03 | global_batch_size: 64 | lm_loss: 1.133509E+01 | lr: 4.800E-06 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 2716309.250 [ip-26-0-154-121:0]:06/21/2023 17:13:23 [INFO|DP=0|PP=0|TP=0]: iteration: 33 / 150000 | consumed_samples: 2112 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785005E+05 | tokens_per_sec_per_gpu: 9.039070E+03 | global_batch_size: 64 | lm_loss: 1.126637E+01 | lr: 4.950E-06 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 2711224.750 [ip-26-0-154-121:0]:06/21/2023 17:13:24 [INFO|DP=0|PP=0|TP=0]: iteration: 34 / 150000 | consumed_samples: 2176 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785766E+05 | tokens_per_sec_per_gpu: 9.040259E+03 | global_batch_size: 64 | lm_loss: 1.132244E+01 | lr: 5.100E-06 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 2703920.250 [ip-26-0-154-121:0]:06/21/2023 17:13:25 [INFO|DP=0|PP=0|TP=0]: iteration: 35 / 150000 | consumed_samples: 2240 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784165E+05 | tokens_per_sec_per_gpu: 9.037758E+03 | global_batch_size: 64 | lm_loss: 1.127319E+01 | lr: 5.250E-06 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 2701279.500 [ip-26-0-154-121:0]:06/21/2023 17:13:26 [INFO|DP=0|PP=0|TP=0]: iteration: 36 / 150000 | consumed_samples: 2304 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782836E+05 | tokens_per_sec_per_gpu: 9.035681E+03 | global_batch_size: 64 | lm_loss: 1.123580E+01 | lr: 5.400E-06 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 2696941.750 [ip-26-0-154-121:0]:06/21/2023 17:13:27 [INFO|DP=0|PP=0|TP=0]: iteration: 37 / 150000 | consumed_samples: 2368 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784758E+05 | tokens_per_sec_per_gpu: 9.038685E+03 | global_batch_size: 64 | lm_loss: 1.110655E+01 | lr: 5.550E-06 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 2694366.000 [ip-26-0-154-121:0]:06/21/2023 17:13:28 [INFO|DP=0|PP=0|TP=0]: iteration: 38 / 150000 | consumed_samples: 2432 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782603E+05 | tokens_per_sec_per_gpu: 9.035317E+03 | global_batch_size: 64 | lm_loss: 1.119555E+01 | lr: 5.700E-06 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 2692538.750 [ip-26-0-154-121:0]:06/21/2023 17:13:29 [INFO|DP=0|PP=0|TP=0]: iteration: 39 / 150000 | consumed_samples: 2496 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784591E+05 | tokens_per_sec_per_gpu: 9.038423E+03 | global_batch_size: 64 | lm_loss: 1.124487E+01 | lr: 5.850E-06 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 2690940.250 [ip-26-0-154-121:0]:06/21/2023 17:13:30 [INFO|DP=0|PP=0|TP=0]: iteration: 40 / 150000 | consumed_samples: 2560 | elapsed_time_per_iteration_ms: 905.0 | tokens_per_sec: 5.793390E+05 | tokens_per_sec_per_gpu: 9.052172E+03 | global_batch_size: 64 | lm_loss: 1.136754E+01 | lr: 6.000E-06 | model_tflops_per_gpu: 104.56 | hardware_tflops_per_gpu: 104.56 | grad_norm: 2691237.000 [ip-26-0-154-121:0]:06/21/2023 17:13:30 [INFO|DP=0|PP=0|TP=0]: iteration: 41 / 150000 | consumed_samples: 2624 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786587E+05 | tokens_per_sec_per_gpu: 9.041542E+03 | global_batch_size: 64 | lm_loss: 1.125544E+01 | lr: 6.150E-06 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2690108.500 [ip-26-0-154-121:0]:06/21/2023 17:13:31 [INFO|DP=0|PP=0|TP=0]: iteration: 42 / 150000 | consumed_samples: 2688 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782576E+05 | tokens_per_sec_per_gpu: 9.035274E+03 | global_batch_size: 64 | lm_loss: 1.115549E+01 | lr: 6.300E-06 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 2691639.500 [ip-26-0-154-121:0]:06/21/2023 17:13:32 [INFO|DP=0|PP=0|TP=0]: iteration: 43 / 150000 | consumed_samples: 2752 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782150E+05 | tokens_per_sec_per_gpu: 9.034609E+03 | global_batch_size: 64 | lm_loss: 1.124399E+01 | lr: 6.450E-06 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 2690670.250 [ip-26-0-154-121:0]:06/21/2023 17:13:33 [INFO|DP=0|PP=0|TP=0]: iteration: 44 / 150000 | consumed_samples: 2816 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784022E+05 | tokens_per_sec_per_gpu: 9.037534E+03 | global_batch_size: 64 | lm_loss: 1.128615E+01 | lr: 6.600E-06 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 2691347.250 [ip-26-0-154-121:0]:06/21/2023 17:13:34 [INFO|DP=0|PP=0|TP=0]: iteration: 45 / 150000 | consumed_samples: 2880 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785471E+05 | tokens_per_sec_per_gpu: 9.039798E+03 | global_batch_size: 64 | lm_loss: 1.122488E+01 | lr: 6.750E-06 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 2691752.000 [ip-26-0-154-121:0]:06/21/2023 17:13:35 [INFO|DP=0|PP=0|TP=0]: iteration: 46 / 150000 | consumed_samples: 2944 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785332E+05 | tokens_per_sec_per_gpu: 9.039581E+03 | global_batch_size: 64 | lm_loss: 1.128928E+01 | lr: 6.900E-06 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 2694173.250 [ip-26-0-154-121:0]:06/21/2023 17:13:36 [INFO|DP=0|PP=0|TP=0]: iteration: 47 / 150000 | consumed_samples: 3008 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782924E+05 | tokens_per_sec_per_gpu: 9.035818E+03 | global_batch_size: 64 | lm_loss: 1.143239E+01 | lr: 7.050E-06 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 2693705.750 [ip-26-0-154-121:0]:06/21/2023 17:13:37 [INFO|DP=0|PP=0|TP=0]: iteration: 48 / 150000 | consumed_samples: 3072 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783480E+05 | tokens_per_sec_per_gpu: 9.036688E+03 | global_batch_size: 64 | lm_loss: 1.134153E+01 | lr: 7.200E-06 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 2698809.500 [ip-26-0-154-121:0]:06/21/2023 17:13:38 [INFO|DP=0|PP=0|TP=0]: iteration: 49 / 150000 | consumed_samples: 3136 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784647E+05 | tokens_per_sec_per_gpu: 9.038511E+03 | global_batch_size: 64 | lm_loss: 1.128792E+01 | lr: 7.350E-06 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 2695810.250 [ip-26-0-154-121:0]:06/21/2023 17:13:39 [INFO|DP=0|PP=0|TP=0]: iteration: 50 / 150000 | consumed_samples: 3200 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784081E+05 | tokens_per_sec_per_gpu: 9.037627E+03 | global_batch_size: 64 | lm_loss: 1.136118E+01 | lr: 7.500E-06 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 2696124.500 [ip-26-0-154-121:0]:06/21/2023 17:13:40 [INFO|DP=0|PP=0|TP=0]: iteration: 51 / 150000 | consumed_samples: 3264 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781759E+05 | tokens_per_sec_per_gpu: 9.033999E+03 | global_batch_size: 64 | lm_loss: 1.132016E+01 | lr: 7.650E-06 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 2699571.250 [ip-26-0-154-121:0]:06/21/2023 17:13:40 [INFO|DP=0|PP=0|TP=0]: iteration: 52 / 150000 | consumed_samples: 3328 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782585E+05 | tokens_per_sec_per_gpu: 9.035288E+03 | global_batch_size: 64 | lm_loss: 1.134900E+01 | lr: 7.800E-06 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 2702980.750 [ip-26-0-154-121:0]:06/21/2023 17:13:41 [INFO|DP=0|PP=0|TP=0]: iteration: 53 / 150000 | consumed_samples: 3392 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788974E+05 | tokens_per_sec_per_gpu: 9.045271E+03 | global_batch_size: 64 | lm_loss: 1.136253E+01 | lr: 7.950E-06 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 2703191.250 [ip-26-0-154-121:0]:06/21/2023 17:13:42 [INFO|DP=0|PP=0|TP=0]: iteration: 54 / 150000 | consumed_samples: 3456 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781675E+05 | tokens_per_sec_per_gpu: 9.033868E+03 | global_batch_size: 64 | lm_loss: 1.139807E+01 | lr: 8.100E-06 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 2705256.000 [ip-26-0-154-121:0]:06/21/2023 17:13:43 [INFO|DP=0|PP=0|TP=0]: iteration: 55 / 150000 | consumed_samples: 3520 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782422E+05 | tokens_per_sec_per_gpu: 9.035034E+03 | global_batch_size: 64 | lm_loss: 1.137458E+01 | lr: 8.250E-06 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 2705829.000 [ip-26-0-154-121:0]:06/21/2023 17:13:44 [INFO|DP=0|PP=0|TP=0]: iteration: 56 / 150000 | consumed_samples: 3584 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785650E+05 | tokens_per_sec_per_gpu: 9.040079E+03 | global_batch_size: 64 | lm_loss: 1.132314E+01 | lr: 8.400E-06 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 2705123.750 [ip-26-0-154-121:0]:06/21/2023 17:13:45 [INFO|DP=0|PP=0|TP=0]: iteration: 57 / 150000 | consumed_samples: 3648 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781738E+05 | tokens_per_sec_per_gpu: 9.033965E+03 | global_batch_size: 64 | lm_loss: 1.134929E+01 | lr: 8.550E-06 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 2706513.750 [ip-26-0-154-121:0]:06/21/2023 17:13:46 [INFO|DP=0|PP=0|TP=0]: iteration: 58 / 150000 | consumed_samples: 3712 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782582E+05 | tokens_per_sec_per_gpu: 9.035284E+03 | global_batch_size: 64 | lm_loss: 1.130360E+01 | lr: 8.700E-06 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 2707594.500 [ip-26-0-154-121:0]:06/21/2023 17:13:47 [INFO|DP=0|PP=0|TP=0]: iteration: 59 / 150000 | consumed_samples: 3776 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785002E+05 | tokens_per_sec_per_gpu: 9.039065E+03 | global_batch_size: 64 | lm_loss: 1.129681E+01 | lr: 8.850E-06 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 2712489.250 [ip-26-0-154-121:0]:06/21/2023 17:13:48 [INFO|DP=0|PP=0|TP=0]: iteration: 60 / 150000 | consumed_samples: 3840 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786480E+05 | tokens_per_sec_per_gpu: 9.041375E+03 | global_batch_size: 64 | lm_loss: 1.130687E+01 | lr: 9.000E-06 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2716007.250 [ip-26-0-154-121:0]:06/21/2023 17:13:49 [INFO|DP=0|PP=0|TP=0]: iteration: 61 / 150000 | consumed_samples: 3904 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786051E+05 | tokens_per_sec_per_gpu: 9.040704E+03 | global_batch_size: 64 | lm_loss: 1.110800E+01 | lr: 9.150E-06 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 2715829.500 [ip-26-0-154-121:0]:06/21/2023 17:13:49 [INFO|DP=0|PP=0|TP=0]: iteration: 62 / 150000 | consumed_samples: 3968 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787106E+05 | tokens_per_sec_per_gpu: 9.042353E+03 | global_batch_size: 64 | lm_loss: 1.122438E+01 | lr: 9.300E-06 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 2716065.000 [ip-26-0-154-121:0]:06/21/2023 17:13:50 [INFO|DP=0|PP=0|TP=0]: iteration: 63 / 150000 | consumed_samples: 4032 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785203E+05 | tokens_per_sec_per_gpu: 9.039379E+03 | global_batch_size: 64 | lm_loss: 1.126728E+01 | lr: 9.450E-06 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 2719558.500 [ip-26-0-154-121:0]:06/21/2023 17:13:51 [INFO|DP=0|PP=0|TP=0]: iteration: 64 / 150000 | consumed_samples: 4096 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785002E+05 | tokens_per_sec_per_gpu: 9.039065E+03 | global_batch_size: 64 | lm_loss: 1.124612E+01 | lr: 9.600E-06 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 2721330.500 [ip-26-0-154-121:0]:06/21/2023 17:13:52 [INFO|DP=0|PP=0|TP=0]: iteration: 65 / 150000 | consumed_samples: 4160 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786389E+05 | tokens_per_sec_per_gpu: 9.041232E+03 | global_batch_size: 64 | lm_loss: 1.129102E+01 | lr: 9.750E-06 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2724852.250 [ip-26-0-154-121:0]:06/21/2023 17:13:53 [INFO|DP=0|PP=0|TP=0]: iteration: 66 / 150000 | consumed_samples: 4224 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786684E+05 | tokens_per_sec_per_gpu: 9.041694E+03 | global_batch_size: 64 | lm_loss: 1.127793E+01 | lr: 9.900E-06 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2727458.000 [ip-26-0-154-121:0]:06/21/2023 17:13:54 [INFO|DP=0|PP=0|TP=0]: iteration: 67 / 150000 | consumed_samples: 4288 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779566E+05 | tokens_per_sec_per_gpu: 9.030572E+03 | global_batch_size: 64 | lm_loss: 1.122266E+01 | lr: 1.005E-05 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 2726544.250 [ip-26-0-154-121:0]:06/21/2023 17:13:55 [INFO|DP=0|PP=0|TP=0]: iteration: 68 / 150000 | consumed_samples: 4352 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786725E+05 | tokens_per_sec_per_gpu: 9.041758E+03 | global_batch_size: 64 | lm_loss: 1.129015E+01 | lr: 1.020E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2734474.250 [ip-26-0-154-121:0]:06/21/2023 17:13:56 [INFO|DP=0|PP=0|TP=0]: iteration: 69 / 150000 | consumed_samples: 4416 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786043E+05 | tokens_per_sec_per_gpu: 9.040692E+03 | global_batch_size: 64 | lm_loss: 1.110947E+01 | lr: 1.035E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 2739501.750 [ip-26-0-154-121:0]:06/21/2023 17:13:57 [INFO|DP=0|PP=0|TP=0]: iteration: 70 / 150000 | consumed_samples: 4480 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784731E+05 | tokens_per_sec_per_gpu: 9.038642E+03 | global_batch_size: 64 | lm_loss: 1.125624E+01 | lr: 1.050E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 2748815.250 [ip-26-0-154-121:0]:06/21/2023 17:13:58 [INFO|DP=0|PP=0|TP=0]: iteration: 71 / 150000 | consumed_samples: 4544 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779194E+05 | tokens_per_sec_per_gpu: 9.029991E+03 | global_batch_size: 64 | lm_loss: 1.125892E+01 | lr: 1.065E-05 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 2750242.750 [ip-26-0-154-121:0]:06/21/2023 17:13:59 [INFO|DP=0|PP=0|TP=0]: iteration: 72 / 150000 | consumed_samples: 4608 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788027E+05 | tokens_per_sec_per_gpu: 9.043793E+03 | global_batch_size: 64 | lm_loss: 1.122857E+01 | lr: 1.080E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 2757370.250 [ip-26-0-154-121:0]:06/21/2023 17:13:59 [INFO|DP=0|PP=0|TP=0]: iteration: 73 / 150000 | consumed_samples: 4672 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786057E+05 | tokens_per_sec_per_gpu: 9.040714E+03 | global_batch_size: 64 | lm_loss: 1.123703E+01 | lr: 1.095E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 2759064.750 [ip-26-0-154-121:0]:06/21/2023 17:14:00 [INFO|DP=0|PP=0|TP=0]: iteration: 74 / 150000 | consumed_samples: 4736 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782872E+05 | tokens_per_sec_per_gpu: 9.035738E+03 | global_batch_size: 64 | lm_loss: 1.128650E+01 | lr: 1.110E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 2760898.250 [ip-26-0-154-121:0]:06/21/2023 17:14:01 [INFO|DP=0|PP=0|TP=0]: iteration: 75 / 150000 | consumed_samples: 4800 | elapsed_time_per_iteration_ms: 904.8 | tokens_per_sec: 5.794296E+05 | tokens_per_sec_per_gpu: 9.053587E+03 | global_batch_size: 64 | lm_loss: 1.130974E+01 | lr: 1.125E-05 | model_tflops_per_gpu: 104.58 | hardware_tflops_per_gpu: 104.58 | grad_norm: 2771438.000 [ip-26-0-154-121:0]:06/21/2023 17:14:02 [INFO|DP=0|PP=0|TP=0]: iteration: 76 / 150000 | consumed_samples: 4864 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784355E+05 | tokens_per_sec_per_gpu: 9.038055E+03 | global_batch_size: 64 | lm_loss: 1.118525E+01 | lr: 1.140E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 2791731.500 [ip-26-0-154-121:0]:06/21/2023 17:14:03 [INFO|DP=0|PP=0|TP=0]: iteration: 77 / 150000 | consumed_samples: 4928 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787816E+05 | tokens_per_sec_per_gpu: 9.043462E+03 | global_batch_size: 64 | lm_loss: 1.133557E+01 | lr: 1.155E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 2806460.000 [ip-26-0-154-121:0]:06/21/2023 17:14:04 [INFO|DP=0|PP=0|TP=0]: iteration: 78 / 150000 | consumed_samples: 4992 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786608E+05 | tokens_per_sec_per_gpu: 9.041575E+03 | global_batch_size: 64 | lm_loss: 1.139178E+01 | lr: 1.170E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2818592.250 [ip-26-0-154-121:0]:06/21/2023 17:14:05 [INFO|DP=0|PP=0|TP=0]: iteration: 79 / 150000 | consumed_samples: 5056 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786136E+05 | tokens_per_sec_per_gpu: 9.040837E+03 | global_batch_size: 64 | lm_loss: 1.136041E+01 | lr: 1.185E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 2831583.500 [ip-26-0-154-121:0]:06/21/2023 17:14:06 [INFO|DP=0|PP=0|TP=0]: iteration: 80 / 150000 | consumed_samples: 5120 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786769E+05 | tokens_per_sec_per_gpu: 9.041827E+03 | global_batch_size: 64 | lm_loss: 1.128443E+01 | lr: 1.200E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2851380.000 [ip-26-0-154-121:0]:06/21/2023 17:14:07 [INFO|DP=0|PP=0|TP=0]: iteration: 81 / 150000 | consumed_samples: 5184 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783868E+05 | tokens_per_sec_per_gpu: 9.037294E+03 | global_batch_size: 64 | lm_loss: 1.126523E+01 | lr: 1.215E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 2872513.750 [ip-26-0-154-121:0]:06/21/2023 17:14:08 [INFO|DP=0|PP=0|TP=0]: iteration: 82 / 150000 | consumed_samples: 5248 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787316E+05 | tokens_per_sec_per_gpu: 9.042681E+03 | global_batch_size: 64 | lm_loss: 1.127977E+01 | lr: 1.230E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 2892766.000 [ip-26-0-154-121:0]:06/21/2023 17:14:09 [INFO|DP=0|PP=0|TP=0]: iteration: 83 / 150000 | consumed_samples: 5312 | elapsed_time_per_iteration_ms: 904.4 | tokens_per_sec: 5.796773E+05 | tokens_per_sec_per_gpu: 9.057458E+03 | global_batch_size: 64 | lm_loss: 1.129905E+01 | lr: 1.245E-05 | model_tflops_per_gpu: 104.62 | hardware_tflops_per_gpu: 104.62 | grad_norm: 2907067.250 [ip-26-0-154-121:0]:06/21/2023 17:14:09 [INFO|DP=0|PP=0|TP=0]: iteration: 84 / 150000 | consumed_samples: 5376 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789134E+05 | tokens_per_sec_per_gpu: 9.045521E+03 | global_batch_size: 64 | lm_loss: 1.134858E+01 | lr: 1.260E-05 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 2931178.500 [ip-26-0-154-121:0]:06/21/2023 17:14:10 [INFO|DP=0|PP=0|TP=0]: iteration: 85 / 150000 | consumed_samples: 5440 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789827E+05 | tokens_per_sec_per_gpu: 9.046605E+03 | global_batch_size: 64 | lm_loss: 1.123650E+01 | lr: 1.275E-05 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 2949062.500 [ip-26-0-154-121:0]:06/21/2023 17:14:11 [INFO|DP=0|PP=0|TP=0]: iteration: 86 / 150000 | consumed_samples: 5504 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784349E+05 | tokens_per_sec_per_gpu: 9.038045E+03 | global_batch_size: 64 | lm_loss: 1.138417E+01 | lr: 1.290E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 2968375.500 [ip-26-0-154-121:0]:06/21/2023 17:14:12 [INFO|DP=0|PP=0|TP=0]: iteration: 87 / 150000 | consumed_samples: 5568 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780923E+05 | tokens_per_sec_per_gpu: 9.032692E+03 | global_batch_size: 64 | lm_loss: 1.132360E+01 | lr: 1.305E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 2991970.250 [ip-26-0-154-121:0]:06/21/2023 17:14:13 [INFO|DP=0|PP=0|TP=0]: iteration: 88 / 150000 | consumed_samples: 5632 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784170E+05 | tokens_per_sec_per_gpu: 9.037765E+03 | global_batch_size: 64 | lm_loss: 1.119613E+01 | lr: 1.320E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3017793.750 [ip-26-0-154-121:0]:06/21/2023 17:14:14 [INFO|DP=0|PP=0|TP=0]: iteration: 89 / 150000 | consumed_samples: 5696 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789406E+05 | tokens_per_sec_per_gpu: 9.045948E+03 | global_batch_size: 64 | lm_loss: 1.131017E+01 | lr: 1.335E-05 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 3042368.000 [ip-26-0-154-121:0]:06/21/2023 17:14:15 [INFO|DP=0|PP=0|TP=0]: iteration: 90 / 150000 | consumed_samples: 5760 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787543E+05 | tokens_per_sec_per_gpu: 9.043036E+03 | global_batch_size: 64 | lm_loss: 1.123054E+01 | lr: 1.350E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3072944.750 [ip-26-0-154-121:0]:06/21/2023 17:14:16 [INFO|DP=0|PP=0|TP=0]: iteration: 91 / 150000 | consumed_samples: 5824 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787345E+05 | tokens_per_sec_per_gpu: 9.042727E+03 | global_batch_size: 64 | lm_loss: 1.128947E+01 | lr: 1.365E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3111447.750 [ip-26-0-154-121:0]:06/21/2023 17:14:17 [INFO|DP=0|PP=0|TP=0]: iteration: 92 / 150000 | consumed_samples: 5888 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788468E+05 | tokens_per_sec_per_gpu: 9.044481E+03 | global_batch_size: 64 | lm_loss: 1.121503E+01 | lr: 1.380E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 3147550.000 [ip-26-0-154-121:0]:06/21/2023 17:14:18 [INFO|DP=0|PP=0|TP=0]: iteration: 93 / 150000 | consumed_samples: 5952 | elapsed_time_per_iteration_ms: 904.8 | tokens_per_sec: 5.794593E+05 | tokens_per_sec_per_gpu: 9.054052E+03 | global_batch_size: 64 | lm_loss: 1.126038E+01 | lr: 1.395E-05 | model_tflops_per_gpu: 104.58 | hardware_tflops_per_gpu: 104.58 | grad_norm: 3195712.500 [ip-26-0-154-121:0]:06/21/2023 17:14:19 [INFO|DP=0|PP=0|TP=0]: iteration: 94 / 150000 | consumed_samples: 6016 | elapsed_time_per_iteration_ms: 904.4 | tokens_per_sec: 5.797277E+05 | tokens_per_sec_per_gpu: 9.058246E+03 | global_batch_size: 64 | lm_loss: 1.112224E+01 | lr: 1.410E-05 | model_tflops_per_gpu: 104.63 | hardware_tflops_per_gpu: 104.63 | grad_norm: 3234036.250 [ip-26-0-154-121:0]:06/21/2023 17:14:19 [INFO|DP=0|PP=0|TP=0]: iteration: 95 / 150000 | consumed_samples: 6080 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789118E+05 | tokens_per_sec_per_gpu: 9.045498E+03 | global_batch_size: 64 | lm_loss: 1.122448E+01 | lr: 1.425E-05 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 3269518.000 [ip-26-0-154-121:0]:06/21/2023 17:14:20 [INFO|DP=0|PP=0|TP=0]: iteration: 96 / 150000 | consumed_samples: 6144 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789567E+05 | tokens_per_sec_per_gpu: 9.046198E+03 | global_batch_size: 64 | lm_loss: 1.115002E+01 | lr: 1.440E-05 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 3302625.500 [ip-26-0-154-121:0]:06/21/2023 17:14:21 [INFO|DP=0|PP=0|TP=0]: iteration: 97 / 150000 | consumed_samples: 6208 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787775E+05 | tokens_per_sec_per_gpu: 9.043398E+03 | global_batch_size: 64 | lm_loss: 1.127715E+01 | lr: 1.455E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3336833.250 [ip-26-0-154-121:0]:06/21/2023 17:14:22 [INFO|DP=0|PP=0|TP=0]: iteration: 98 / 150000 | consumed_samples: 6272 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783482E+05 | tokens_per_sec_per_gpu: 9.036690E+03 | global_batch_size: 64 | lm_loss: 1.118300E+01 | lr: 1.470E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3366506.250 [ip-26-0-154-121:0]:06/21/2023 17:14:23 [INFO|DP=0|PP=0|TP=0]: iteration: 99 / 150000 | consumed_samples: 6336 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784113E+05 | tokens_per_sec_per_gpu: 9.037677E+03 | global_batch_size: 64 | lm_loss: 1.104807E+01 | lr: 1.485E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3394185.750 [ip-26-0-154-121:0]:06/21/2023 17:14:24 [INFO|DP=0|PP=0|TP=0]: iteration: 100 / 150000 | consumed_samples: 6400 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786908E+05 | tokens_per_sec_per_gpu: 9.042044E+03 | global_batch_size: 64 | lm_loss: 1.125530E+01 | lr: 1.500E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3426798.250 [ip-26-0-154-121:0]:06/21/2023 17:14:25 [INFO|DP=0|PP=0|TP=0]: iteration: 101 / 150000 | consumed_samples: 6464 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782936E+05 | tokens_per_sec_per_gpu: 9.035837E+03 | global_batch_size: 64 | lm_loss: 1.146535E+01 | lr: 1.515E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3461937.250 [ip-26-0-154-121:0]:06/21/2023 17:14:26 [INFO|DP=0|PP=0|TP=0]: iteration: 102 / 150000 | consumed_samples: 6528 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782586E+05 | tokens_per_sec_per_gpu: 9.035291E+03 | global_batch_size: 64 | lm_loss: 1.134027E+01 | lr: 1.530E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3503290.250 [ip-26-0-154-121:0]:06/21/2023 17:14:27 [INFO|DP=0|PP=0|TP=0]: iteration: 103 / 150000 | consumed_samples: 6592 | elapsed_time_per_iteration_ms: 905.1 | tokens_per_sec: 5.792301E+05 | tokens_per_sec_per_gpu: 9.050470E+03 | global_batch_size: 64 | lm_loss: 1.125573E+01 | lr: 1.545E-05 | model_tflops_per_gpu: 104.54 | hardware_tflops_per_gpu: 104.54 | grad_norm: 3538312.750 [ip-26-0-154-121:0]:06/21/2023 17:14:28 [INFO|DP=0|PP=0|TP=0]: iteration: 104 / 150000 | consumed_samples: 6656 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787825E+05 | tokens_per_sec_per_gpu: 9.043476E+03 | global_batch_size: 64 | lm_loss: 1.127545E+01 | lr: 1.560E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3562449.750 [ip-26-0-154-121:0]:06/21/2023 17:14:29 [INFO|DP=0|PP=0|TP=0]: iteration: 105 / 150000 | consumed_samples: 6720 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787855E+05 | tokens_per_sec_per_gpu: 9.043524E+03 | global_batch_size: 64 | lm_loss: 1.120245E+01 | lr: 1.575E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3576042.250 [ip-26-0-154-121:0]:06/21/2023 17:14:29 [INFO|DP=0|PP=0|TP=0]: iteration: 106 / 150000 | consumed_samples: 6784 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783661E+05 | tokens_per_sec_per_gpu: 9.036971E+03 | global_batch_size: 64 | lm_loss: 1.126130E+01 | lr: 1.590E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3593465.750 [ip-26-0-154-121:0]:06/21/2023 17:14:30 [INFO|DP=0|PP=0|TP=0]: iteration: 107 / 150000 | consumed_samples: 6848 | elapsed_time_per_iteration_ms: 905.1 | tokens_per_sec: 5.792794E+05 | tokens_per_sec_per_gpu: 9.051240E+03 | global_batch_size: 64 | lm_loss: 1.118159E+01 | lr: 1.605E-05 | model_tflops_per_gpu: 104.55 | hardware_tflops_per_gpu: 104.55 | grad_norm: 3612840.000 [ip-26-0-154-121:0]:06/21/2023 17:14:31 [INFO|DP=0|PP=0|TP=0]: iteration: 108 / 150000 | consumed_samples: 6912 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780995E+05 | tokens_per_sec_per_gpu: 9.032804E+03 | global_batch_size: 64 | lm_loss: 1.130796E+01 | lr: 1.620E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3629347.250 [ip-26-0-154-121:0]:06/21/2023 17:14:32 [INFO|DP=0|PP=0|TP=0]: iteration: 109 / 150000 | consumed_samples: 6976 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785366E+05 | tokens_per_sec_per_gpu: 9.039634E+03 | global_batch_size: 64 | lm_loss: 1.132406E+01 | lr: 1.635E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3644591.750 [ip-26-0-154-121:0]:06/21/2023 17:14:33 [INFO|DP=0|PP=0|TP=0]: iteration: 110 / 150000 | consumed_samples: 7040 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786507E+05 | tokens_per_sec_per_gpu: 9.041418E+03 | global_batch_size: 64 | lm_loss: 1.126451E+01 | lr: 1.650E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3655416.750 [ip-26-0-154-121:0]:06/21/2023 17:14:34 [INFO|DP=0|PP=0|TP=0]: iteration: 111 / 150000 | consumed_samples: 7104 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790496E+05 | tokens_per_sec_per_gpu: 9.047651E+03 | global_batch_size: 64 | lm_loss: 1.130538E+01 | lr: 1.665E-05 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 3664946.500 [ip-26-0-154-121:0]:06/21/2023 17:14:35 [INFO|DP=0|PP=0|TP=0]: iteration: 112 / 150000 | consumed_samples: 7168 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788064E+05 | tokens_per_sec_per_gpu: 9.043850E+03 | global_batch_size: 64 | lm_loss: 1.118199E+01 | lr: 1.680E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 3672914.000 [ip-26-0-154-121:0]:06/21/2023 17:14:36 [INFO|DP=0|PP=0|TP=0]: iteration: 113 / 150000 | consumed_samples: 7232 | elapsed_time_per_iteration_ms: 904.7 | tokens_per_sec: 5.795433E+05 | tokens_per_sec_per_gpu: 9.055364E+03 | global_batch_size: 64 | lm_loss: 1.125194E+01 | lr: 1.695E-05 | model_tflops_per_gpu: 104.60 | hardware_tflops_per_gpu: 104.60 | grad_norm: 3676176.500 [ip-26-0-154-121:0]:06/21/2023 17:14:37 [INFO|DP=0|PP=0|TP=0]: iteration: 114 / 150000 | consumed_samples: 7296 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789661E+05 | tokens_per_sec_per_gpu: 9.046345E+03 | global_batch_size: 64 | lm_loss: 1.134168E+01 | lr: 1.710E-05 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 3681760.000 [ip-26-0-154-121:0]:06/21/2023 17:14:38 [INFO|DP=0|PP=0|TP=0]: iteration: 115 / 150000 | consumed_samples: 7360 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790195E+05 | tokens_per_sec_per_gpu: 9.047179E+03 | global_batch_size: 64 | lm_loss: 1.121826E+01 | lr: 1.725E-05 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 3688060.000 [ip-26-0-154-121:0]:06/21/2023 17:14:39 [INFO|DP=0|PP=0|TP=0]: iteration: 116 / 150000 | consumed_samples: 7424 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788044E+05 | tokens_per_sec_per_gpu: 9.043819E+03 | global_batch_size: 64 | lm_loss: 1.120510E+01 | lr: 1.740E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 3691937.250 [ip-26-0-154-121:0]:06/21/2023 17:14:39 [INFO|DP=0|PP=0|TP=0]: iteration: 117 / 150000 | consumed_samples: 7488 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782839E+05 | tokens_per_sec_per_gpu: 9.035685E+03 | global_batch_size: 64 | lm_loss: 1.125864E+01 | lr: 1.755E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3694791.750 [ip-26-0-154-121:0]:06/21/2023 17:14:40 [INFO|DP=0|PP=0|TP=0]: iteration: 118 / 150000 | consumed_samples: 7552 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785665E+05 | tokens_per_sec_per_gpu: 9.040102E+03 | global_batch_size: 64 | lm_loss: 1.117951E+01 | lr: 1.770E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3696906.250 [ip-26-0-154-121:0]:06/21/2023 17:14:41 [INFO|DP=0|PP=0|TP=0]: iteration: 119 / 150000 | consumed_samples: 7616 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791087E+05 | tokens_per_sec_per_gpu: 9.048573E+03 | global_batch_size: 64 | lm_loss: 1.125314E+01 | lr: 1.785E-05 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 3698125.750 [ip-26-0-154-121:0]:06/21/2023 17:14:42 [INFO|DP=0|PP=0|TP=0]: iteration: 120 / 150000 | consumed_samples: 7680 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786469E+05 | tokens_per_sec_per_gpu: 9.041358E+03 | global_batch_size: 64 | lm_loss: 1.107393E+01 | lr: 1.800E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3698128.000 [ip-26-0-154-121:0]:06/21/2023 17:14:43 [INFO|DP=0|PP=0|TP=0]: iteration: 121 / 150000 | consumed_samples: 7744 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787983E+05 | tokens_per_sec_per_gpu: 9.043724E+03 | global_batch_size: 64 | lm_loss: 1.116362E+01 | lr: 1.815E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 3699341.750 [ip-26-0-154-121:0]:06/21/2023 17:14:44 [INFO|DP=0|PP=0|TP=0]: iteration: 122 / 150000 | consumed_samples: 7808 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784594E+05 | tokens_per_sec_per_gpu: 9.038428E+03 | global_batch_size: 64 | lm_loss: 1.132048E+01 | lr: 1.830E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3701213.000 [ip-26-0-154-121:0]:06/21/2023 17:14:45 [INFO|DP=0|PP=0|TP=0]: iteration: 123 / 150000 | consumed_samples: 7872 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783280E+05 | tokens_per_sec_per_gpu: 9.036374E+03 | global_batch_size: 64 | lm_loss: 1.117099E+01 | lr: 1.845E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3705152.000 [ip-26-0-154-121:0]:06/21/2023 17:14:46 [INFO|DP=0|PP=0|TP=0]: iteration: 124 / 150000 | consumed_samples: 7936 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785355E+05 | tokens_per_sec_per_gpu: 9.039617E+03 | global_batch_size: 64 | lm_loss: 1.118265E+01 | lr: 1.860E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3709306.750 [ip-26-0-154-121:0]:06/21/2023 17:14:47 [INFO|DP=0|PP=0|TP=0]: iteration: 125 / 150000 | consumed_samples: 8000 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778828E+05 | tokens_per_sec_per_gpu: 9.029419E+03 | global_batch_size: 64 | lm_loss: 1.123837E+01 | lr: 1.875E-05 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3715756.750 [ip-26-0-154-121:0]:06/21/2023 17:14:48 [INFO|DP=0|PP=0|TP=0]: iteration: 126 / 150000 | consumed_samples: 8064 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780271E+05 | tokens_per_sec_per_gpu: 9.031674E+03 | global_batch_size: 64 | lm_loss: 1.063576E+01 | lr: 1.890E-05 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3720213.500 [ip-26-0-154-121:0]:06/21/2023 17:14:48 [INFO|DP=0|PP=0|TP=0]: iteration: 127 / 150000 | consumed_samples: 8128 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785186E+05 | tokens_per_sec_per_gpu: 9.039353E+03 | global_batch_size: 64 | lm_loss: 1.112781E+01 | lr: 1.905E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3723946.000 [ip-26-0-154-121:0]:06/21/2023 17:14:49 [INFO|DP=0|PP=0|TP=0]: iteration: 128 / 150000 | consumed_samples: 8192 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784742E+05 | tokens_per_sec_per_gpu: 9.038659E+03 | global_batch_size: 64 | lm_loss: 1.081104E+01 | lr: 1.920E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3726060.750 [ip-26-0-154-121:0]:06/21/2023 17:14:50 [INFO|DP=0|PP=0|TP=0]: iteration: 129 / 150000 | consumed_samples: 8256 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786046E+05 | tokens_per_sec_per_gpu: 9.040697E+03 | global_batch_size: 64 | lm_loss: 1.124340E+01 | lr: 1.935E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3725864.750 [ip-26-0-154-121:0]:06/21/2023 17:14:51 [INFO|DP=0|PP=0|TP=0]: iteration: 130 / 150000 | consumed_samples: 8320 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785863E+05 | tokens_per_sec_per_gpu: 9.040412E+03 | global_batch_size: 64 | lm_loss: 1.125957E+01 | lr: 1.950E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3724730.250 [ip-26-0-154-121:0]:06/21/2023 17:14:52 [INFO|DP=0|PP=0|TP=0]: iteration: 131 / 150000 | consumed_samples: 8384 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784583E+05 | tokens_per_sec_per_gpu: 9.038412E+03 | global_batch_size: 64 | lm_loss: 1.125910E+01 | lr: 1.965E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3724418.500 [ip-26-0-154-121:0]:06/21/2023 17:14:53 [INFO|DP=0|PP=0|TP=0]: iteration: 132 / 150000 | consumed_samples: 8448 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784198E+05 | tokens_per_sec_per_gpu: 9.037810E+03 | global_batch_size: 64 | lm_loss: 1.121938E+01 | lr: 1.980E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3727007.000 [ip-26-0-154-121:0]:06/21/2023 17:14:54 [INFO|DP=0|PP=0|TP=0]: iteration: 133 / 150000 | consumed_samples: 8512 | elapsed_time_per_iteration_ms: 904.9 | tokens_per_sec: 5.793948E+05 | tokens_per_sec_per_gpu: 9.053043E+03 | global_batch_size: 64 | lm_loss: 1.125534E+01 | lr: 1.995E-05 | model_tflops_per_gpu: 104.57 | hardware_tflops_per_gpu: 104.57 | grad_norm: 3733272.500 [ip-26-0-154-121:0]:06/21/2023 17:14:55 [INFO|DP=0|PP=0|TP=0]: iteration: 134 / 150000 | consumed_samples: 8576 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785758E+05 | tokens_per_sec_per_gpu: 9.040247E+03 | global_batch_size: 64 | lm_loss: 1.117842E+01 | lr: 2.010E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3738163.750 [ip-26-0-154-121:0]:06/21/2023 17:14:56 [INFO|DP=0|PP=0|TP=0]: iteration: 135 / 150000 | consumed_samples: 8640 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787212E+05 | tokens_per_sec_per_gpu: 9.042519E+03 | global_batch_size: 64 | lm_loss: 1.134311E+01 | lr: 2.025E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3741085.750 [ip-26-0-154-121:0]:06/21/2023 17:14:57 [INFO|DP=0|PP=0|TP=0]: iteration: 136 / 150000 | consumed_samples: 8704 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787059E+05 | tokens_per_sec_per_gpu: 9.042279E+03 | global_batch_size: 64 | lm_loss: 1.118125E+01 | lr: 2.040E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3743221.000 [ip-26-0-154-121:0]:06/21/2023 17:14:58 [INFO|DP=0|PP=0|TP=0]: iteration: 137 / 150000 | consumed_samples: 8768 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790346E+05 | tokens_per_sec_per_gpu: 9.047415E+03 | global_batch_size: 64 | lm_loss: 1.135051E+01 | lr: 2.055E-05 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 3744039.000 [ip-26-0-154-121:0]:06/21/2023 17:14:58 [INFO|DP=0|PP=0|TP=0]: iteration: 138 / 150000 | consumed_samples: 8832 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790092E+05 | tokens_per_sec_per_gpu: 9.047019E+03 | global_batch_size: 64 | lm_loss: 1.114974E+01 | lr: 2.070E-05 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 3742966.750 [ip-26-0-154-121:0]:06/21/2023 17:14:59 [INFO|DP=0|PP=0|TP=0]: iteration: 139 / 150000 | consumed_samples: 8896 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785466E+05 | tokens_per_sec_per_gpu: 9.039791E+03 | global_batch_size: 64 | lm_loss: 1.127138E+01 | lr: 2.085E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3742736.750 [ip-26-0-154-121:0]:06/21/2023 17:15:00 [INFO|DP=0|PP=0|TP=0]: iteration: 140 / 150000 | consumed_samples: 8960 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785522E+05 | tokens_per_sec_per_gpu: 9.039879E+03 | global_batch_size: 64 | lm_loss: 1.123721E+01 | lr: 2.100E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3743141.500 [ip-26-0-154-121:0]:06/21/2023 17:15:01 [INFO|DP=0|PP=0|TP=0]: iteration: 141 / 150000 | consumed_samples: 9024 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785369E+05 | tokens_per_sec_per_gpu: 9.039639E+03 | global_batch_size: 64 | lm_loss: 1.122475E+01 | lr: 2.115E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3745039.000 [ip-26-0-154-121:0]:06/21/2023 17:15:02 [INFO|DP=0|PP=0|TP=0]: iteration: 142 / 150000 | consumed_samples: 9088 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781475E+05 | tokens_per_sec_per_gpu: 9.033554E+03 | global_batch_size: 64 | lm_loss: 1.118054E+01 | lr: 2.130E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3747449.750 [ip-26-0-154-121:0]:06/21/2023 17:15:03 [INFO|DP=0|PP=0|TP=0]: iteration: 143 / 150000 | consumed_samples: 9152 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784115E+05 | tokens_per_sec_per_gpu: 9.037679E+03 | global_batch_size: 64 | lm_loss: 1.129802E+01 | lr: 2.145E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3748644.500 [ip-26-0-154-121:0]:06/21/2023 17:15:04 [INFO|DP=0|PP=0|TP=0]: iteration: 144 / 150000 | consumed_samples: 9216 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781561E+05 | tokens_per_sec_per_gpu: 9.033690E+03 | global_batch_size: 64 | lm_loss: 1.117590E+01 | lr: 2.160E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3749657.000 [ip-26-0-154-121:0]:06/21/2023 17:15:05 [INFO|DP=0|PP=0|TP=0]: iteration: 145 / 150000 | consumed_samples: 9280 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779540E+05 | tokens_per_sec_per_gpu: 9.030532E+03 | global_batch_size: 64 | lm_loss: 1.125221E+01 | lr: 2.175E-05 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 3749831.500 [ip-26-0-154-121:0]:06/21/2023 17:15:06 [INFO|DP=0|PP=0|TP=0]: iteration: 146 / 150000 | consumed_samples: 9344 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787842E+05 | tokens_per_sec_per_gpu: 9.043502E+03 | global_batch_size: 64 | lm_loss: 1.129962E+01 | lr: 2.190E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3750716.500 [ip-26-0-154-121:0]:06/21/2023 17:15:07 [INFO|DP=0|PP=0|TP=0]: iteration: 147 / 150000 | consumed_samples: 9408 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784927E+05 | tokens_per_sec_per_gpu: 9.038949E+03 | global_batch_size: 64 | lm_loss: 1.130404E+01 | lr: 2.205E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3751649.000 [ip-26-0-154-121:0]:06/21/2023 17:15:08 [INFO|DP=0|PP=0|TP=0]: iteration: 148 / 150000 | consumed_samples: 9472 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786070E+05 | tokens_per_sec_per_gpu: 9.040735E+03 | global_batch_size: 64 | lm_loss: 1.121896E+01 | lr: 2.220E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3752531.250 [ip-26-0-154-121:0]:06/21/2023 17:15:08 [INFO|DP=0|PP=0|TP=0]: iteration: 149 / 150000 | consumed_samples: 9536 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786824E+05 | tokens_per_sec_per_gpu: 9.041913E+03 | global_batch_size: 64 | lm_loss: 1.126455E+01 | lr: 2.235E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3752194.750 [ip-26-0-154-121:0]:06/21/2023 17:15:09 [INFO|DP=0|PP=0|TP=0]: iteration: 150 / 150000 | consumed_samples: 9600 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786821E+05 | tokens_per_sec_per_gpu: 9.041908E+03 | global_batch_size: 64 | lm_loss: 1.121209E+01 | lr: 2.250E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3751646.750 [ip-26-0-154-121:0]:06/21/2023 17:15:10 [INFO|DP=0|PP=0|TP=0]: iteration: 151 / 150000 | consumed_samples: 9664 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789253E+05 | tokens_per_sec_per_gpu: 9.045707E+03 | global_batch_size: 64 | lm_loss: 1.125487E+01 | lr: 2.265E-05 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 3750583.500 [ip-26-0-154-121:0]:06/21/2023 17:15:11 [INFO|DP=0|PP=0|TP=0]: iteration: 152 / 150000 | consumed_samples: 9728 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783823E+05 | tokens_per_sec_per_gpu: 9.037223E+03 | global_batch_size: 64 | lm_loss: 1.123166E+01 | lr: 2.280E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3749629.250 [ip-26-0-154-121:0]:06/21/2023 17:15:12 [INFO|DP=0|PP=0|TP=0]: iteration: 153 / 150000 | consumed_samples: 9792 | elapsed_time_per_iteration_ms: 905.2 | tokens_per_sec: 5.791871E+05 | tokens_per_sec_per_gpu: 9.049798E+03 | global_batch_size: 64 | lm_loss: 1.129999E+01 | lr: 2.295E-05 | model_tflops_per_gpu: 104.54 | hardware_tflops_per_gpu: 104.54 | grad_norm: 3755276.250 [ip-26-0-154-121:0]:06/21/2023 17:15:13 [INFO|DP=0|PP=0|TP=0]: iteration: 154 / 150000 | consumed_samples: 9856 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785886E+05 | tokens_per_sec_per_gpu: 9.040447E+03 | global_batch_size: 64 | lm_loss: 1.130202E+01 | lr: 2.310E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3810444.500 [ip-26-0-154-121:0]:06/21/2023 17:15:14 [INFO|DP=0|PP=0|TP=0]: iteration: 155 / 150000 | consumed_samples: 9920 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782401E+05 | tokens_per_sec_per_gpu: 9.035001E+03 | global_batch_size: 64 | lm_loss: 1.116440E+01 | lr: 2.325E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 4126103.000 [ip-26-0-154-121:0]:06/21/2023 17:15:15 [INFO|DP=0|PP=0|TP=0]: iteration: 156 / 150000 | consumed_samples: 9984 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783064E+05 | tokens_per_sec_per_gpu: 9.036037E+03 | global_batch_size: 64 | lm_loss: 1.122109E+01 | lr: 2.340E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 4854479.000 [ip-26-0-154-121:0]:06/21/2023 17:15:16 [INFO|DP=0|PP=0|TP=0]: iteration: 157 / 150000 | consumed_samples: 10048 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784075E+05 | tokens_per_sec_per_gpu: 9.037617E+03 | global_batch_size: 64 | lm_loss: 1.113436E+01 | lr: 2.355E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 5648486.500 [ip-26-0-154-121:0]:06/21/2023 17:15:17 [INFO|DP=0|PP=0|TP=0]: iteration: 158 / 150000 | consumed_samples: 10112 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786906E+05 | tokens_per_sec_per_gpu: 9.042041E+03 | global_batch_size: 64 | lm_loss: 1.120999E+01 | lr: 2.370E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 6200587.000 [ip-26-0-154-121:0]:06/21/2023 17:15:18 [INFO|DP=0|PP=0|TP=0]: iteration: 159 / 150000 | consumed_samples: 10176 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784186E+05 | tokens_per_sec_per_gpu: 9.037791E+03 | global_batch_size: 64 | lm_loss: 1.126694E+01 | lr: 2.385E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 6535885.000 [ip-26-0-154-121:0]:06/21/2023 17:15:18 [INFO|DP=0|PP=0|TP=0]: iteration: 160 / 150000 | consumed_samples: 10240 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785932E+05 | tokens_per_sec_per_gpu: 9.040519E+03 | global_batch_size: 64 | lm_loss: 1.124480E+01 | lr: 2.400E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 6756217.000 [ip-26-0-154-121:0]:06/21/2023 17:15:19 [INFO|DP=0|PP=0|TP=0]: iteration: 161 / 150000 | consumed_samples: 10304 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781093E+05 | tokens_per_sec_per_gpu: 9.032958E+03 | global_batch_size: 64 | lm_loss: 1.133193E+01 | lr: 2.415E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 6919563.000 [ip-26-0-154-121:0]:06/21/2023 17:15:20 [INFO|DP=0|PP=0|TP=0]: iteration: 162 / 150000 | consumed_samples: 10368 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785653E+05 | tokens_per_sec_per_gpu: 9.040083E+03 | global_batch_size: 64 | lm_loss: 1.138409E+01 | lr: 2.430E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 7028067.500 [ip-26-0-154-121:0]:06/21/2023 17:15:21 [INFO|DP=0|PP=0|TP=0]: iteration: 163 / 150000 | consumed_samples: 10432 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781028E+05 | tokens_per_sec_per_gpu: 9.032856E+03 | global_batch_size: 64 | lm_loss: 1.131164E+01 | lr: 2.445E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 7094505.000 [ip-26-0-154-121:0]:06/21/2023 17:15:22 [INFO|DP=0|PP=0|TP=0]: iteration: 164 / 150000 | consumed_samples: 10496 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782306E+05 | tokens_per_sec_per_gpu: 9.034854E+03 | global_batch_size: 64 | lm_loss: 1.129566E+01 | lr: 2.460E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 7124218.000 [ip-26-0-154-121:0]:06/21/2023 17:15:23 [INFO|DP=0|PP=0|TP=0]: iteration: 165 / 150000 | consumed_samples: 10560 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784629E+05 | tokens_per_sec_per_gpu: 9.038483E+03 | global_batch_size: 64 | lm_loss: 1.132617E+01 | lr: 2.475E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 7129894.000 [ip-26-0-154-121:0]:06/21/2023 17:15:24 [INFO|DP=0|PP=0|TP=0]: iteration: 166 / 150000 | consumed_samples: 10624 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785871E+05 | tokens_per_sec_per_gpu: 9.040423E+03 | global_batch_size: 64 | lm_loss: 1.130732E+01 | lr: 2.490E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 7108112.500 [ip-26-0-154-121:0]:06/21/2023 17:15:25 [INFO|DP=0|PP=0|TP=0]: iteration: 167 / 150000 | consumed_samples: 10688 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789501E+05 | tokens_per_sec_per_gpu: 9.046095E+03 | global_batch_size: 64 | lm_loss: 1.124102E+01 | lr: 2.505E-05 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 7068577.500 [ip-26-0-154-121:0]:06/21/2023 17:15:26 [INFO|DP=0|PP=0|TP=0]: iteration: 168 / 150000 | consumed_samples: 10752 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786926E+05 | tokens_per_sec_per_gpu: 9.042072E+03 | global_batch_size: 64 | lm_loss: 1.127120E+01 | lr: 2.520E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 7019912.500 [ip-26-0-154-121:0]:06/21/2023 17:15:27 [INFO|DP=0|PP=0|TP=0]: iteration: 169 / 150000 | consumed_samples: 10816 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788180E+05 | tokens_per_sec_per_gpu: 9.044031E+03 | global_batch_size: 64 | lm_loss: 1.128648E+01 | lr: 2.535E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 6951413.000 [ip-26-0-154-121:0]:06/21/2023 17:15:28 [INFO|DP=0|PP=0|TP=0]: iteration: 170 / 150000 | consumed_samples: 10880 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785294E+05 | tokens_per_sec_per_gpu: 9.039522E+03 | global_batch_size: 64 | lm_loss: 1.118503E+01 | lr: 2.550E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 6875160.000 [ip-26-0-154-121:0]:06/21/2023 17:15:28 [INFO|DP=0|PP=0|TP=0]: iteration: 171 / 150000 | consumed_samples: 10944 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781192E+05 | tokens_per_sec_per_gpu: 9.033113E+03 | global_batch_size: 64 | lm_loss: 1.120725E+01 | lr: 2.565E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 6791087.000 [ip-26-0-154-121:0]:06/21/2023 17:15:29 [INFO|DP=0|PP=0|TP=0]: iteration: 172 / 150000 | consumed_samples: 11008 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779498E+05 | tokens_per_sec_per_gpu: 9.030466E+03 | global_batch_size: 64 | lm_loss: 1.118017E+01 | lr: 2.580E-05 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 6705995.000 [ip-26-0-154-121:0]:06/21/2023 17:15:30 [INFO|DP=0|PP=0|TP=0]: iteration: 173 / 150000 | consumed_samples: 11072 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787915E+05 | tokens_per_sec_per_gpu: 9.043617E+03 | global_batch_size: 64 | lm_loss: 1.120997E+01 | lr: 2.595E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 6621875.500 [ip-26-0-154-121:0]:06/21/2023 17:15:31 [INFO|DP=0|PP=0|TP=0]: iteration: 174 / 150000 | consumed_samples: 11136 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790280E+05 | tokens_per_sec_per_gpu: 9.047312E+03 | global_batch_size: 64 | lm_loss: 1.122008E+01 | lr: 2.610E-05 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 6537400.000 [ip-26-0-154-121:0]:06/21/2023 17:15:32 [INFO|DP=0|PP=0|TP=0]: iteration: 175 / 150000 | consumed_samples: 11200 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789347E+05 | tokens_per_sec_per_gpu: 9.045855E+03 | global_batch_size: 64 | lm_loss: 1.125539E+01 | lr: 2.625E-05 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 6464563.000 [ip-26-0-154-121:0]:06/21/2023 17:15:33 [INFO|DP=0|PP=0|TP=0]: iteration: 176 / 150000 | consumed_samples: 11264 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788547E+05 | tokens_per_sec_per_gpu: 9.044605E+03 | global_batch_size: 64 | lm_loss: 1.123763E+01 | lr: 2.640E-05 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 6402869.000 [ip-26-0-154-121:0]:06/21/2023 17:15:34 [INFO|DP=0|PP=0|TP=0]: iteration: 177 / 150000 | consumed_samples: 11328 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783596E+05 | tokens_per_sec_per_gpu: 9.036869E+03 | global_batch_size: 64 | lm_loss: 1.125682E+01 | lr: 2.655E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 6346137.500 [ip-26-0-154-121:0]:06/21/2023 17:15:35 [INFO|DP=0|PP=0|TP=0]: iteration: 178 / 150000 | consumed_samples: 11392 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786319E+05 | tokens_per_sec_per_gpu: 9.041123E+03 | global_batch_size: 64 | lm_loss: 1.119689E+01 | lr: 2.670E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 6293749.500 [ip-26-0-154-121:0]:06/21/2023 17:15:36 [INFO|DP=0|PP=0|TP=0]: iteration: 179 / 150000 | consumed_samples: 11456 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787436E+05 | tokens_per_sec_per_gpu: 9.042869E+03 | global_batch_size: 64 | lm_loss: 1.117183E+01 | lr: 2.685E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 6248674.500 [ip-26-0-154-121:0]:06/21/2023 17:15:37 [INFO|DP=0|PP=0|TP=0]: iteration: 180 / 150000 | consumed_samples: 11520 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786509E+05 | tokens_per_sec_per_gpu: 9.041420E+03 | global_batch_size: 64 | lm_loss: 1.124650E+01 | lr: 2.700E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 6201405.500 [ip-26-0-154-121:0]:06/21/2023 17:15:37 [INFO|DP=0|PP=0|TP=0]: iteration: 181 / 150000 | consumed_samples: 11584 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787316E+05 | tokens_per_sec_per_gpu: 9.042681E+03 | global_batch_size: 64 | lm_loss: 1.116766E+01 | lr: 2.715E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 6151269.500 [ip-26-0-154-121:0]:06/21/2023 17:15:38 [INFO|DP=0|PP=0|TP=0]: iteration: 182 / 150000 | consumed_samples: 11648 | elapsed_time_per_iteration_ms: 905.2 | tokens_per_sec: 5.791718E+05 | tokens_per_sec_per_gpu: 9.049559E+03 | global_batch_size: 64 | lm_loss: 1.119528E+01 | lr: 2.730E-05 | model_tflops_per_gpu: 104.53 | hardware_tflops_per_gpu: 104.53 | grad_norm: 6101059.500 [ip-26-0-154-121:0]:06/21/2023 17:15:39 [INFO|DP=0|PP=0|TP=0]: iteration: 183 / 150000 | consumed_samples: 11712 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788026E+05 | tokens_per_sec_per_gpu: 9.043790E+03 | global_batch_size: 64 | lm_loss: 1.120036E+01 | lr: 2.745E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 6054343.500 [ip-26-0-154-121:0]:06/21/2023 17:15:40 [INFO|DP=0|PP=0|TP=0]: iteration: 184 / 150000 | consumed_samples: 11776 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786753E+05 | tokens_per_sec_per_gpu: 9.041801E+03 | global_batch_size: 64 | lm_loss: 1.124461E+01 | lr: 2.760E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 6001778.000 [ip-26-0-154-121:0]:06/21/2023 17:15:41 [INFO|DP=0|PP=0|TP=0]: iteration: 185 / 150000 | consumed_samples: 11840 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787340E+05 | tokens_per_sec_per_gpu: 9.042719E+03 | global_batch_size: 64 | lm_loss: 1.120117E+01 | lr: 2.775E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 5945831.000 [ip-26-0-154-121:0]:06/21/2023 17:15:42 [INFO|DP=0|PP=0|TP=0]: iteration: 186 / 150000 | consumed_samples: 11904 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786271E+05 | tokens_per_sec_per_gpu: 9.041049E+03 | global_batch_size: 64 | lm_loss: 1.114743E+01 | lr: 2.790E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 5883981.000 [ip-26-0-154-121:0]:06/21/2023 17:15:43 [INFO|DP=0|PP=0|TP=0]: iteration: 187 / 150000 | consumed_samples: 11968 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788174E+05 | tokens_per_sec_per_gpu: 9.044021E+03 | global_batch_size: 64 | lm_loss: 1.115882E+01 | lr: 2.805E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 5813536.500 [ip-26-0-154-121:0]:06/21/2023 17:15:44 [INFO|DP=0|PP=0|TP=0]: iteration: 188 / 150000 | consumed_samples: 12032 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784037E+05 | tokens_per_sec_per_gpu: 9.037558E+03 | global_batch_size: 64 | lm_loss: 1.117689E+01 | lr: 2.820E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 5734861.000 [ip-26-0-154-121:0]:06/21/2023 17:15:45 [INFO|DP=0|PP=0|TP=0]: iteration: 189 / 150000 | consumed_samples: 12096 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784495E+05 | tokens_per_sec_per_gpu: 9.038274E+03 | global_batch_size: 64 | lm_loss: 1.117687E+01 | lr: 2.835E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 5649296.000 [ip-26-0-154-121:0]:06/21/2023 17:15:46 [INFO|DP=0|PP=0|TP=0]: iteration: 190 / 150000 | consumed_samples: 12160 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787957E+05 | tokens_per_sec_per_gpu: 9.043683E+03 | global_batch_size: 64 | lm_loss: 1.117093E+01 | lr: 2.850E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 5554399.500 [ip-26-0-154-121:0]:06/21/2023 17:15:47 [INFO|DP=0|PP=0|TP=0]: iteration: 191 / 150000 | consumed_samples: 12224 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782863E+05 | tokens_per_sec_per_gpu: 9.035723E+03 | global_batch_size: 64 | lm_loss: 1.120192E+01 | lr: 2.865E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 5454009.000 [ip-26-0-154-121:0]:06/21/2023 17:15:47 [INFO|DP=0|PP=0|TP=0]: iteration: 192 / 150000 | consumed_samples: 12288 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783938E+05 | tokens_per_sec_per_gpu: 9.037404E+03 | global_batch_size: 64 | lm_loss: 1.114122E+01 | lr: 2.880E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 5348593.000 [ip-26-0-154-121:0]:06/21/2023 17:15:48 [INFO|DP=0|PP=0|TP=0]: iteration: 193 / 150000 | consumed_samples: 12352 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785183E+05 | tokens_per_sec_per_gpu: 9.039348E+03 | global_batch_size: 64 | lm_loss: 1.119316E+01 | lr: 2.895E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 5240137.000 [ip-26-0-154-121:0]:06/21/2023 17:15:49 [INFO|DP=0|PP=0|TP=0]: iteration: 194 / 150000 | consumed_samples: 12416 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783896E+05 | tokens_per_sec_per_gpu: 9.037337E+03 | global_batch_size: 64 | lm_loss: 1.120935E+01 | lr: 2.910E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 5126873.000 [ip-26-0-154-121:0]:06/21/2023 17:15:50 [INFO|DP=0|PP=0|TP=0]: iteration: 195 / 150000 | consumed_samples: 12480 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781907E+05 | tokens_per_sec_per_gpu: 9.034229E+03 | global_batch_size: 64 | lm_loss: 1.119566E+01 | lr: 2.925E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 5014886.500 [ip-26-0-154-121:0]:06/21/2023 17:15:51 [INFO|DP=0|PP=0|TP=0]: iteration: 196 / 150000 | consumed_samples: 12544 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781995E+05 | tokens_per_sec_per_gpu: 9.034367E+03 | global_batch_size: 64 | lm_loss: 1.117810E+01 | lr: 2.940E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 4902865.000 [ip-26-0-154-121:0]:06/21/2023 17:15:52 [INFO|DP=0|PP=0|TP=0]: iteration: 197 / 150000 | consumed_samples: 12608 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788200E+05 | tokens_per_sec_per_gpu: 9.044062E+03 | global_batch_size: 64 | lm_loss: 1.118340E+01 | lr: 2.955E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 4788966.500 [ip-26-0-154-121:0]:06/21/2023 17:15:53 [INFO|DP=0|PP=0|TP=0]: iteration: 198 / 150000 | consumed_samples: 12672 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785248E+05 | tokens_per_sec_per_gpu: 9.039451E+03 | global_batch_size: 64 | lm_loss: 1.120786E+01 | lr: 2.970E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 4675877.500 [ip-26-0-154-121:0]:06/21/2023 17:15:54 [INFO|DP=0|PP=0|TP=0]: iteration: 199 / 150000 | consumed_samples: 12736 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788852E+05 | tokens_per_sec_per_gpu: 9.045081E+03 | global_batch_size: 64 | lm_loss: 1.115701E+01 | lr: 2.985E-05 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 4565971.000 [ip-26-0-154-121:0]:06/21/2023 17:15:55 [INFO|DP=0|PP=0|TP=0]: iteration: 200 / 150000 | consumed_samples: 12800 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786319E+05 | tokens_per_sec_per_gpu: 9.041123E+03 | global_batch_size: 64 | lm_loss: 1.113647E+01 | lr: 3.000E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 4461048.000 [ip-26-0-154-121:0]:06/21/2023 17:15:56 [INFO|DP=0|PP=0|TP=0]: iteration: 201 / 150000 | consumed_samples: 12864 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786005E+05 | tokens_per_sec_per_gpu: 9.040633E+03 | global_batch_size: 64 | lm_loss: 1.125785E+01 | lr: 3.015E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4367290.500 [ip-26-0-154-121:0]:06/21/2023 17:15:57 [INFO|DP=0|PP=0|TP=0]: iteration: 202 / 150000 | consumed_samples: 12928 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783400E+05 | tokens_per_sec_per_gpu: 9.036562E+03 | global_batch_size: 64 | lm_loss: 1.106133E+01 | lr: 3.030E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 4281076.000 [ip-26-0-154-121:0]:06/21/2023 17:15:57 [INFO|DP=0|PP=0|TP=0]: iteration: 203 / 150000 | consumed_samples: 12992 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785732E+05 | tokens_per_sec_per_gpu: 9.040207E+03 | global_batch_size: 64 | lm_loss: 1.106068E+01 | lr: 3.045E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 4202504.000 [ip-26-0-154-121:0]:06/21/2023 17:15:58 [INFO|DP=0|PP=0|TP=0]: iteration: 204 / 150000 | consumed_samples: 13056 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786360E+05 | tokens_per_sec_per_gpu: 9.041187E+03 | global_batch_size: 64 | lm_loss: 1.126363E+01 | lr: 3.060E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 4133337.000 [ip-26-0-154-121:0]:06/21/2023 17:15:59 [INFO|DP=0|PP=0|TP=0]: iteration: 205 / 150000 | consumed_samples: 13120 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784098E+05 | tokens_per_sec_per_gpu: 9.037653E+03 | global_batch_size: 64 | lm_loss: 1.122264E+01 | lr: 3.075E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 4073496.000 [ip-26-0-154-121:0]:06/21/2023 17:16:00 [INFO|DP=0|PP=0|TP=0]: iteration: 206 / 150000 | consumed_samples: 13184 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783663E+05 | tokens_per_sec_per_gpu: 9.036973E+03 | global_batch_size: 64 | lm_loss: 1.127466E+01 | lr: 3.090E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 4022810.500 [ip-26-0-154-121:0]:06/21/2023 17:16:01 [INFO|DP=0|PP=0|TP=0]: iteration: 207 / 150000 | consumed_samples: 13248 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782916E+05 | tokens_per_sec_per_gpu: 9.035806E+03 | global_batch_size: 64 | lm_loss: 1.114106E+01 | lr: 3.105E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3980407.250 [ip-26-0-154-121:0]:06/21/2023 17:16:02 [INFO|DP=0|PP=0|TP=0]: iteration: 208 / 150000 | consumed_samples: 13312 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783779E+05 | tokens_per_sec_per_gpu: 9.037154E+03 | global_batch_size: 64 | lm_loss: 1.125817E+01 | lr: 3.120E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3945030.500 [ip-26-0-154-121:0]:06/21/2023 17:16:03 [INFO|DP=0|PP=0|TP=0]: iteration: 209 / 150000 | consumed_samples: 13376 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786955E+05 | tokens_per_sec_per_gpu: 9.042117E+03 | global_batch_size: 64 | lm_loss: 1.122058E+01 | lr: 3.135E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3915708.500 [ip-26-0-154-121:0]:06/21/2023 17:16:04 [INFO|DP=0|PP=0|TP=0]: iteration: 210 / 150000 | consumed_samples: 13440 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781276E+05 | tokens_per_sec_per_gpu: 9.033243E+03 | global_batch_size: 64 | lm_loss: 1.112046E+01 | lr: 3.150E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3891744.500 [ip-26-0-154-121:0]:06/21/2023 17:16:05 [INFO|DP=0|PP=0|TP=0]: iteration: 211 / 150000 | consumed_samples: 13504 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781981E+05 | tokens_per_sec_per_gpu: 9.034345E+03 | global_batch_size: 64 | lm_loss: 1.120331E+01 | lr: 3.165E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3872038.750 [ip-26-0-154-121:0]:06/21/2023 17:16:06 [INFO|DP=0|PP=0|TP=0]: iteration: 212 / 150000 | consumed_samples: 13568 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788094E+05 | tokens_per_sec_per_gpu: 9.043898E+03 | global_batch_size: 64 | lm_loss: 1.125089E+01 | lr: 3.180E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 3854986.000 [ip-26-0-154-121:0]:06/21/2023 17:16:07 [INFO|DP=0|PP=0|TP=0]: iteration: 213 / 150000 | consumed_samples: 13632 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782884E+05 | tokens_per_sec_per_gpu: 9.035757E+03 | global_batch_size: 64 | lm_loss: 1.119707E+01 | lr: 3.195E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3839875.250 [ip-26-0-154-121:0]:06/21/2023 17:16:07 [INFO|DP=0|PP=0|TP=0]: iteration: 214 / 150000 | consumed_samples: 13696 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785124E+05 | tokens_per_sec_per_gpu: 9.039256E+03 | global_batch_size: 64 | lm_loss: 1.122725E+01 | lr: 3.210E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3826342.750 [ip-26-0-154-121:0]:06/21/2023 17:16:08 [INFO|DP=0|PP=0|TP=0]: iteration: 215 / 150000 | consumed_samples: 13760 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786512E+05 | tokens_per_sec_per_gpu: 9.041425E+03 | global_batch_size: 64 | lm_loss: 1.115901E+01 | lr: 3.225E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3814278.500 [ip-26-0-154-121:0]:06/21/2023 17:16:09 [INFO|DP=0|PP=0|TP=0]: iteration: 216 / 150000 | consumed_samples: 13824 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781484E+05 | tokens_per_sec_per_gpu: 9.033569E+03 | global_batch_size: 64 | lm_loss: 1.121590E+01 | lr: 3.240E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3803332.000 [ip-26-0-154-121:0]:06/21/2023 17:16:10 [INFO|DP=0|PP=0|TP=0]: iteration: 217 / 150000 | consumed_samples: 13888 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786107E+05 | tokens_per_sec_per_gpu: 9.040792E+03 | global_batch_size: 64 | lm_loss: 1.122497E+01 | lr: 3.255E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3793089.750 [ip-26-0-154-121:0]:06/21/2023 17:16:11 [INFO|DP=0|PP=0|TP=0]: iteration: 218 / 150000 | consumed_samples: 13952 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785579E+05 | tokens_per_sec_per_gpu: 9.039967E+03 | global_batch_size: 64 | lm_loss: 1.117932E+01 | lr: 3.270E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3783676.250 [ip-26-0-154-121:0]:06/21/2023 17:16:12 [INFO|DP=0|PP=0|TP=0]: iteration: 219 / 150000 | consumed_samples: 14016 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783838E+05 | tokens_per_sec_per_gpu: 9.037247E+03 | global_batch_size: 64 | lm_loss: 1.128364E+01 | lr: 3.285E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3775014.500 [ip-26-0-154-121:0]:06/21/2023 17:16:13 [INFO|DP=0|PP=0|TP=0]: iteration: 220 / 150000 | consumed_samples: 14080 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785229E+05 | tokens_per_sec_per_gpu: 9.039420E+03 | global_batch_size: 64 | lm_loss: 1.119592E+01 | lr: 3.300E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3767153.000 [ip-26-0-154-121:0]:06/21/2023 17:16:14 [INFO|DP=0|PP=0|TP=0]: iteration: 221 / 150000 | consumed_samples: 14144 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781941E+05 | tokens_per_sec_per_gpu: 9.034284E+03 | global_batch_size: 64 | lm_loss: 1.127788E+01 | lr: 3.315E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3759547.750 [ip-26-0-154-121:0]:06/21/2023 17:16:15 [INFO|DP=0|PP=0|TP=0]: iteration: 222 / 150000 | consumed_samples: 14208 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785095E+05 | tokens_per_sec_per_gpu: 9.039210E+03 | global_batch_size: 64 | lm_loss: 1.118293E+01 | lr: 3.330E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3751860.000 [ip-26-0-154-121:0]:06/21/2023 17:16:16 [INFO|DP=0|PP=0|TP=0]: iteration: 223 / 150000 | consumed_samples: 14272 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788343E+05 | tokens_per_sec_per_gpu: 9.044286E+03 | global_batch_size: 64 | lm_loss: 1.121514E+01 | lr: 3.345E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 3744213.750 [ip-26-0-154-121:0]:06/21/2023 17:16:17 [INFO|DP=0|PP=0|TP=0]: iteration: 224 / 150000 | consumed_samples: 14336 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786696E+05 | tokens_per_sec_per_gpu: 9.041713E+03 | global_batch_size: 64 | lm_loss: 1.118155E+01 | lr: 3.360E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3736768.750 [ip-26-0-154-121:0]:06/21/2023 17:16:17 [INFO|DP=0|PP=0|TP=0]: iteration: 225 / 150000 | consumed_samples: 14400 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790756E+05 | tokens_per_sec_per_gpu: 9.048056E+03 | global_batch_size: 64 | lm_loss: 1.114981E+01 | lr: 3.375E-05 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 3729451.750 [ip-26-0-154-121:0]:06/21/2023 17:16:18 [INFO|DP=0|PP=0|TP=0]: iteration: 226 / 150000 | consumed_samples: 14464 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787584E+05 | tokens_per_sec_per_gpu: 9.043100E+03 | global_batch_size: 64 | lm_loss: 1.113122E+01 | lr: 3.390E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3722636.250 [ip-26-0-154-121:0]:06/21/2023 17:16:19 [INFO|DP=0|PP=0|TP=0]: iteration: 227 / 150000 | consumed_samples: 14528 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789781E+05 | tokens_per_sec_per_gpu: 9.046533E+03 | global_batch_size: 64 | lm_loss: 1.126753E+01 | lr: 3.405E-05 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 3717140.750 [ip-26-0-154-121:0]:06/21/2023 17:16:20 [INFO|DP=0|PP=0|TP=0]: iteration: 228 / 150000 | consumed_samples: 14592 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786961E+05 | tokens_per_sec_per_gpu: 9.042127E+03 | global_batch_size: 64 | lm_loss: 1.118175E+01 | lr: 3.420E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3712564.750 [ip-26-0-154-121:0]:06/21/2023 17:16:21 [INFO|DP=0|PP=0|TP=0]: iteration: 229 / 150000 | consumed_samples: 14656 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785854E+05 | tokens_per_sec_per_gpu: 9.040397E+03 | global_batch_size: 64 | lm_loss: 1.112082E+01 | lr: 3.435E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3707852.750 [ip-26-0-154-121:0]:06/21/2023 17:16:22 [INFO|DP=0|PP=0|TP=0]: iteration: 230 / 150000 | consumed_samples: 14720 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790995E+05 | tokens_per_sec_per_gpu: 9.048430E+03 | global_batch_size: 64 | lm_loss: 1.118228E+01 | lr: 3.450E-05 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 3702418.750 [ip-26-0-154-121:0]:06/21/2023 17:16:23 [INFO|DP=0|PP=0|TP=0]: iteration: 231 / 150000 | consumed_samples: 14784 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786931E+05 | tokens_per_sec_per_gpu: 9.042079E+03 | global_batch_size: 64 | lm_loss: 1.120804E+01 | lr: 3.465E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3696869.750 [ip-26-0-154-121:0]:06/21/2023 17:16:24 [INFO|DP=0|PP=0|TP=0]: iteration: 232 / 150000 | consumed_samples: 14848 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786218E+05 | tokens_per_sec_per_gpu: 9.040966E+03 | global_batch_size: 64 | lm_loss: 1.126171E+01 | lr: 3.480E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3691680.000 [ip-26-0-154-121:0]:06/21/2023 17:16:25 [INFO|DP=0|PP=0|TP=0]: iteration: 233 / 150000 | consumed_samples: 14912 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787366E+05 | tokens_per_sec_per_gpu: 9.042760E+03 | global_batch_size: 64 | lm_loss: 1.126005E+01 | lr: 3.495E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3686654.500 [ip-26-0-154-121:0]:06/21/2023 17:16:26 [INFO|DP=0|PP=0|TP=0]: iteration: 234 / 150000 | consumed_samples: 14976 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784842E+05 | tokens_per_sec_per_gpu: 9.038816E+03 | global_batch_size: 64 | lm_loss: 1.130757E+01 | lr: 3.510E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3682621.750 [ip-26-0-154-121:0]:06/21/2023 17:16:26 [INFO|DP=0|PP=0|TP=0]: iteration: 235 / 150000 | consumed_samples: 15040 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788861E+05 | tokens_per_sec_per_gpu: 9.045095E+03 | global_batch_size: 64 | lm_loss: 1.121975E+01 | lr: 3.525E-05 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 3680698.250 [ip-26-0-154-121:0]:06/21/2023 17:16:27 [INFO|DP=0|PP=0|TP=0]: iteration: 236 / 150000 | consumed_samples: 15104 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784150E+05 | tokens_per_sec_per_gpu: 9.037734E+03 | global_batch_size: 64 | lm_loss: 1.124732E+01 | lr: 3.540E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3680072.750 [ip-26-0-154-121:0]:06/21/2023 17:16:28 [INFO|DP=0|PP=0|TP=0]: iteration: 237 / 150000 | consumed_samples: 15168 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786986E+05 | tokens_per_sec_per_gpu: 9.042165E+03 | global_batch_size: 64 | lm_loss: 1.123268E+01 | lr: 3.555E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3681317.500 [ip-26-0-154-121:0]:06/21/2023 17:16:29 [INFO|DP=0|PP=0|TP=0]: iteration: 238 / 150000 | consumed_samples: 15232 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787849E+05 | tokens_per_sec_per_gpu: 9.043514E+03 | global_batch_size: 64 | lm_loss: 1.119794E+01 | lr: 3.570E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3686851.750 [ip-26-0-154-121:0]:06/21/2023 17:16:30 [INFO|DP=0|PP=0|TP=0]: iteration: 239 / 150000 | consumed_samples: 15296 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787593E+05 | tokens_per_sec_per_gpu: 9.043114E+03 | global_batch_size: 64 | lm_loss: 1.118047E+01 | lr: 3.585E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3698713.000 [ip-26-0-154-121:0]:06/21/2023 17:16:31 [INFO|DP=0|PP=0|TP=0]: iteration: 240 / 150000 | consumed_samples: 15360 | elapsed_time_per_iteration_ms: 905.2 | tokens_per_sec: 5.791825E+05 | tokens_per_sec_per_gpu: 9.049726E+03 | global_batch_size: 64 | lm_loss: 1.123423E+01 | lr: 3.600E-05 | model_tflops_per_gpu: 104.53 | hardware_tflops_per_gpu: 104.53 | grad_norm: 3716179.750 [ip-26-0-154-121:0]:06/21/2023 17:16:32 [INFO|DP=0|PP=0|TP=0]: iteration: 241 / 150000 | consumed_samples: 15424 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789934E+05 | tokens_per_sec_per_gpu: 9.046772E+03 | global_batch_size: 64 | lm_loss: 1.127500E+01 | lr: 3.615E-05 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 3736281.750 [ip-26-0-154-121:0]:06/21/2023 17:16:33 [INFO|DP=0|PP=0|TP=0]: iteration: 242 / 150000 | consumed_samples: 15488 | elapsed_time_per_iteration_ms: 909.1 | tokens_per_sec: 5.767399E+05 | tokens_per_sec_per_gpu: 9.011561E+03 | global_batch_size: 64 | lm_loss: 1.116160E+01 | lr: 3.630E-05 | model_tflops_per_gpu: 104.09 | hardware_tflops_per_gpu: 104.09 | grad_norm: 3758838.250 [ip-26-0-154-121:0]:06/21/2023 17:16:34 [INFO|DP=0|PP=0|TP=0]: iteration: 243 / 150000 | consumed_samples: 15552 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788148E+05 | tokens_per_sec_per_gpu: 9.043981E+03 | global_batch_size: 64 | lm_loss: 1.120788E+01 | lr: 3.645E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 3785991.750 [ip-26-0-154-121:0]:06/21/2023 17:16:35 [INFO|DP=0|PP=0|TP=0]: iteration: 244 / 150000 | consumed_samples: 15616 | elapsed_time_per_iteration_ms: 905.1 | tokens_per_sec: 5.792794E+05 | tokens_per_sec_per_gpu: 9.051240E+03 | global_batch_size: 64 | lm_loss: 1.117925E+01 | lr: 3.660E-05 | model_tflops_per_gpu: 104.55 | hardware_tflops_per_gpu: 104.55 | grad_norm: 3820681.250 [ip-26-0-154-121:0]:06/21/2023 17:16:36 [INFO|DP=0|PP=0|TP=0]: iteration: 245 / 150000 | consumed_samples: 15680 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788544E+05 | tokens_per_sec_per_gpu: 9.044600E+03 | global_batch_size: 64 | lm_loss: 1.116735E+01 | lr: 3.675E-05 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 3864698.250 [ip-26-0-154-121:0]:06/21/2023 17:16:36 [INFO|DP=0|PP=0|TP=0]: iteration: 246 / 150000 | consumed_samples: 15744 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788684E+05 | tokens_per_sec_per_gpu: 9.044819E+03 | global_batch_size: 64 | lm_loss: 1.103830E+01 | lr: 3.690E-05 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 3915671.000 [ip-26-0-154-121:0]:06/21/2023 17:16:37 [INFO|DP=0|PP=0|TP=0]: iteration: 247 / 150000 | consumed_samples: 15808 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788521E+05 | tokens_per_sec_per_gpu: 9.044564E+03 | global_batch_size: 64 | lm_loss: 1.119781E+01 | lr: 3.705E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 3966491.000 [ip-26-0-154-121:0]:06/21/2023 17:16:38 [INFO|DP=0|PP=0|TP=0]: iteration: 248 / 150000 | consumed_samples: 15872 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784621E+05 | tokens_per_sec_per_gpu: 9.038471E+03 | global_batch_size: 64 | lm_loss: 1.121106E+01 | lr: 3.720E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 4008586.750 [ip-26-0-154-121:0]:06/21/2023 17:16:39 [INFO|DP=0|PP=0|TP=0]: iteration: 249 / 150000 | consumed_samples: 15936 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782769E+05 | tokens_per_sec_per_gpu: 9.035576E+03 | global_batch_size: 64 | lm_loss: 1.114229E+01 | lr: 3.735E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 4036721.500 [ip-26-0-154-121:0]:06/21/2023 17:16:40 [INFO|DP=0|PP=0|TP=0]: iteration: 250 / 150000 | consumed_samples: 16000 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785810E+05 | tokens_per_sec_per_gpu: 9.040328E+03 | global_batch_size: 64 | lm_loss: 1.116018E+01 | lr: 3.750E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4050960.250 [ip-26-0-154-121:0]:06/21/2023 17:16:41 [INFO|DP=0|PP=0|TP=0]: iteration: 251 / 150000 | consumed_samples: 16064 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783211E+05 | tokens_per_sec_per_gpu: 9.036267E+03 | global_batch_size: 64 | lm_loss: 1.134858E+01 | lr: 3.765E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 4055108.000 [ip-26-0-154-121:0]:06/21/2023 17:16:42 [INFO|DP=0|PP=0|TP=0]: iteration: 252 / 150000 | consumed_samples: 16128 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784176E+05 | tokens_per_sec_per_gpu: 9.037774E+03 | global_batch_size: 64 | lm_loss: 1.118578E+01 | lr: 3.780E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 4053685.000 [ip-26-0-154-121:0]:06/21/2023 17:16:43 [INFO|DP=0|PP=0|TP=0]: iteration: 253 / 150000 | consumed_samples: 16192 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787037E+05 | tokens_per_sec_per_gpu: 9.042246E+03 | global_batch_size: 64 | lm_loss: 1.128851E+01 | lr: 3.795E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 4050340.000 [ip-26-0-154-121:0]:06/21/2023 17:16:44 [INFO|DP=0|PP=0|TP=0]: iteration: 254 / 150000 | consumed_samples: 16256 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787848E+05 | tokens_per_sec_per_gpu: 9.043512E+03 | global_batch_size: 64 | lm_loss: 1.122431E+01 | lr: 3.810E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 4048695.000 [ip-26-0-154-121:0]:06/21/2023 17:16:45 [INFO|DP=0|PP=0|TP=0]: iteration: 255 / 150000 | consumed_samples: 16320 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786626E+05 | tokens_per_sec_per_gpu: 9.041603E+03 | global_batch_size: 64 | lm_loss: 1.102060E+01 | lr: 3.825E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 4048039.250 [ip-26-0-154-121:0]:06/21/2023 17:16:46 [INFO|DP=0|PP=0|TP=0]: iteration: 256 / 150000 | consumed_samples: 16384 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786795E+05 | tokens_per_sec_per_gpu: 9.041867E+03 | global_batch_size: 64 | lm_loss: 1.115492E+01 | lr: 3.840E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 4048824.500 [ip-26-0-154-121:0]:06/21/2023 17:16:46 [INFO|DP=0|PP=0|TP=0]: iteration: 257 / 150000 | consumed_samples: 16448 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786539E+05 | tokens_per_sec_per_gpu: 9.041468E+03 | global_batch_size: 64 | lm_loss: 1.120474E+01 | lr: 3.855E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 4048740.000 [ip-26-0-154-121:0]:06/21/2023 17:16:47 [INFO|DP=0|PP=0|TP=0]: iteration: 258 / 150000 | consumed_samples: 16512 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787426E+05 | tokens_per_sec_per_gpu: 9.042853E+03 | global_batch_size: 64 | lm_loss: 1.122235E+01 | lr: 3.870E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 4047047.000 [ip-26-0-154-121:0]:06/21/2023 17:16:48 [INFO|DP=0|PP=0|TP=0]: iteration: 259 / 150000 | consumed_samples: 16576 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786160E+05 | tokens_per_sec_per_gpu: 9.040875E+03 | global_batch_size: 64 | lm_loss: 1.122745E+01 | lr: 3.885E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4043721.500 [ip-26-0-154-121:0]:06/21/2023 17:16:49 [INFO|DP=0|PP=0|TP=0]: iteration: 260 / 150000 | consumed_samples: 16640 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785869E+05 | tokens_per_sec_per_gpu: 9.040421E+03 | global_batch_size: 64 | lm_loss: 1.113070E+01 | lr: 3.900E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4039539.500 [ip-26-0-154-121:0]:06/21/2023 17:16:50 [INFO|DP=0|PP=0|TP=0]: iteration: 261 / 150000 | consumed_samples: 16704 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788454E+05 | tokens_per_sec_per_gpu: 9.044459E+03 | global_batch_size: 64 | lm_loss: 1.112677E+01 | lr: 3.915E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 4034025.250 [ip-26-0-154-121:0]:06/21/2023 17:16:51 [INFO|DP=0|PP=0|TP=0]: iteration: 262 / 150000 | consumed_samples: 16768 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787188E+05 | tokens_per_sec_per_gpu: 9.042481E+03 | global_batch_size: 64 | lm_loss: 1.114348E+01 | lr: 3.930E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 4027271.500 [ip-26-0-154-121:0]:06/21/2023 17:16:52 [INFO|DP=0|PP=0|TP=0]: iteration: 263 / 150000 | consumed_samples: 16832 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790053E+05 | tokens_per_sec_per_gpu: 9.046957E+03 | global_batch_size: 64 | lm_loss: 1.125009E+01 | lr: 3.945E-05 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 4021052.750 [ip-26-0-154-121:0]:06/21/2023 17:16:53 [INFO|DP=0|PP=0|TP=0]: iteration: 264 / 150000 | consumed_samples: 16896 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787046E+05 | tokens_per_sec_per_gpu: 9.042260E+03 | global_batch_size: 64 | lm_loss: 1.118801E+01 | lr: 3.960E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 4015808.250 [ip-26-0-154-121:0]:06/21/2023 17:16:54 [INFO|DP=0|PP=0|TP=0]: iteration: 265 / 150000 | consumed_samples: 16960 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784612E+05 | tokens_per_sec_per_gpu: 9.038457E+03 | global_batch_size: 64 | lm_loss: 1.119627E+01 | lr: 3.975E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 4012098.750 [ip-26-0-154-121:0]:06/21/2023 17:16:55 [INFO|DP=0|PP=0|TP=0]: iteration: 266 / 150000 | consumed_samples: 17024 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789903E+05 | tokens_per_sec_per_gpu: 9.046724E+03 | global_batch_size: 64 | lm_loss: 1.106635E+01 | lr: 3.990E-05 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 4009717.000 [ip-26-0-154-121:0]:06/21/2023 17:16:56 [INFO|DP=0|PP=0|TP=0]: iteration: 267 / 150000 | consumed_samples: 17088 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784096E+05 | tokens_per_sec_per_gpu: 9.037651E+03 | global_batch_size: 64 | lm_loss: 1.114413E+01 | lr: 4.005E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 4007291.500 [ip-26-0-154-121:0]:06/21/2023 17:16:56 [INFO|DP=0|PP=0|TP=0]: iteration: 268 / 150000 | consumed_samples: 17152 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786836E+05 | tokens_per_sec_per_gpu: 9.041932E+03 | global_batch_size: 64 | lm_loss: 1.122965E+01 | lr: 4.020E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 4004268.750 [ip-26-0-154-121:0]:06/21/2023 17:16:57 [INFO|DP=0|PP=0|TP=0]: iteration: 269 / 150000 | consumed_samples: 17216 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786148E+05 | tokens_per_sec_per_gpu: 9.040856E+03 | global_batch_size: 64 | lm_loss: 1.116263E+01 | lr: 4.035E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4002491.250 [ip-26-0-154-121:0]:06/21/2023 17:16:58 [INFO|DP=0|PP=0|TP=0]: iteration: 270 / 150000 | consumed_samples: 17280 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788005E+05 | tokens_per_sec_per_gpu: 9.043757E+03 | global_batch_size: 64 | lm_loss: 1.117043E+01 | lr: 4.050E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 4006302.750 [ip-26-0-154-121:0]:06/21/2023 17:16:59 [INFO|DP=0|PP=0|TP=0]: iteration: 271 / 150000 | consumed_samples: 17344 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788151E+05 | tokens_per_sec_per_gpu: 9.043986E+03 | global_batch_size: 64 | lm_loss: 1.125112E+01 | lr: 4.065E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 4024651.000 [ip-26-0-154-121:0]:06/21/2023 17:17:00 [INFO|DP=0|PP=0|TP=0]: iteration: 272 / 150000 | consumed_samples: 17408 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788835E+05 | tokens_per_sec_per_gpu: 9.045055E+03 | global_batch_size: 64 | lm_loss: 1.118246E+01 | lr: 4.080E-05 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 4069600.500 [ip-26-0-154-121:0]:06/21/2023 17:17:01 [INFO|DP=0|PP=0|TP=0]: iteration: 273 / 150000 | consumed_samples: 17472 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789737E+05 | tokens_per_sec_per_gpu: 9.046464E+03 | global_batch_size: 64 | lm_loss: 1.114564E+01 | lr: 4.095E-05 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 4154957.750 [ip-26-0-154-121:0]:06/21/2023 17:17:02 [INFO|DP=0|PP=0|TP=0]: iteration: 274 / 150000 | consumed_samples: 17536 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783716E+05 | tokens_per_sec_per_gpu: 9.037057E+03 | global_batch_size: 64 | lm_loss: 1.115795E+01 | lr: 4.110E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 4287121.500 [ip-26-0-154-121:0]:06/21/2023 17:17:03 [INFO|DP=0|PP=0|TP=0]: iteration: 275 / 150000 | consumed_samples: 17600 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781849E+05 | tokens_per_sec_per_gpu: 9.034139E+03 | global_batch_size: 64 | lm_loss: 1.114984E+01 | lr: 4.125E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 4445660.000 [ip-26-0-154-121:0]:06/21/2023 17:17:04 [INFO|DP=0|PP=0|TP=0]: iteration: 276 / 150000 | consumed_samples: 17664 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783657E+05 | tokens_per_sec_per_gpu: 9.036964E+03 | global_batch_size: 64 | lm_loss: 1.115405E+01 | lr: 4.140E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 4589345.500 [ip-26-0-154-121:0]:06/21/2023 17:17:05 [INFO|DP=0|PP=0|TP=0]: iteration: 277 / 150000 | consumed_samples: 17728 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787244E+05 | tokens_per_sec_per_gpu: 9.042569E+03 | global_batch_size: 64 | lm_loss: 1.118797E+01 | lr: 4.155E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 4692133.000 [ip-26-0-154-121:0]:06/21/2023 17:17:06 [INFO|DP=0|PP=0|TP=0]: iteration: 278 / 150000 | consumed_samples: 17792 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783797E+05 | tokens_per_sec_per_gpu: 9.037182E+03 | global_batch_size: 64 | lm_loss: 1.118327E+01 | lr: 4.170E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 4753627.500 [ip-26-0-154-121:0]:06/21/2023 17:17:06 [INFO|DP=0|PP=0|TP=0]: iteration: 279 / 150000 | consumed_samples: 17856 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782405E+05 | tokens_per_sec_per_gpu: 9.035008E+03 | global_batch_size: 64 | lm_loss: 1.121466E+01 | lr: 4.185E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 4781529.000 [ip-26-0-154-121:0]:06/21/2023 17:17:07 [INFO|DP=0|PP=0|TP=0]: iteration: 280 / 150000 | consumed_samples: 17920 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783847E+05 | tokens_per_sec_per_gpu: 9.037261E+03 | global_batch_size: 64 | lm_loss: 1.115666E+01 | lr: 4.200E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 4786227.500 [ip-26-0-154-121:0]:06/21/2023 17:17:08 [INFO|DP=0|PP=0|TP=0]: iteration: 281 / 150000 | consumed_samples: 17984 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790756E+05 | tokens_per_sec_per_gpu: 9.048056E+03 | global_batch_size: 64 | lm_loss: 1.120294E+01 | lr: 4.215E-05 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 4777381.500 [ip-26-0-154-121:0]:06/21/2023 17:17:09 [INFO|DP=0|PP=0|TP=0]: iteration: 282 / 150000 | consumed_samples: 18048 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790948E+05 | tokens_per_sec_per_gpu: 9.048356E+03 | global_batch_size: 64 | lm_loss: 1.117583E+01 | lr: 4.230E-05 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 4763460.000 [ip-26-0-154-121:0]:06/21/2023 17:17:10 [INFO|DP=0|PP=0|TP=0]: iteration: 283 / 150000 | consumed_samples: 18112 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786696E+05 | tokens_per_sec_per_gpu: 9.041713E+03 | global_batch_size: 64 | lm_loss: 1.120888E+01 | lr: 4.245E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 4746832.500 [ip-26-0-154-121:0]:06/21/2023 17:17:11 [INFO|DP=0|PP=0|TP=0]: iteration: 284 / 150000 | consumed_samples: 18176 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782898E+05 | tokens_per_sec_per_gpu: 9.035778E+03 | global_batch_size: 64 | lm_loss: 1.120938E+01 | lr: 4.260E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 4727856.000 [ip-26-0-154-121:0]:06/21/2023 17:17:12 [INFO|DP=0|PP=0|TP=0]: iteration: 285 / 150000 | consumed_samples: 18240 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787960E+05 | tokens_per_sec_per_gpu: 9.043688E+03 | global_batch_size: 64 | lm_loss: 1.113580E+01 | lr: 4.275E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 4706270.500 [ip-26-0-154-121:0]:06/21/2023 17:17:13 [INFO|DP=0|PP=0|TP=0]: iteration: 286 / 150000 | consumed_samples: 18304 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786687E+05 | tokens_per_sec_per_gpu: 9.041699E+03 | global_batch_size: 64 | lm_loss: 1.119276E+01 | lr: 4.290E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 4683054.500 [ip-26-0-154-121:0]:06/21/2023 17:17:14 [INFO|DP=0|PP=0|TP=0]: iteration: 287 / 150000 | consumed_samples: 18368 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786110E+05 | tokens_per_sec_per_gpu: 9.040797E+03 | global_batch_size: 64 | lm_loss: 1.116742E+01 | lr: 4.305E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4659018.000 [ip-26-0-154-121:0]:06/21/2023 17:17:15 [INFO|DP=0|PP=0|TP=0]: iteration: 288 / 150000 | consumed_samples: 18432 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782668E+05 | tokens_per_sec_per_gpu: 9.035419E+03 | global_batch_size: 64 | lm_loss: 1.122571E+01 | lr: 4.320E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 4633393.000 [ip-26-0-154-121:0]:06/21/2023 17:17:15 [INFO|DP=0|PP=0|TP=0]: iteration: 289 / 150000 | consumed_samples: 18496 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785358E+05 | tokens_per_sec_per_gpu: 9.039622E+03 | global_batch_size: 64 | lm_loss: 1.117860E+01 | lr: 4.335E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 4606828.000 [ip-26-0-154-121:0]:06/21/2023 17:17:16 [INFO|DP=0|PP=0|TP=0]: iteration: 290 / 150000 | consumed_samples: 18560 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790205E+05 | tokens_per_sec_per_gpu: 9.047196E+03 | global_batch_size: 64 | lm_loss: 1.118991E+01 | lr: 4.350E-05 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 4578359.500 [ip-26-0-154-121:0]:06/21/2023 17:17:17 [INFO|DP=0|PP=0|TP=0]: iteration: 291 / 150000 | consumed_samples: 18624 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789551E+05 | tokens_per_sec_per_gpu: 9.046174E+03 | global_batch_size: 64 | lm_loss: 1.121650E+01 | lr: 4.365E-05 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 4547030.000 [ip-26-0-154-121:0]:06/21/2023 17:17:18 [INFO|DP=0|PP=0|TP=0]: iteration: 292 / 150000 | consumed_samples: 18688 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786249E+05 | tokens_per_sec_per_gpu: 9.041013E+03 | global_batch_size: 64 | lm_loss: 1.127637E+01 | lr: 4.380E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4513176.000 [ip-26-0-154-121:0]:06/21/2023 17:17:19 [INFO|DP=0|PP=0|TP=0]: iteration: 293 / 150000 | consumed_samples: 18752 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786108E+05 | tokens_per_sec_per_gpu: 9.040794E+03 | global_batch_size: 64 | lm_loss: 1.118171E+01 | lr: 4.395E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4477760.500 [ip-26-0-154-121:0]:06/21/2023 17:17:20 [INFO|DP=0|PP=0|TP=0]: iteration: 294 / 150000 | consumed_samples: 18816 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779070E+05 | tokens_per_sec_per_gpu: 9.029796E+03 | global_batch_size: 64 | lm_loss: 1.112465E+01 | lr: 4.410E-05 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 4442429.000 [ip-26-0-154-121:0]:06/21/2023 17:17:21 [INFO|DP=0|PP=0|TP=0]: iteration: 295 / 150000 | consumed_samples: 18880 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783458E+05 | tokens_per_sec_per_gpu: 9.036652E+03 | global_batch_size: 64 | lm_loss: 1.117818E+01 | lr: 4.425E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 4408000.000 [ip-26-0-154-121:0]:06/21/2023 17:17:22 [INFO|DP=0|PP=0|TP=0]: iteration: 296 / 150000 | consumed_samples: 18944 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787832E+05 | tokens_per_sec_per_gpu: 9.043488E+03 | global_batch_size: 64 | lm_loss: 1.115689E+01 | lr: 4.440E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 4375229.000 [ip-26-0-154-121:0]:06/21/2023 17:17:23 [INFO|DP=0|PP=0|TP=0]: iteration: 297 / 150000 | consumed_samples: 19008 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784133E+05 | tokens_per_sec_per_gpu: 9.037708E+03 | global_batch_size: 64 | lm_loss: 1.115752E+01 | lr: 4.455E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 4344339.500 [ip-26-0-154-121:0]:06/21/2023 17:17:24 [INFO|DP=0|PP=0|TP=0]: iteration: 298 / 150000 | consumed_samples: 19072 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786689E+05 | tokens_per_sec_per_gpu: 9.041701E+03 | global_batch_size: 64 | lm_loss: 1.121605E+01 | lr: 4.470E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 4315529.500 [ip-26-0-154-121:0]:06/21/2023 17:17:25 [INFO|DP=0|PP=0|TP=0]: iteration: 299 / 150000 | consumed_samples: 19136 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785165E+05 | tokens_per_sec_per_gpu: 9.039320E+03 | global_batch_size: 64 | lm_loss: 1.107724E+01 | lr: 4.485E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 4288243.000 [ip-26-0-154-121:0]:06/21/2023 17:17:25 [INFO|DP=0|PP=0|TP=0]: iteration: 300 / 150000 | consumed_samples: 19200 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785752E+05 | tokens_per_sec_per_gpu: 9.040238E+03 | global_batch_size: 64 | lm_loss: 1.123601E+01 | lr: 4.500E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 4262599.500 [ip-26-0-154-121:0]:06/21/2023 17:17:26 [INFO|DP=0|PP=0|TP=0]: iteration: 301 / 150000 | consumed_samples: 19264 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787922E+05 | tokens_per_sec_per_gpu: 9.043629E+03 | global_batch_size: 64 | lm_loss: 1.117019E+01 | lr: 4.515E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 4237702.500 [ip-26-0-154-121:0]:06/21/2023 17:17:27 [INFO|DP=0|PP=0|TP=0]: iteration: 302 / 150000 | consumed_samples: 19328 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788410E+05 | tokens_per_sec_per_gpu: 9.044390E+03 | global_batch_size: 64 | lm_loss: 1.109536E+01 | lr: 4.530E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 4212672.000 [ip-26-0-154-121:0]:06/21/2023 17:17:28 [INFO|DP=0|PP=0|TP=0]: iteration: 303 / 150000 | consumed_samples: 19392 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786719E+05 | tokens_per_sec_per_gpu: 9.041749E+03 | global_batch_size: 64 | lm_loss: 1.115776E+01 | lr: 4.545E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 4186076.750 [ip-26-0-154-121:0]:06/21/2023 17:17:29 [INFO|DP=0|PP=0|TP=0]: iteration: 304 / 150000 | consumed_samples: 19456 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785399E+05 | tokens_per_sec_per_gpu: 9.039686E+03 | global_batch_size: 64 | lm_loss: 1.115790E+01 | lr: 4.560E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 4156658.750 [ip-26-0-154-121:0]:06/21/2023 17:17:30 [INFO|DP=0|PP=0|TP=0]: iteration: 305 / 150000 | consumed_samples: 19520 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784367E+05 | tokens_per_sec_per_gpu: 9.038074E+03 | global_batch_size: 64 | lm_loss: 1.126458E+01 | lr: 4.575E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 4123136.250 [ip-26-0-154-121:0]:06/21/2023 17:17:31 [INFO|DP=0|PP=0|TP=0]: iteration: 306 / 150000 | consumed_samples: 19584 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787266E+05 | tokens_per_sec_per_gpu: 9.042603E+03 | global_batch_size: 64 | lm_loss: 1.127309E+01 | lr: 4.590E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 4086773.000 [ip-26-0-154-121:0]:06/21/2023 17:17:32 [INFO|DP=0|PP=0|TP=0]: iteration: 307 / 150000 | consumed_samples: 19648 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787627E+05 | tokens_per_sec_per_gpu: 9.043167E+03 | global_batch_size: 64 | lm_loss: 1.093032E+01 | lr: 4.605E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 4048406.000 [ip-26-0-154-121:0]:06/21/2023 17:17:33 [INFO|DP=0|PP=0|TP=0]: iteration: 308 / 150000 | consumed_samples: 19712 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786711E+05 | tokens_per_sec_per_gpu: 9.041737E+03 | global_batch_size: 64 | lm_loss: 1.115769E+01 | lr: 4.620E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 4008897.250 [ip-26-0-154-121:0]:06/21/2023 17:17:34 [INFO|DP=0|PP=0|TP=0]: iteration: 309 / 150000 | consumed_samples: 19776 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787520E+05 | tokens_per_sec_per_gpu: 9.043000E+03 | global_batch_size: 64 | lm_loss: 1.110807E+01 | lr: 4.635E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3970255.500 [ip-26-0-154-121:0]:06/21/2023 17:17:35 [INFO|DP=0|PP=0|TP=0]: iteration: 310 / 150000 | consumed_samples: 19840 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786112E+05 | tokens_per_sec_per_gpu: 9.040799E+03 | global_batch_size: 64 | lm_loss: 1.111218E+01 | lr: 4.650E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3933104.000 [ip-26-0-154-121:0]:06/21/2023 17:17:35 [INFO|DP=0|PP=0|TP=0]: iteration: 311 / 150000 | consumed_samples: 19904 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788507E+05 | tokens_per_sec_per_gpu: 9.044543E+03 | global_batch_size: 64 | lm_loss: 1.092911E+01 | lr: 4.665E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 3899037.250 [ip-26-0-154-121:0]:06/21/2023 17:17:36 [INFO|DP=0|PP=0|TP=0]: iteration: 312 / 150000 | consumed_samples: 19968 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787566E+05 | tokens_per_sec_per_gpu: 9.043072E+03 | global_batch_size: 64 | lm_loss: 1.113628E+01 | lr: 4.680E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3868006.500 [ip-26-0-154-121:0]:06/21/2023 17:17:37 [INFO|DP=0|PP=0|TP=0]: iteration: 313 / 150000 | consumed_samples: 20032 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785729E+05 | tokens_per_sec_per_gpu: 9.040202E+03 | global_batch_size: 64 | lm_loss: 1.111782E+01 | lr: 4.695E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3840368.750 [ip-26-0-154-121:0]:06/21/2023 17:17:38 [INFO|DP=0|PP=0|TP=0]: iteration: 314 / 150000 | consumed_samples: 20096 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789909E+05 | tokens_per_sec_per_gpu: 9.046734E+03 | global_batch_size: 64 | lm_loss: 1.100877E+01 | lr: 4.710E-05 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 3815501.750 [ip-26-0-154-121:0]:06/21/2023 17:17:39 [INFO|DP=0|PP=0|TP=0]: iteration: 315 / 150000 | consumed_samples: 20160 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781800E+05 | tokens_per_sec_per_gpu: 9.034063E+03 | global_batch_size: 64 | lm_loss: 1.111978E+01 | lr: 4.725E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3793007.500 [ip-26-0-154-121:0]:06/21/2023 17:17:40 [INFO|DP=0|PP=0|TP=0]: iteration: 316 / 150000 | consumed_samples: 20224 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780317E+05 | tokens_per_sec_per_gpu: 9.031745E+03 | global_batch_size: 64 | lm_loss: 1.110864E+01 | lr: 4.740E-05 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3772963.500 [ip-26-0-154-121:0]:06/21/2023 17:17:41 [INFO|DP=0|PP=0|TP=0]: iteration: 317 / 150000 | consumed_samples: 20288 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784459E+05 | tokens_per_sec_per_gpu: 9.038217E+03 | global_batch_size: 64 | lm_loss: 1.109646E+01 | lr: 4.755E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3755668.750 [ip-26-0-154-121:0]:06/21/2023 17:17:42 [INFO|DP=0|PP=0|TP=0]: iteration: 318 / 150000 | consumed_samples: 20352 | elapsed_time_per_iteration_ms: 912.0 | tokens_per_sec: 5.748992E+05 | tokens_per_sec_per_gpu: 8.982800E+03 | global_batch_size: 64 | lm_loss: 1.109273E+01 | lr: 4.770E-05 | model_tflops_per_gpu: 103.76 | hardware_tflops_per_gpu: 103.76 | grad_norm: 3740855.500 [ip-26-0-154-121:0]:06/21/2023 17:17:43 [INFO|DP=0|PP=0|TP=0]: iteration: 319 / 150000 | consumed_samples: 20416 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782718E+05 | tokens_per_sec_per_gpu: 9.035498E+03 | global_batch_size: 64 | lm_loss: 1.109966E+01 | lr: 4.785E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3728236.500 [ip-26-0-154-121:0]:06/21/2023 17:17:44 [INFO|DP=0|PP=0|TP=0]: iteration: 320 / 150000 | consumed_samples: 20480 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784854E+05 | tokens_per_sec_per_gpu: 9.038835E+03 | global_batch_size: 64 | lm_loss: 1.113495E+01 | lr: 4.800E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3718090.500 [ip-26-0-154-121:0]:06/21/2023 17:17:45 [INFO|DP=0|PP=0|TP=0]: iteration: 321 / 150000 | consumed_samples: 20544 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786795E+05 | tokens_per_sec_per_gpu: 9.041867E+03 | global_batch_size: 64 | lm_loss: 1.115109E+01 | lr: 4.815E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3709105.000 [ip-26-0-154-121:0]:06/21/2023 17:17:45 [INFO|DP=0|PP=0|TP=0]: iteration: 322 / 150000 | consumed_samples: 20608 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791180E+05 | tokens_per_sec_per_gpu: 9.048718E+03 | global_batch_size: 64 | lm_loss: 1.113483E+01 | lr: 4.830E-05 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 3701202.000 [ip-26-0-154-121:0]:06/21/2023 17:17:46 [INFO|DP=0|PP=0|TP=0]: iteration: 323 / 150000 | consumed_samples: 20672 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787584E+05 | tokens_per_sec_per_gpu: 9.043100E+03 | global_batch_size: 64 | lm_loss: 1.053373E+01 | lr: 4.845E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3694034.250 [ip-26-0-154-121:0]:06/21/2023 17:17:47 [INFO|DP=0|PP=0|TP=0]: iteration: 324 / 150000 | consumed_samples: 20736 | elapsed_time_per_iteration_ms: 927.2 | tokens_per_sec: 5.654515E+05 | tokens_per_sec_per_gpu: 8.835179E+03 | global_batch_size: 64 | lm_loss: 1.107660E+01 | lr: 4.860E-05 | model_tflops_per_gpu: 102.06 | hardware_tflops_per_gpu: 102.06 | grad_norm: 3686267.750 [ip-26-0-154-121:0]:06/21/2023 17:17:48 [INFO|DP=0|PP=0|TP=0]: iteration: 325 / 150000 | consumed_samples: 20800 | elapsed_time_per_iteration_ms: 904.9 | tokens_per_sec: 5.794067E+05 | tokens_per_sec_per_gpu: 9.053229E+03 | global_batch_size: 64 | lm_loss: 1.115190E+01 | lr: 4.875E-05 | model_tflops_per_gpu: 104.57 | hardware_tflops_per_gpu: 104.57 | grad_norm: 3679722.000 [ip-26-0-154-121:0]:06/21/2023 17:17:49 [INFO|DP=0|PP=0|TP=0]: iteration: 326 / 150000 | consumed_samples: 20864 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786140E+05 | tokens_per_sec_per_gpu: 9.040844E+03 | global_batch_size: 64 | lm_loss: 1.108704E+01 | lr: 4.890E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3673566.750 [ip-26-0-154-121:0]:06/21/2023 17:17:50 [INFO|DP=0|PP=0|TP=0]: iteration: 327 / 150000 | consumed_samples: 20928 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790134E+05 | tokens_per_sec_per_gpu: 9.047084E+03 | global_batch_size: 64 | lm_loss: 1.117388E+01 | lr: 4.905E-05 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 3668736.000 [ip-26-0-154-121:0]:06/21/2023 17:17:51 [INFO|DP=0|PP=0|TP=0]: iteration: 328 / 150000 | consumed_samples: 20992 | elapsed_time_per_iteration_ms: 904.8 | tokens_per_sec: 5.794412E+05 | tokens_per_sec_per_gpu: 9.053768E+03 | global_batch_size: 64 | lm_loss: 1.126222E+01 | lr: 4.920E-05 | model_tflops_per_gpu: 104.58 | hardware_tflops_per_gpu: 104.58 | grad_norm: 3662933.500 [ip-26-0-154-121:0]:06/21/2023 17:17:52 [INFO|DP=0|PP=0|TP=0]: iteration: 329 / 150000 | consumed_samples: 21056 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790532E+05 | tokens_per_sec_per_gpu: 9.047706E+03 | global_batch_size: 64 | lm_loss: 1.112756E+01 | lr: 4.935E-05 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 3657033.500 [ip-26-0-154-121:0]:06/21/2023 17:17:53 [INFO|DP=0|PP=0|TP=0]: iteration: 330 / 150000 | consumed_samples: 21120 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784303E+05 | tokens_per_sec_per_gpu: 9.037974E+03 | global_batch_size: 64 | lm_loss: 1.107878E+01 | lr: 4.950E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3651927.000 [ip-26-0-154-121:0]:06/21/2023 17:17:54 [INFO|DP=0|PP=0|TP=0]: iteration: 331 / 150000 | consumed_samples: 21184 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785908E+05 | tokens_per_sec_per_gpu: 9.040480E+03 | global_batch_size: 64 | lm_loss: 1.113996E+01 | lr: 4.965E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3647475.000 [ip-26-0-154-121:0]:06/21/2023 17:17:55 [INFO|DP=0|PP=0|TP=0]: iteration: 332 / 150000 | consumed_samples: 21248 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786775E+05 | tokens_per_sec_per_gpu: 9.041837E+03 | global_batch_size: 64 | lm_loss: 1.114939E+01 | lr: 4.980E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3643720.000 [ip-26-0-154-121:0]:06/21/2023 17:17:55 [INFO|DP=0|PP=0|TP=0]: iteration: 333 / 150000 | consumed_samples: 21312 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787656E+05 | tokens_per_sec_per_gpu: 9.043212E+03 | global_batch_size: 64 | lm_loss: 1.111600E+01 | lr: 4.995E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3639116.000 [ip-26-0-154-121:0]:06/21/2023 17:17:56 [INFO|DP=0|PP=0|TP=0]: iteration: 334 / 150000 | consumed_samples: 21376 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785092E+05 | tokens_per_sec_per_gpu: 9.039206E+03 | global_batch_size: 64 | lm_loss: 1.045738E+01 | lr: 5.010E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3634937.250 [ip-26-0-154-121:0]:06/21/2023 17:17:57 [INFO|DP=0|PP=0|TP=0]: iteration: 335 / 150000 | consumed_samples: 21440 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791244E+05 | tokens_per_sec_per_gpu: 9.048818E+03 | global_batch_size: 64 | lm_loss: 1.125290E+01 | lr: 5.025E-05 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 3631092.250 [ip-26-0-154-121:0]:06/21/2023 17:17:58 [INFO|DP=0|PP=0|TP=0]: iteration: 336 / 150000 | consumed_samples: 21504 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786617E+05 | tokens_per_sec_per_gpu: 9.041589E+03 | global_batch_size: 64 | lm_loss: 1.116858E+01 | lr: 5.040E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3628308.250 [ip-26-0-154-121:0]:06/21/2023 17:17:59 [INFO|DP=0|PP=0|TP=0]: iteration: 337 / 150000 | consumed_samples: 21568 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787217E+05 | tokens_per_sec_per_gpu: 9.042527E+03 | global_batch_size: 64 | lm_loss: 1.119941E+01 | lr: 5.055E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3624724.000 [ip-26-0-154-121:0]:06/21/2023 17:18:00 [INFO|DP=0|PP=0|TP=0]: iteration: 338 / 150000 | consumed_samples: 21632 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789460E+05 | tokens_per_sec_per_gpu: 9.046031E+03 | global_batch_size: 64 | lm_loss: 1.089740E+01 | lr: 5.070E-05 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 3621481.750 [ip-26-0-154-121:0]:06/21/2023 17:18:01 [INFO|DP=0|PP=0|TP=0]: iteration: 339 / 150000 | consumed_samples: 21696 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790557E+05 | tokens_per_sec_per_gpu: 9.047746E+03 | global_batch_size: 64 | lm_loss: 1.112902E+01 | lr: 5.085E-05 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 3619020.250 [ip-26-0-154-121:0]:06/21/2023 17:18:02 [INFO|DP=0|PP=0|TP=0]: iteration: 340 / 150000 | consumed_samples: 21760 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788771E+05 | tokens_per_sec_per_gpu: 9.044955E+03 | global_batch_size: 64 | lm_loss: 1.118571E+01 | lr: 5.100E-05 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 3616466.000 [ip-26-0-154-121:0]:06/21/2023 17:18:03 [INFO|DP=0|PP=0|TP=0]: iteration: 341 / 150000 | consumed_samples: 21824 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787608E+05 | tokens_per_sec_per_gpu: 9.043138E+03 | global_batch_size: 64 | lm_loss: 1.118640E+01 | lr: 5.115E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3615278.250 [ip-26-0-154-121:0]:06/21/2023 17:18:04 [INFO|DP=0|PP=0|TP=0]: iteration: 342 / 150000 | consumed_samples: 21888 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785883E+05 | tokens_per_sec_per_gpu: 9.040442E+03 | global_batch_size: 64 | lm_loss: 1.109119E+01 | lr: 5.130E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3612405.750 [ip-26-0-154-121:0]:06/21/2023 17:18:05 [INFO|DP=0|PP=0|TP=0]: iteration: 343 / 150000 | consumed_samples: 21952 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786759E+05 | tokens_per_sec_per_gpu: 9.041810E+03 | global_batch_size: 64 | lm_loss: 1.110293E+01 | lr: 5.145E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3609878.250 [ip-26-0-154-121:0]:06/21/2023 17:18:05 [INFO|DP=0|PP=0|TP=0]: iteration: 344 / 150000 | consumed_samples: 22016 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785749E+05 | tokens_per_sec_per_gpu: 9.040233E+03 | global_batch_size: 64 | lm_loss: 1.112522E+01 | lr: 5.160E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3609038.250 [ip-26-0-154-121:0]:06/21/2023 17:18:06 [INFO|DP=0|PP=0|TP=0]: iteration: 345 / 150000 | consumed_samples: 22080 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790521E+05 | tokens_per_sec_per_gpu: 9.047689E+03 | global_batch_size: 64 | lm_loss: 1.116761E+01 | lr: 5.175E-05 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 3606625.500 [ip-26-0-154-121:0]:06/21/2023 17:18:07 [INFO|DP=0|PP=0|TP=0]: iteration: 346 / 150000 | consumed_samples: 22144 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787086E+05 | tokens_per_sec_per_gpu: 9.042322E+03 | global_batch_size: 64 | lm_loss: 1.115820E+01 | lr: 5.190E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3606842.000 [ip-26-0-154-121:0]:06/21/2023 17:18:08 [INFO|DP=0|PP=0|TP=0]: iteration: 347 / 150000 | consumed_samples: 22208 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786681E+05 | tokens_per_sec_per_gpu: 9.041689E+03 | global_batch_size: 64 | lm_loss: 1.117161E+01 | lr: 5.205E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3604326.500 [ip-26-0-154-121:0]:06/21/2023 17:18:09 [INFO|DP=0|PP=0|TP=0]: iteration: 348 / 150000 | consumed_samples: 22272 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783567E+05 | tokens_per_sec_per_gpu: 9.036824E+03 | global_batch_size: 64 | lm_loss: 1.112422E+01 | lr: 5.220E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3602333.000 [ip-26-0-154-121:0]:06/21/2023 17:18:10 [INFO|DP=0|PP=0|TP=0]: iteration: 349 / 150000 | consumed_samples: 22336 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786644E+05 | tokens_per_sec_per_gpu: 9.041632E+03 | global_batch_size: 64 | lm_loss: 1.115650E+01 | lr: 5.235E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3602690.250 [ip-26-0-154-121:0]:06/21/2023 17:18:11 [INFO|DP=0|PP=0|TP=0]: iteration: 350 / 150000 | consumed_samples: 22400 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787954E+05 | tokens_per_sec_per_gpu: 9.043679E+03 | global_batch_size: 64 | lm_loss: 1.108691E+01 | lr: 5.250E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3601497.000 [ip-26-0-154-121:0]:06/21/2023 17:18:12 [INFO|DP=0|PP=0|TP=0]: iteration: 351 / 150000 | consumed_samples: 22464 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781084E+05 | tokens_per_sec_per_gpu: 9.032944E+03 | global_batch_size: 64 | lm_loss: 1.110615E+01 | lr: 5.265E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3603163.750 [ip-26-0-154-121:0]:06/21/2023 17:18:13 [INFO|DP=0|PP=0|TP=0]: iteration: 352 / 150000 | consumed_samples: 22528 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787275E+05 | tokens_per_sec_per_gpu: 9.042617E+03 | global_batch_size: 64 | lm_loss: 1.070443E+01 | lr: 5.280E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3608165.000 [ip-26-0-154-121:0]:06/21/2023 17:18:14 [INFO|DP=0|PP=0|TP=0]: iteration: 353 / 150000 | consumed_samples: 22592 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785148E+05 | tokens_per_sec_per_gpu: 9.039294E+03 | global_batch_size: 64 | lm_loss: 1.106926E+01 | lr: 5.295E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3617592.500 [ip-26-0-154-121:0]:06/21/2023 17:18:15 [INFO|DP=0|PP=0|TP=0]: iteration: 354 / 150000 | consumed_samples: 22656 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788849E+05 | tokens_per_sec_per_gpu: 9.045076E+03 | global_batch_size: 64 | lm_loss: 1.111356E+01 | lr: 5.310E-05 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 3642278.250 [ip-26-0-154-121:0]:06/21/2023 17:18:15 [INFO|DP=0|PP=0|TP=0]: iteration: 355 / 150000 | consumed_samples: 22720 | elapsed_time_per_iteration_ms: 908.8 | tokens_per_sec: 5.768751E+05 | tokens_per_sec_per_gpu: 9.013674E+03 | global_batch_size: 64 | lm_loss: 1.107862E+01 | lr: 5.325E-05 | model_tflops_per_gpu: 104.12 | hardware_tflops_per_gpu: 104.12 | grad_norm: 3680912.500 [ip-26-0-154-121:0]:06/21/2023 17:18:16 [INFO|DP=0|PP=0|TP=0]: iteration: 356 / 150000 | consumed_samples: 22784 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785528E+05 | tokens_per_sec_per_gpu: 9.039888E+03 | global_batch_size: 64 | lm_loss: 1.117276E+01 | lr: 5.340E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3741433.750 [ip-26-0-154-121:0]:06/21/2023 17:18:17 [INFO|DP=0|PP=0|TP=0]: iteration: 357 / 150000 | consumed_samples: 22848 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789013E+05 | tokens_per_sec_per_gpu: 9.045333E+03 | global_batch_size: 64 | lm_loss: 1.116348E+01 | lr: 5.355E-05 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 3829343.250 [ip-26-0-154-121:0]:06/21/2023 17:18:18 [INFO|DP=0|PP=0|TP=0]: iteration: 358 / 150000 | consumed_samples: 22912 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783873E+05 | tokens_per_sec_per_gpu: 9.037301E+03 | global_batch_size: 64 | lm_loss: 1.108219E+01 | lr: 5.370E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3936736.750 [ip-26-0-154-121:0]:06/21/2023 17:18:19 [INFO|DP=0|PP=0|TP=0]: iteration: 359 / 150000 | consumed_samples: 22976 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785894E+05 | tokens_per_sec_per_gpu: 9.040459E+03 | global_batch_size: 64 | lm_loss: 1.122030E+01 | lr: 5.385E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4054450.000 [ip-26-0-154-121:0]:06/21/2023 17:18:20 [INFO|DP=0|PP=0|TP=0]: iteration: 360 / 150000 | consumed_samples: 23040 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789092E+05 | tokens_per_sec_per_gpu: 9.045457E+03 | global_batch_size: 64 | lm_loss: 1.120922E+01 | lr: 5.400E-05 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 4162179.500 [ip-26-0-154-121:0]:06/21/2023 17:18:21 [INFO|DP=0|PP=0|TP=0]: iteration: 361 / 150000 | consumed_samples: 23104 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787496E+05 | tokens_per_sec_per_gpu: 9.042962E+03 | global_batch_size: 64 | lm_loss: 1.115334E+01 | lr: 5.415E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 4245090.000 [ip-26-0-154-121:0]:06/21/2023 17:18:22 [INFO|DP=0|PP=0|TP=0]: iteration: 362 / 150000 | consumed_samples: 23168 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784195E+05 | tokens_per_sec_per_gpu: 9.037805E+03 | global_batch_size: 64 | lm_loss: 1.113764E+01 | lr: 5.430E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 4299668.000 [ip-26-0-154-121:0]:06/21/2023 17:18:23 [INFO|DP=0|PP=0|TP=0]: iteration: 363 / 150000 | consumed_samples: 23232 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783213E+05 | tokens_per_sec_per_gpu: 9.036270E+03 | global_batch_size: 64 | lm_loss: 1.108342E+01 | lr: 5.445E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 4332002.500 [ip-26-0-154-121:0]:06/21/2023 17:18:24 [INFO|DP=0|PP=0|TP=0]: iteration: 364 / 150000 | consumed_samples: 23296 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789062E+05 | tokens_per_sec_per_gpu: 9.045409E+03 | global_batch_size: 64 | lm_loss: 1.113737E+01 | lr: 5.460E-05 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 4346112.500 [ip-26-0-154-121:0]:06/21/2023 17:18:24 [INFO|DP=0|PP=0|TP=0]: iteration: 365 / 150000 | consumed_samples: 23360 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783891E+05 | tokens_per_sec_per_gpu: 9.037330E+03 | global_batch_size: 64 | lm_loss: 1.108411E+01 | lr: 5.475E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 4347530.500 [ip-26-0-154-121:0]:06/21/2023 17:18:25 [INFO|DP=0|PP=0|TP=0]: iteration: 366 / 150000 | consumed_samples: 23424 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785130E+05 | tokens_per_sec_per_gpu: 9.039265E+03 | global_batch_size: 64 | lm_loss: 1.119439E+01 | lr: 5.490E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 4337910.000 [ip-26-0-154-121:0]:06/21/2023 17:18:26 [INFO|DP=0|PP=0|TP=0]: iteration: 367 / 150000 | consumed_samples: 23488 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788949E+05 | tokens_per_sec_per_gpu: 9.045233E+03 | global_batch_size: 64 | lm_loss: 1.118405E+01 | lr: 5.505E-05 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 4318695.500 [ip-26-0-154-121:0]:06/21/2023 17:18:27 [INFO|DP=0|PP=0|TP=0]: iteration: 368 / 150000 | consumed_samples: 23552 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785210E+05 | tokens_per_sec_per_gpu: 9.039391E+03 | global_batch_size: 64 | lm_loss: 1.096971E+01 | lr: 5.520E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 4290987.500 [ip-26-0-154-121:0]:06/21/2023 17:18:28 [INFO|DP=0|PP=0|TP=0]: iteration: 369 / 150000 | consumed_samples: 23616 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779847E+05 | tokens_per_sec_per_gpu: 9.031011E+03 | global_batch_size: 64 | lm_loss: 1.108742E+01 | lr: 5.535E-05 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4256784.500 [ip-26-0-154-121:0]:06/21/2023 17:18:29 [INFO|DP=0|PP=0|TP=0]: iteration: 370 / 150000 | consumed_samples: 23680 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781934E+05 | tokens_per_sec_per_gpu: 9.034272E+03 | global_batch_size: 64 | lm_loss: 1.112884E+01 | lr: 5.550E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 4216916.000 [ip-26-0-154-121:0]:06/21/2023 17:18:30 [INFO|DP=0|PP=0|TP=0]: iteration: 371 / 150000 | consumed_samples: 23744 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784241E+05 | tokens_per_sec_per_gpu: 9.037877E+03 | global_batch_size: 64 | lm_loss: 1.113804E+01 | lr: 5.565E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 4173030.500 [ip-26-0-154-121:0]:06/21/2023 17:18:31 [INFO|DP=0|PP=0|TP=0]: iteration: 372 / 150000 | consumed_samples: 23808 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783698E+05 | tokens_per_sec_per_gpu: 9.037028E+03 | global_batch_size: 64 | lm_loss: 1.108183E+01 | lr: 5.580E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 4124578.250 [ip-26-0-154-121:0]:06/21/2023 17:18:32 [INFO|DP=0|PP=0|TP=0]: iteration: 373 / 150000 | consumed_samples: 23872 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784766E+05 | tokens_per_sec_per_gpu: 9.038697E+03 | global_batch_size: 64 | lm_loss: 1.108091E+01 | lr: 5.595E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 4074790.000 [ip-26-0-154-121:0]:06/21/2023 17:18:33 [INFO|DP=0|PP=0|TP=0]: iteration: 374 / 150000 | consumed_samples: 23936 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784278E+05 | tokens_per_sec_per_gpu: 9.037934E+03 | global_batch_size: 64 | lm_loss: 1.113343E+01 | lr: 5.610E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 4026360.500 [ip-26-0-154-121:0]:06/21/2023 17:18:34 [INFO|DP=0|PP=0|TP=0]: iteration: 375 / 150000 | consumed_samples: 24000 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783920E+05 | tokens_per_sec_per_gpu: 9.037375E+03 | global_batch_size: 64 | lm_loss: 1.107133E+01 | lr: 5.625E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3979678.750 [ip-26-0-154-121:0]:06/21/2023 17:18:34 [INFO|DP=0|PP=0|TP=0]: iteration: 376 / 150000 | consumed_samples: 24064 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778769E+05 | tokens_per_sec_per_gpu: 9.029326E+03 | global_batch_size: 64 | lm_loss: 1.109678E+01 | lr: 5.640E-05 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3938212.250 [ip-26-0-154-121:0]:06/21/2023 17:18:35 [INFO|DP=0|PP=0|TP=0]: iteration: 377 / 150000 | consumed_samples: 24128 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775381E+05 | tokens_per_sec_per_gpu: 9.024033E+03 | global_batch_size: 64 | lm_loss: 1.081735E+01 | lr: 5.655E-05 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 3898809.250 [ip-26-0-154-121:0]:06/21/2023 17:18:36 [INFO|DP=0|PP=0|TP=0]: iteration: 378 / 150000 | consumed_samples: 24192 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786450E+05 | tokens_per_sec_per_gpu: 9.041327E+03 | global_batch_size: 64 | lm_loss: 1.105460E+01 | lr: 5.670E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3863780.000 [ip-26-0-154-121:0]:06/21/2023 17:18:37 [INFO|DP=0|PP=0|TP=0]: iteration: 379 / 150000 | consumed_samples: 24256 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785845E+05 | tokens_per_sec_per_gpu: 9.040383E+03 | global_batch_size: 64 | lm_loss: 1.117142E+01 | lr: 5.685E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3832330.250 [ip-26-0-154-121:0]:06/21/2023 17:18:38 [INFO|DP=0|PP=0|TP=0]: iteration: 380 / 150000 | consumed_samples: 24320 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786245E+05 | tokens_per_sec_per_gpu: 9.041009E+03 | global_batch_size: 64 | lm_loss: 1.113468E+01 | lr: 5.700E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3804534.500 [ip-26-0-154-121:0]:06/21/2023 17:18:39 [INFO|DP=0|PP=0|TP=0]: iteration: 381 / 150000 | consumed_samples: 24384 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784036E+05 | tokens_per_sec_per_gpu: 9.037556E+03 | global_batch_size: 64 | lm_loss: 1.110428E+01 | lr: 5.715E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3779542.500 [ip-26-0-154-121:0]:06/21/2023 17:18:40 [INFO|DP=0|PP=0|TP=0]: iteration: 382 / 150000 | consumed_samples: 24448 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787220E+05 | tokens_per_sec_per_gpu: 9.042531E+03 | global_batch_size: 64 | lm_loss: 1.114857E+01 | lr: 5.730E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3758022.500 [ip-26-0-154-121:0]:06/21/2023 17:18:41 [INFO|DP=0|PP=0|TP=0]: iteration: 383 / 150000 | consumed_samples: 24512 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787033E+05 | tokens_per_sec_per_gpu: 9.042239E+03 | global_batch_size: 64 | lm_loss: 1.114039E+01 | lr: 5.745E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3740124.000 [ip-26-0-154-121:0]:06/21/2023 17:18:42 [INFO|DP=0|PP=0|TP=0]: iteration: 384 / 150000 | consumed_samples: 24576 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781548E+05 | tokens_per_sec_per_gpu: 9.033668E+03 | global_batch_size: 64 | lm_loss: 1.112029E+01 | lr: 5.760E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3724820.750 [ip-26-0-154-121:0]:06/21/2023 17:18:43 [INFO|DP=0|PP=0|TP=0]: iteration: 385 / 150000 | consumed_samples: 24640 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780820E+05 | tokens_per_sec_per_gpu: 9.032531E+03 | global_batch_size: 64 | lm_loss: 1.113805E+01 | lr: 5.775E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3707863.250 [ip-26-0-154-121:0]:06/21/2023 17:18:44 [INFO|DP=0|PP=0|TP=0]: iteration: 386 / 150000 | consumed_samples: 24704 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786226E+05 | tokens_per_sec_per_gpu: 9.040978E+03 | global_batch_size: 64 | lm_loss: 1.106097E+01 | lr: 5.790E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3695622.750 [ip-26-0-154-121:0]:06/21/2023 17:18:44 [INFO|DP=0|PP=0|TP=0]: iteration: 387 / 150000 | consumed_samples: 24768 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781726E+05 | tokens_per_sec_per_gpu: 9.033946E+03 | global_batch_size: 64 | lm_loss: 1.120526E+01 | lr: 5.805E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3682213.500 [ip-26-0-154-121:0]:06/21/2023 17:18:45 [INFO|DP=0|PP=0|TP=0]: iteration: 388 / 150000 | consumed_samples: 24832 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.778874E+05 | tokens_per_sec_per_gpu: 9.029490E+03 | global_batch_size: 64 | lm_loss: 1.104589E+01 | lr: 5.820E-05 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3671704.500 [ip-26-0-154-121:0]:06/21/2023 17:18:46 [INFO|DP=0|PP=0|TP=0]: iteration: 389 / 150000 | consumed_samples: 24896 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779782E+05 | tokens_per_sec_per_gpu: 9.030909E+03 | global_batch_size: 64 | lm_loss: 1.108980E+01 | lr: 5.835E-05 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 3662037.500 [ip-26-0-154-121:0]:06/21/2023 17:18:47 [INFO|DP=0|PP=0|TP=0]: iteration: 390 / 150000 | consumed_samples: 24960 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.783982E+05 | tokens_per_sec_per_gpu: 9.037472E+03 | global_batch_size: 64 | lm_loss: 1.092413E+01 | lr: 5.850E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3656014.250 [ip-26-0-154-121:0]:06/21/2023 17:18:48 [INFO|DP=0|PP=0|TP=0]: iteration: 391 / 150000 | consumed_samples: 25024 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789922E+05 | tokens_per_sec_per_gpu: 9.046753E+03 | global_batch_size: 64 | lm_loss: 1.109369E+01 | lr: 5.865E-05 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 3644500.000 [ip-26-0-154-121:0]:06/21/2023 17:18:49 [INFO|DP=0|PP=0|TP=0]: iteration: 392 / 150000 | consumed_samples: 25088 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788580E+05 | tokens_per_sec_per_gpu: 9.044657E+03 | global_batch_size: 64 | lm_loss: 1.104853E+01 | lr: 5.880E-05 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 3639992.500 [ip-26-0-154-121:0]:06/21/2023 17:18:50 [INFO|DP=0|PP=0|TP=0]: iteration: 393 / 150000 | consumed_samples: 25152 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786724E+05 | tokens_per_sec_per_gpu: 9.041756E+03 | global_batch_size: 64 | lm_loss: 1.113853E+01 | lr: 5.895E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3630340.750 [ip-26-0-154-121:0]:06/21/2023 17:18:51 [INFO|DP=0|PP=0|TP=0]: iteration: 394 / 150000 | consumed_samples: 25216 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782284E+05 | tokens_per_sec_per_gpu: 9.034818E+03 | global_batch_size: 64 | lm_loss: 1.100858E+01 | lr: 5.910E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3627208.250 [ip-26-0-154-121:0]:06/21/2023 17:18:52 [INFO|DP=0|PP=0|TP=0]: iteration: 395 / 150000 | consumed_samples: 25280 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783217E+05 | tokens_per_sec_per_gpu: 9.036277E+03 | global_batch_size: 64 | lm_loss: 1.125860E+01 | lr: 5.925E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3618812.500 [ip-26-0-154-121:0]:06/21/2023 17:18:53 [INFO|DP=0|PP=0|TP=0]: iteration: 396 / 150000 | consumed_samples: 25344 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785174E+05 | tokens_per_sec_per_gpu: 9.039334E+03 | global_batch_size: 64 | lm_loss: 1.110206E+01 | lr: 5.940E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3614011.750 [ip-26-0-154-121:0]:06/21/2023 17:18:54 [INFO|DP=0|PP=0|TP=0]: iteration: 397 / 150000 | consumed_samples: 25408 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786743E+05 | tokens_per_sec_per_gpu: 9.041787E+03 | global_batch_size: 64 | lm_loss: 1.099786E+01 | lr: 5.955E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3609487.000 [ip-26-0-154-121:0]:06/21/2023 17:18:54 [INFO|DP=0|PP=0|TP=0]: iteration: 398 / 150000 | consumed_samples: 25472 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787647E+05 | tokens_per_sec_per_gpu: 9.043198E+03 | global_batch_size: 64 | lm_loss: 1.107677E+01 | lr: 5.970E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3604553.250 [ip-26-0-154-121:0]:06/21/2023 17:18:55 [INFO|DP=0|PP=0|TP=0]: iteration: 399 / 150000 | consumed_samples: 25536 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781554E+05 | tokens_per_sec_per_gpu: 9.033678E+03 | global_batch_size: 64 | lm_loss: 1.108530E+01 | lr: 5.985E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3603355.000 [ip-26-0-154-121:0]:06/21/2023 17:18:56 [INFO|DP=0|PP=0|TP=0]: iteration: 400 / 150000 | consumed_samples: 25600 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786503E+05 | tokens_per_sec_per_gpu: 9.041411E+03 | global_batch_size: 64 | lm_loss: 1.108955E+01 | lr: 6.000E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3596988.250 [ip-26-0-154-121:0]:06/21/2023 17:18:57 [INFO|DP=0|PP=0|TP=0]: iteration: 401 / 150000 | consumed_samples: 25664 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783923E+05 | tokens_per_sec_per_gpu: 9.037380E+03 | global_batch_size: 64 | lm_loss: 1.111629E+01 | lr: 6.015E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3595538.250 [ip-26-0-154-121:0]:06/21/2023 17:18:58 [INFO|DP=0|PP=0|TP=0]: iteration: 402 / 150000 | consumed_samples: 25728 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789021E+05 | tokens_per_sec_per_gpu: 9.045345E+03 | global_batch_size: 64 | lm_loss: 1.108335E+01 | lr: 6.030E-05 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 3590724.000 [ip-26-0-154-121:0]:06/21/2023 17:18:59 [INFO|DP=0|PP=0|TP=0]: iteration: 403 / 150000 | consumed_samples: 25792 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788017E+05 | tokens_per_sec_per_gpu: 9.043776E+03 | global_batch_size: 64 | lm_loss: 1.103696E+01 | lr: 6.045E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 3589245.750 [ip-26-0-154-121:0]:06/21/2023 17:19:00 [INFO|DP=0|PP=0|TP=0]: iteration: 404 / 150000 | consumed_samples: 25856 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784509E+05 | tokens_per_sec_per_gpu: 9.038295E+03 | global_batch_size: 64 | lm_loss: 1.106113E+01 | lr: 6.060E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3586624.500 [ip-26-0-154-121:0]:06/21/2023 17:19:01 [INFO|DP=0|PP=0|TP=0]: iteration: 405 / 150000 | consumed_samples: 25920 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784443E+05 | tokens_per_sec_per_gpu: 9.038193E+03 | global_batch_size: 64 | lm_loss: 1.103998E+01 | lr: 6.075E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3584653.000 [ip-26-0-154-121:0]:06/21/2023 17:19:02 [INFO|DP=0|PP=0|TP=0]: iteration: 406 / 150000 | consumed_samples: 25984 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787228E+05 | tokens_per_sec_per_gpu: 9.042543E+03 | global_batch_size: 64 | lm_loss: 1.109659E+01 | lr: 6.090E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3586194.000 [ip-26-0-154-121:0]:06/21/2023 17:19:03 [INFO|DP=0|PP=0|TP=0]: iteration: 407 / 150000 | consumed_samples: 26048 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785672E+05 | tokens_per_sec_per_gpu: 9.040112E+03 | global_batch_size: 64 | lm_loss: 1.119142E+01 | lr: 6.105E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3589041.000 [ip-26-0-154-121:0]:06/21/2023 17:19:04 [INFO|DP=0|PP=0|TP=0]: iteration: 408 / 150000 | consumed_samples: 26112 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780141E+05 | tokens_per_sec_per_gpu: 9.031470E+03 | global_batch_size: 64 | lm_loss: 1.109324E+01 | lr: 6.120E-05 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 3594821.750 [ip-26-0-154-121:0]:06/21/2023 17:19:04 [INFO|DP=0|PP=0|TP=0]: iteration: 409 / 150000 | consumed_samples: 26176 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782610E+05 | tokens_per_sec_per_gpu: 9.035329E+03 | global_batch_size: 64 | lm_loss: 1.100499E+01 | lr: 6.135E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3603498.250 [ip-26-0-154-121:0]:06/21/2023 17:19:05 [INFO|DP=0|PP=0|TP=0]: iteration: 410 / 150000 | consumed_samples: 26240 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786728E+05 | tokens_per_sec_per_gpu: 9.041763E+03 | global_batch_size: 64 | lm_loss: 1.102046E+01 | lr: 6.150E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3615370.500 [ip-26-0-154-121:0]:06/21/2023 17:19:06 [INFO|DP=0|PP=0|TP=0]: iteration: 411 / 150000 | consumed_samples: 26304 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785598E+05 | tokens_per_sec_per_gpu: 9.039998E+03 | global_batch_size: 64 | lm_loss: 1.101275E+01 | lr: 6.165E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3625046.500 [ip-26-0-154-121:0]:06/21/2023 17:19:07 [INFO|DP=0|PP=0|TP=0]: iteration: 412 / 150000 | consumed_samples: 26368 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785598E+05 | tokens_per_sec_per_gpu: 9.039998E+03 | global_batch_size: 64 | lm_loss: 1.099459E+01 | lr: 6.180E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3631800.000 [ip-26-0-154-121:0]:06/21/2023 17:19:08 [INFO|DP=0|PP=0|TP=0]: iteration: 413 / 150000 | consumed_samples: 26432 | elapsed_time_per_iteration_ms: 909.5 | tokens_per_sec: 5.764602E+05 | tokens_per_sec_per_gpu: 9.007190E+03 | global_batch_size: 64 | lm_loss: 1.109467E+01 | lr: 6.195E-05 | model_tflops_per_gpu: 104.04 | hardware_tflops_per_gpu: 104.04 | grad_norm: 3637834.000 [ip-26-0-154-121:0]:06/21/2023 17:19:09 [INFO|DP=0|PP=0|TP=0]: iteration: 414 / 150000 | consumed_samples: 26496 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783196E+05 | tokens_per_sec_per_gpu: 9.036244E+03 | global_batch_size: 64 | lm_loss: 1.105632E+01 | lr: 6.210E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3641450.250 [ip-26-0-154-121:0]:06/21/2023 17:19:10 [INFO|DP=0|PP=0|TP=0]: iteration: 415 / 150000 | consumed_samples: 26560 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786105E+05 | tokens_per_sec_per_gpu: 9.040790E+03 | global_batch_size: 64 | lm_loss: 1.104957E+01 | lr: 6.225E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3642832.000 [ip-26-0-154-121:0]:06/21/2023 17:19:11 [INFO|DP=0|PP=0|TP=0]: iteration: 416 / 150000 | consumed_samples: 26624 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785224E+05 | tokens_per_sec_per_gpu: 9.039413E+03 | global_batch_size: 64 | lm_loss: 1.109383E+01 | lr: 6.240E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3642693.500 [ip-26-0-154-121:0]:06/21/2023 17:19:12 [INFO|DP=0|PP=0|TP=0]: iteration: 417 / 150000 | consumed_samples: 26688 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785856E+05 | tokens_per_sec_per_gpu: 9.040400E+03 | global_batch_size: 64 | lm_loss: 1.107047E+01 | lr: 6.255E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3651700.750 [ip-26-0-154-121:0]:06/21/2023 17:19:13 [INFO|DP=0|PP=0|TP=0]: iteration: 418 / 150000 | consumed_samples: 26752 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784311E+05 | tokens_per_sec_per_gpu: 9.037986E+03 | global_batch_size: 64 | lm_loss: 1.114168E+01 | lr: 6.270E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3662433.250 [ip-26-0-154-121:0]:06/21/2023 17:19:14 [INFO|DP=0|PP=0|TP=0]: iteration: 419 / 150000 | consumed_samples: 26816 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777758E+05 | tokens_per_sec_per_gpu: 9.027746E+03 | global_batch_size: 64 | lm_loss: 1.112201E+01 | lr: 6.285E-05 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 3665843.000 [ip-26-0-154-121:0]:06/21/2023 17:19:14 [INFO|DP=0|PP=0|TP=0]: iteration: 420 / 150000 | consumed_samples: 26880 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785554E+05 | tokens_per_sec_per_gpu: 9.039929E+03 | global_batch_size: 64 | lm_loss: 1.102009E+01 | lr: 6.300E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3707827.250 [ip-26-0-154-121:0]:06/21/2023 17:19:15 [INFO|DP=0|PP=0|TP=0]: iteration: 421 / 150000 | consumed_samples: 26944 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778859E+05 | tokens_per_sec_per_gpu: 9.029466E+03 | global_batch_size: 64 | lm_loss: 1.108428E+01 | lr: 6.315E-05 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3740895.750 [ip-26-0-154-121:0]:06/21/2023 17:19:16 [INFO|DP=0|PP=0|TP=0]: iteration: 422 / 150000 | consumed_samples: 27008 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780475E+05 | tokens_per_sec_per_gpu: 9.031992E+03 | global_batch_size: 64 | lm_loss: 1.100699E+01 | lr: 6.330E-05 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3824406.500 [ip-26-0-154-121:0]:06/21/2023 17:19:17 [INFO|DP=0|PP=0|TP=0]: iteration: 423 / 150000 | consumed_samples: 27072 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785865E+05 | tokens_per_sec_per_gpu: 9.040414E+03 | global_batch_size: 64 | lm_loss: 1.108091E+01 | lr: 6.345E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3866975.250 [ip-26-0-154-121:0]:06/21/2023 17:19:18 [INFO|DP=0|PP=0|TP=0]: iteration: 424 / 150000 | consumed_samples: 27136 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782113E+05 | tokens_per_sec_per_gpu: 9.034552E+03 | global_batch_size: 64 | lm_loss: 1.111057E+01 | lr: 6.360E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3952734.500 [ip-26-0-154-121:0]:06/21/2023 17:19:19 [INFO|DP=0|PP=0|TP=0]: iteration: 425 / 150000 | consumed_samples: 27200 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782496E+05 | tokens_per_sec_per_gpu: 9.035151E+03 | global_batch_size: 64 | lm_loss: 1.113269E+01 | lr: 6.375E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3962595.750 [ip-26-0-154-121:0]:06/21/2023 17:19:20 [INFO|DP=0|PP=0|TP=0]: iteration: 426 / 150000 | consumed_samples: 27264 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785156E+05 | tokens_per_sec_per_gpu: 9.039306E+03 | global_batch_size: 64 | lm_loss: 1.110994E+01 | lr: 6.390E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3990756.250 [ip-26-0-154-121:0]:06/21/2023 17:19:21 [INFO|DP=0|PP=0|TP=0]: iteration: 427 / 150000 | consumed_samples: 27328 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787400E+05 | tokens_per_sec_per_gpu: 9.042812E+03 | global_batch_size: 64 | lm_loss: 1.106223E+01 | lr: 6.405E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3999907.000 [ip-26-0-154-121:0]:06/21/2023 17:19:22 [INFO|DP=0|PP=0|TP=0]: iteration: 428 / 150000 | consumed_samples: 27392 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776798E+05 | tokens_per_sec_per_gpu: 9.026248E+03 | global_batch_size: 64 | lm_loss: 1.109256E+01 | lr: 6.420E-05 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 3992886.500 [ip-26-0-154-121:0]:06/21/2023 17:19:23 [INFO|DP=0|PP=0|TP=0]: iteration: 429 / 150000 | consumed_samples: 27456 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778424E+05 | tokens_per_sec_per_gpu: 9.028788E+03 | global_batch_size: 64 | lm_loss: 1.107314E+01 | lr: 6.435E-05 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 3989018.000 [ip-26-0-154-121:0]:06/21/2023 17:19:23 [INFO|DP=0|PP=0|TP=0]: iteration: 430 / 150000 | consumed_samples: 27520 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787455E+05 | tokens_per_sec_per_gpu: 9.042898E+03 | global_batch_size: 64 | lm_loss: 1.105520E+01 | lr: 6.450E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3962690.500 [ip-26-0-154-121:0]:06/21/2023 17:19:24 [INFO|DP=0|PP=0|TP=0]: iteration: 431 / 150000 | consumed_samples: 27584 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786328E+05 | tokens_per_sec_per_gpu: 9.041137E+03 | global_batch_size: 64 | lm_loss: 1.109378E+01 | lr: 6.465E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3961124.250 [ip-26-0-154-121:0]:06/21/2023 17:19:25 [INFO|DP=0|PP=0|TP=0]: iteration: 432 / 150000 | consumed_samples: 27648 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782723E+05 | tokens_per_sec_per_gpu: 9.035505E+03 | global_batch_size: 64 | lm_loss: 1.111727E+01 | lr: 6.480E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3927029.500 [ip-26-0-154-121:0]:06/21/2023 17:19:26 [INFO|DP=0|PP=0|TP=0]: iteration: 433 / 150000 | consumed_samples: 27712 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782509E+05 | tokens_per_sec_per_gpu: 9.035170E+03 | global_batch_size: 64 | lm_loss: 1.108480E+01 | lr: 6.495E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3907641.500 [ip-26-0-154-121:0]:06/21/2023 17:19:27 [INFO|DP=0|PP=0|TP=0]: iteration: 434 / 150000 | consumed_samples: 27776 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776712E+05 | tokens_per_sec_per_gpu: 9.026112E+03 | global_batch_size: 64 | lm_loss: 1.105085E+01 | lr: 6.510E-05 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 3893545.000 [ip-26-0-154-121:0]:06/21/2023 17:19:28 [INFO|DP=0|PP=0|TP=0]: iteration: 435 / 150000 | consumed_samples: 27840 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785350E+05 | tokens_per_sec_per_gpu: 9.039610E+03 | global_batch_size: 64 | lm_loss: 1.106369E+01 | lr: 6.525E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3861474.750 [ip-26-0-154-121:0]:06/21/2023 17:19:29 [INFO|DP=0|PP=0|TP=0]: iteration: 436 / 150000 | consumed_samples: 27904 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786503E+05 | tokens_per_sec_per_gpu: 9.041411E+03 | global_batch_size: 64 | lm_loss: 1.108676E+01 | lr: 6.540E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3837130.000 [ip-26-0-154-121:0]:06/21/2023 17:19:30 [INFO|DP=0|PP=0|TP=0]: iteration: 437 / 150000 | consumed_samples: 27968 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783427E+05 | tokens_per_sec_per_gpu: 9.036605E+03 | global_batch_size: 64 | lm_loss: 1.100342E+01 | lr: 6.555E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3815424.250 [ip-26-0-154-121:0]:06/21/2023 17:19:31 [INFO|DP=0|PP=0|TP=0]: iteration: 438 / 150000 | consumed_samples: 28032 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784844E+05 | tokens_per_sec_per_gpu: 9.038818E+03 | global_batch_size: 64 | lm_loss: 1.100312E+01 | lr: 6.570E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3789140.000 [ip-26-0-154-121:0]:06/21/2023 17:19:32 [INFO|DP=0|PP=0|TP=0]: iteration: 439 / 150000 | consumed_samples: 28096 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785833E+05 | tokens_per_sec_per_gpu: 9.040364E+03 | global_batch_size: 64 | lm_loss: 1.097205E+01 | lr: 6.585E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3770521.750 [ip-26-0-154-121:0]:06/21/2023 17:19:33 [INFO|DP=0|PP=0|TP=0]: iteration: 440 / 150000 | consumed_samples: 28160 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781315E+05 | tokens_per_sec_per_gpu: 9.033305E+03 | global_batch_size: 64 | lm_loss: 1.110869E+01 | lr: 6.600E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3746081.250 [ip-26-0-154-121:0]:06/21/2023 17:19:33 [INFO|DP=0|PP=0|TP=0]: iteration: 441 / 150000 | consumed_samples: 28224 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788233E+05 | tokens_per_sec_per_gpu: 9.044114E+03 | global_batch_size: 64 | lm_loss: 1.104561E+01 | lr: 6.615E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 3728149.250 [ip-26-0-154-121:0]:06/21/2023 17:19:34 [INFO|DP=0|PP=0|TP=0]: iteration: 442 / 150000 | consumed_samples: 28288 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785789E+05 | tokens_per_sec_per_gpu: 9.040295E+03 | global_batch_size: 64 | lm_loss: 1.099457E+01 | lr: 6.630E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3710327.500 [ip-26-0-154-121:0]:06/21/2023 17:19:35 [INFO|DP=0|PP=0|TP=0]: iteration: 443 / 150000 | consumed_samples: 28352 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784308E+05 | tokens_per_sec_per_gpu: 9.037981E+03 | global_batch_size: 64 | lm_loss: 1.105665E+01 | lr: 6.645E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3700207.000 [ip-26-0-154-121:0]:06/21/2023 17:19:36 [INFO|DP=0|PP=0|TP=0]: iteration: 444 / 150000 | consumed_samples: 28416 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785595E+05 | tokens_per_sec_per_gpu: 9.039993E+03 | global_batch_size: 64 | lm_loss: 1.107874E+01 | lr: 6.660E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3682231.000 [ip-26-0-154-121:0]:06/21/2023 17:19:37 [INFO|DP=0|PP=0|TP=0]: iteration: 445 / 150000 | consumed_samples: 28480 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784375E+05 | tokens_per_sec_per_gpu: 9.038086E+03 | global_batch_size: 64 | lm_loss: 1.097873E+01 | lr: 6.675E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3686324.250 [ip-26-0-154-121:0]:06/21/2023 17:19:38 [INFO|DP=0|PP=0|TP=0]: iteration: 446 / 150000 | consumed_samples: 28544 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786614E+05 | tokens_per_sec_per_gpu: 9.041584E+03 | global_batch_size: 64 | lm_loss: 1.103704E+01 | lr: 6.690E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3667853.250 [ip-26-0-154-121:0]:06/21/2023 17:19:39 [INFO|DP=0|PP=0|TP=0]: iteration: 447 / 150000 | consumed_samples: 28608 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781894E+05 | tokens_per_sec_per_gpu: 9.034210E+03 | global_batch_size: 64 | lm_loss: 1.100893E+01 | lr: 6.705E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3650881.750 [ip-26-0-154-121:0]:06/21/2023 17:19:40 [INFO|DP=0|PP=0|TP=0]: iteration: 448 / 150000 | consumed_samples: 28672 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783403E+05 | tokens_per_sec_per_gpu: 9.036567E+03 | global_batch_size: 64 | lm_loss: 1.107380E+01 | lr: 6.720E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3640035.250 [ip-26-0-154-121:0]:06/21/2023 17:19:41 [INFO|DP=0|PP=0|TP=0]: iteration: 449 / 150000 | consumed_samples: 28736 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785656E+05 | tokens_per_sec_per_gpu: 9.040088E+03 | global_batch_size: 64 | lm_loss: 1.109988E+01 | lr: 6.735E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3631014.500 [ip-26-0-154-121:0]:06/21/2023 17:19:42 [INFO|DP=0|PP=0|TP=0]: iteration: 450 / 150000 | consumed_samples: 28800 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784156E+05 | tokens_per_sec_per_gpu: 9.037743E+03 | global_batch_size: 64 | lm_loss: 1.104869E+01 | lr: 6.750E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3621394.000 [ip-26-0-154-121:0]:06/21/2023 17:19:43 [INFO|DP=0|PP=0|TP=0]: iteration: 451 / 150000 | consumed_samples: 28864 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784682E+05 | tokens_per_sec_per_gpu: 9.038566E+03 | global_batch_size: 64 | lm_loss: 1.108697E+01 | lr: 6.765E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3623748.750 [ip-26-0-154-121:0]:06/21/2023 17:19:43 [INFO|DP=0|PP=0|TP=0]: iteration: 452 / 150000 | consumed_samples: 28928 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786297E+05 | tokens_per_sec_per_gpu: 9.041089E+03 | global_batch_size: 64 | lm_loss: 1.106239E+01 | lr: 6.780E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3610864.250 [ip-26-0-154-121:0]:06/21/2023 17:19:44 [INFO|DP=0|PP=0|TP=0]: iteration: 453 / 150000 | consumed_samples: 28992 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790068E+05 | tokens_per_sec_per_gpu: 9.046981E+03 | global_batch_size: 64 | lm_loss: 1.105264E+01 | lr: 6.795E-05 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 3600341.500 [ip-26-0-154-121:0]:06/21/2023 17:19:45 [INFO|DP=0|PP=0|TP=0]: iteration: 454 / 150000 | consumed_samples: 29056 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780953E+05 | tokens_per_sec_per_gpu: 9.032740E+03 | global_batch_size: 64 | lm_loss: 1.109763E+01 | lr: 6.810E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3595385.250 [ip-26-0-154-121:0]:06/21/2023 17:19:46 [INFO|DP=0|PP=0|TP=0]: iteration: 455 / 150000 | consumed_samples: 29120 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780770E+05 | tokens_per_sec_per_gpu: 9.032453E+03 | global_batch_size: 64 | lm_loss: 1.112678E+01 | lr: 6.825E-05 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3588656.000 [ip-26-0-154-121:0]:06/21/2023 17:19:47 [INFO|DP=0|PP=0|TP=0]: iteration: 456 / 150000 | consumed_samples: 29184 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783254E+05 | tokens_per_sec_per_gpu: 9.036334E+03 | global_batch_size: 64 | lm_loss: 1.105738E+01 | lr: 6.840E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3583214.250 [ip-26-0-154-121:0]:06/21/2023 17:19:48 [INFO|DP=0|PP=0|TP=0]: iteration: 457 / 150000 | consumed_samples: 29248 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788690E+05 | tokens_per_sec_per_gpu: 9.044828E+03 | global_batch_size: 64 | lm_loss: 1.109337E+01 | lr: 6.855E-05 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 3578705.500 [ip-26-0-154-121:0]:06/21/2023 17:19:49 [INFO|DP=0|PP=0|TP=0]: iteration: 458 / 150000 | consumed_samples: 29312 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788300E+05 | tokens_per_sec_per_gpu: 9.044219E+03 | global_batch_size: 64 | lm_loss: 1.098846E+01 | lr: 6.870E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 3577196.000 [ip-26-0-154-121:0]:06/21/2023 17:19:50 [INFO|DP=0|PP=0|TP=0]: iteration: 459 / 150000 | consumed_samples: 29376 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784503E+05 | tokens_per_sec_per_gpu: 9.038286E+03 | global_batch_size: 64 | lm_loss: 1.053824E+01 | lr: 6.885E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3578573.750 [ip-26-0-154-121:0]:06/21/2023 17:19:51 [INFO|DP=0|PP=0|TP=0]: iteration: 460 / 150000 | consumed_samples: 29440 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787934E+05 | tokens_per_sec_per_gpu: 9.043648E+03 | global_batch_size: 64 | lm_loss: 1.114891E+01 | lr: 6.900E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3616039.250 [ip-26-0-154-121:0]:06/21/2023 17:19:52 [INFO|DP=0|PP=0|TP=0]: iteration: 461 / 150000 | consumed_samples: 29504 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780204E+05 | tokens_per_sec_per_gpu: 9.031569E+03 | global_batch_size: 64 | lm_loss: 1.104966E+01 | lr: 6.915E-05 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 3566693.500 [ip-26-0-154-121:0]:06/21/2023 17:19:53 [INFO|DP=0|PP=0|TP=0]: iteration: 462 / 150000 | consumed_samples: 29568 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781139E+05 | tokens_per_sec_per_gpu: 9.033030E+03 | global_batch_size: 64 | lm_loss: 1.112763E+01 | lr: 6.930E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3595024.000 [ip-26-0-154-121:0]:06/21/2023 17:19:53 [INFO|DP=0|PP=0|TP=0]: iteration: 463 / 150000 | consumed_samples: 29632 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784326E+05 | tokens_per_sec_per_gpu: 9.038010E+03 | global_batch_size: 64 | lm_loss: 1.109176E+01 | lr: 6.945E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3562843.500 [ip-26-0-154-121:0]:06/21/2023 17:19:54 [INFO|DP=0|PP=0|TP=0]: iteration: 464 / 150000 | consumed_samples: 29696 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779258E+05 | tokens_per_sec_per_gpu: 9.030091E+03 | global_batch_size: 64 | lm_loss: 1.108825E+01 | lr: 6.960E-05 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 3591055.500 [ip-26-0-154-121:0]:06/21/2023 17:19:55 [INFO|DP=0|PP=0|TP=0]: iteration: 465 / 150000 | consumed_samples: 29760 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785034E+05 | tokens_per_sec_per_gpu: 9.039115E+03 | global_batch_size: 64 | lm_loss: 1.103741E+01 | lr: 6.975E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3560746.000 [ip-26-0-154-121:0]:06/21/2023 17:19:56 [INFO|DP=0|PP=0|TP=0]: iteration: 466 / 150000 | consumed_samples: 29824 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786890E+05 | tokens_per_sec_per_gpu: 9.042015E+03 | global_batch_size: 64 | lm_loss: 1.101634E+01 | lr: 6.990E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3566878.500 [ip-26-0-154-121:0]:06/21/2023 17:19:57 [INFO|DP=0|PP=0|TP=0]: iteration: 467 / 150000 | consumed_samples: 29888 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784319E+05 | tokens_per_sec_per_gpu: 9.037998E+03 | global_batch_size: 64 | lm_loss: 1.080492E+01 | lr: 7.005E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3546607.750 [ip-26-0-154-121:0]:06/21/2023 17:19:58 [INFO|DP=0|PP=0|TP=0]: iteration: 468 / 150000 | consumed_samples: 29952 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787366E+05 | tokens_per_sec_per_gpu: 9.042760E+03 | global_batch_size: 64 | lm_loss: 1.096451E+01 | lr: 7.020E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3586165.000 [ip-26-0-154-121:0]:06/21/2023 17:19:59 [INFO|DP=0|PP=0|TP=0]: iteration: 469 / 150000 | consumed_samples: 30016 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789498E+05 | tokens_per_sec_per_gpu: 9.046090E+03 | global_batch_size: 64 | lm_loss: 1.107338E+01 | lr: 7.035E-05 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 3544761.250 [ip-26-0-154-121:0]:06/21/2023 17:20:00 [INFO|DP=0|PP=0|TP=0]: iteration: 470 / 150000 | consumed_samples: 30080 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785661E+05 | tokens_per_sec_per_gpu: 9.040095E+03 | global_batch_size: 64 | lm_loss: 1.105919E+01 | lr: 7.050E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3581368.750 [ip-26-0-154-121:0]:06/21/2023 17:20:01 [INFO|DP=0|PP=0|TP=0]: iteration: 471 / 150000 | consumed_samples: 30144 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785769E+05 | tokens_per_sec_per_gpu: 9.040264E+03 | global_batch_size: 64 | lm_loss: 1.112027E+01 | lr: 7.065E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3548180.750 [ip-26-0-154-121:0]:06/21/2023 17:20:02 [INFO|DP=0|PP=0|TP=0]: iteration: 472 / 150000 | consumed_samples: 30208 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787078E+05 | tokens_per_sec_per_gpu: 9.042310E+03 | global_batch_size: 64 | lm_loss: 1.106706E+01 | lr: 7.080E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3563826.750 [ip-26-0-154-121:0]:06/21/2023 17:20:03 [INFO|DP=0|PP=0|TP=0]: iteration: 473 / 150000 | consumed_samples: 30272 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783628E+05 | tokens_per_sec_per_gpu: 9.036919E+03 | global_batch_size: 64 | lm_loss: 1.083844E+01 | lr: 7.095E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3564457.250 [ip-26-0-154-121:0]:06/21/2023 17:20:03 [INFO|DP=0|PP=0|TP=0]: iteration: 474 / 150000 | consumed_samples: 30336 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786690E+05 | tokens_per_sec_per_gpu: 9.041703E+03 | global_batch_size: 64 | lm_loss: 1.105259E+01 | lr: 7.110E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3574313.250 [ip-26-0-154-121:0]:06/21/2023 17:20:04 [INFO|DP=0|PP=0|TP=0]: iteration: 475 / 150000 | consumed_samples: 30400 | elapsed_time_per_iteration_ms: 909.3 | tokens_per_sec: 5.766009E+05 | tokens_per_sec_per_gpu: 9.009389E+03 | global_batch_size: 64 | lm_loss: 1.117357E+01 | lr: 7.125E-05 | model_tflops_per_gpu: 104.07 | hardware_tflops_per_gpu: 104.07 | grad_norm: 3619669.750 [ip-26-0-154-121:0]:06/21/2023 17:20:05 [INFO|DP=0|PP=0|TP=0]: iteration: 476 / 150000 | consumed_samples: 30464 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786335E+05 | tokens_per_sec_per_gpu: 9.041149E+03 | global_batch_size: 64 | lm_loss: 1.107265E+01 | lr: 7.140E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3627790.250 [ip-26-0-154-121:0]:06/21/2023 17:20:06 [INFO|DP=0|PP=0|TP=0]: iteration: 477 / 150000 | consumed_samples: 30528 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784792E+05 | tokens_per_sec_per_gpu: 9.038737E+03 | global_batch_size: 64 | lm_loss: 1.105877E+01 | lr: 7.155E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3691694.000 [ip-26-0-154-121:0]:06/21/2023 17:20:07 [INFO|DP=0|PP=0|TP=0]: iteration: 478 / 150000 | consumed_samples: 30592 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787933E+05 | tokens_per_sec_per_gpu: 9.043645E+03 | global_batch_size: 64 | lm_loss: 1.108687E+01 | lr: 7.170E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3699732.500 [ip-26-0-154-121:0]:06/21/2023 17:20:08 [INFO|DP=0|PP=0|TP=0]: iteration: 479 / 150000 | consumed_samples: 30656 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787974E+05 | tokens_per_sec_per_gpu: 9.043709E+03 | global_batch_size: 64 | lm_loss: 1.109903E+01 | lr: 7.185E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3755361.500 [ip-26-0-154-121:0]:06/21/2023 17:20:09 [INFO|DP=0|PP=0|TP=0]: iteration: 480 / 150000 | consumed_samples: 30720 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788280E+05 | tokens_per_sec_per_gpu: 9.044188E+03 | global_batch_size: 64 | lm_loss: 1.110820E+01 | lr: 7.200E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 3752825.250 [ip-26-0-154-121:0]:06/21/2023 17:20:10 [INFO|DP=0|PP=0|TP=0]: iteration: 481 / 150000 | consumed_samples: 30784 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782071E+05 | tokens_per_sec_per_gpu: 9.034485E+03 | global_batch_size: 64 | lm_loss: 1.109007E+01 | lr: 7.215E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3768661.500 [ip-26-0-154-121:0]:06/21/2023 17:20:11 [INFO|DP=0|PP=0|TP=0]: iteration: 482 / 150000 | consumed_samples: 30848 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780950E+05 | tokens_per_sec_per_gpu: 9.032735E+03 | global_batch_size: 64 | lm_loss: 1.109368E+01 | lr: 7.230E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3766908.000 [ip-26-0-154-121:0]:06/21/2023 17:20:12 [INFO|DP=0|PP=0|TP=0]: iteration: 483 / 150000 | consumed_samples: 30912 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776703E+05 | tokens_per_sec_per_gpu: 9.026098E+03 | global_batch_size: 64 | lm_loss: 1.110221E+01 | lr: 7.245E-05 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 3763368.000 [ip-26-0-154-121:0]:06/21/2023 17:20:13 [INFO|DP=0|PP=0|TP=0]: iteration: 484 / 150000 | consumed_samples: 30976 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776692E+05 | tokens_per_sec_per_gpu: 9.026082E+03 | global_batch_size: 64 | lm_loss: 1.111520E+01 | lr: 7.260E-05 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 3761443.250 [ip-26-0-154-121:0]:06/21/2023 17:20:13 [INFO|DP=0|PP=0|TP=0]: iteration: 485 / 150000 | consumed_samples: 31040 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781104E+05 | tokens_per_sec_per_gpu: 9.032975E+03 | global_batch_size: 64 | lm_loss: 1.107912E+01 | lr: 7.275E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3742610.750 [ip-26-0-154-121:0]:06/21/2023 17:20:14 [INFO|DP=0|PP=0|TP=0]: iteration: 486 / 150000 | consumed_samples: 31104 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775096E+05 | tokens_per_sec_per_gpu: 9.023588E+03 | global_batch_size: 64 | lm_loss: 1.109546E+01 | lr: 7.290E-05 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 3733380.750 [ip-26-0-154-121:0]:06/21/2023 17:20:15 [INFO|DP=0|PP=0|TP=0]: iteration: 487 / 150000 | consumed_samples: 31168 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780701E+05 | tokens_per_sec_per_gpu: 9.032346E+03 | global_batch_size: 64 | lm_loss: 1.109708E+01 | lr: 7.305E-05 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3720531.250 [ip-26-0-154-121:0]:06/21/2023 17:20:16 [INFO|DP=0|PP=0|TP=0]: iteration: 488 / 150000 | consumed_samples: 31232 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785525E+05 | tokens_per_sec_per_gpu: 9.039883E+03 | global_batch_size: 64 | lm_loss: 1.088488E+01 | lr: 7.320E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3693308.000 [ip-26-0-154-121:0]:06/21/2023 17:20:17 [INFO|DP=0|PP=0|TP=0]: iteration: 489 / 150000 | consumed_samples: 31296 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783013E+05 | tokens_per_sec_per_gpu: 9.035959E+03 | global_batch_size: 64 | lm_loss: 1.104444E+01 | lr: 7.335E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3685243.500 [ip-26-0-154-121:0]:06/21/2023 17:20:18 [INFO|DP=0|PP=0|TP=0]: iteration: 490 / 150000 | consumed_samples: 31360 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784547E+05 | tokens_per_sec_per_gpu: 9.038354E+03 | global_batch_size: 64 | lm_loss: 1.087018E+01 | lr: 7.350E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3668565.750 [ip-26-0-154-121:0]:06/21/2023 17:20:19 [INFO|DP=0|PP=0|TP=0]: iteration: 491 / 150000 | consumed_samples: 31424 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781979E+05 | tokens_per_sec_per_gpu: 9.034343E+03 | global_batch_size: 64 | lm_loss: 1.104518E+01 | lr: 7.365E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3650981.250 [ip-26-0-154-121:0]:06/21/2023 17:20:20 [INFO|DP=0|PP=0|TP=0]: iteration: 492 / 150000 | consumed_samples: 31488 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778562E+05 | tokens_per_sec_per_gpu: 9.029004E+03 | global_batch_size: 64 | lm_loss: 1.107311E+01 | lr: 7.380E-05 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3652460.250 [ip-26-0-154-121:0]:06/21/2023 17:20:21 [INFO|DP=0|PP=0|TP=0]: iteration: 493 / 150000 | consumed_samples: 31552 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781884E+05 | tokens_per_sec_per_gpu: 9.034193E+03 | global_batch_size: 64 | lm_loss: 1.089965E+01 | lr: 7.395E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3626980.000 [ip-26-0-154-121:0]:06/21/2023 17:20:22 [INFO|DP=0|PP=0|TP=0]: iteration: 494 / 150000 | consumed_samples: 31616 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782396E+05 | tokens_per_sec_per_gpu: 9.034994E+03 | global_batch_size: 64 | lm_loss: 1.107622E+01 | lr: 7.410E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3621626.750 [ip-26-0-154-121:0]:06/21/2023 17:20:22 [INFO|DP=0|PP=0|TP=0]: iteration: 495 / 150000 | consumed_samples: 31680 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779986E+05 | tokens_per_sec_per_gpu: 9.031227E+03 | global_batch_size: 64 | lm_loss: 1.097989E+01 | lr: 7.425E-05 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 3612010.000 [ip-26-0-154-121:0]:06/21/2023 17:20:23 [INFO|DP=0|PP=0|TP=0]: iteration: 496 / 150000 | consumed_samples: 31744 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782960E+05 | tokens_per_sec_per_gpu: 9.035875E+03 | global_batch_size: 64 | lm_loss: 1.104337E+01 | lr: 7.440E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3608082.250 [ip-26-0-154-121:0]:06/21/2023 17:20:24 [INFO|DP=0|PP=0|TP=0]: iteration: 497 / 150000 | consumed_samples: 31808 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778476E+05 | tokens_per_sec_per_gpu: 9.028869E+03 | global_batch_size: 64 | lm_loss: 1.104897E+01 | lr: 7.455E-05 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 3598713.500 [ip-26-0-154-121:0]:06/21/2023 17:20:25 [INFO|DP=0|PP=0|TP=0]: iteration: 498 / 150000 | consumed_samples: 31872 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781356E+05 | tokens_per_sec_per_gpu: 9.033369E+03 | global_batch_size: 64 | lm_loss: 1.106824E+01 | lr: 7.470E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3595931.750 [ip-26-0-154-121:0]:06/21/2023 17:20:26 [INFO|DP=0|PP=0|TP=0]: iteration: 499 / 150000 | consumed_samples: 31936 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780469E+05 | tokens_per_sec_per_gpu: 9.031982E+03 | global_batch_size: 64 | lm_loss: 1.110506E+01 | lr: 7.485E-05 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3614982.750 [ip-26-0-154-121:0]:06/21/2023 17:20:27 [INFO|DP=0|PP=0|TP=0]: iteration: 500 / 150000 | consumed_samples: 32000 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.773836E+05 | tokens_per_sec_per_gpu: 9.021619E+03 | global_batch_size: 64 | lm_loss: 1.116818E+01 | lr: 7.500E-05 | model_tflops_per_gpu: 104.21 | hardware_tflops_per_gpu: 104.21 | grad_norm: 3580557.500 [ip-26-0-154-121:0]:06/21/2023 17:20:28 [INFO|DP=0|PP=0|TP=0]: iteration: 501 / 150000 | consumed_samples: 32064 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780855E+05 | tokens_per_sec_per_gpu: 9.032585E+03 | global_batch_size: 64 | lm_loss: 1.105916E+01 | lr: 7.515E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3593786.250 [ip-26-0-154-121:0]:06/21/2023 17:20:29 [INFO|DP=0|PP=0|TP=0]: iteration: 502 / 150000 | consumed_samples: 32128 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779106E+05 | tokens_per_sec_per_gpu: 9.029853E+03 | global_batch_size: 64 | lm_loss: 1.109999E+01 | lr: 7.530E-05 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3614647.250 [ip-26-0-154-121:0]:06/21/2023 17:20:30 [INFO|DP=0|PP=0|TP=0]: iteration: 503 / 150000 | consumed_samples: 32192 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783970E+05 | tokens_per_sec_per_gpu: 9.037453E+03 | global_batch_size: 64 | lm_loss: 1.108590E+01 | lr: 7.545E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3558686.000 [ip-26-0-154-121:0]:06/21/2023 17:20:31 [INFO|DP=0|PP=0|TP=0]: iteration: 504 / 150000 | consumed_samples: 32256 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783520E+05 | tokens_per_sec_per_gpu: 9.036750E+03 | global_batch_size: 64 | lm_loss: 1.081064E+01 | lr: 7.560E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3560194.500 [ip-26-0-154-121:0]:06/21/2023 17:20:32 [INFO|DP=0|PP=0|TP=0]: iteration: 505 / 150000 | consumed_samples: 32320 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783222E+05 | tokens_per_sec_per_gpu: 9.036284E+03 | global_batch_size: 64 | lm_loss: 1.095916E+01 | lr: 7.575E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3550265.000 [ip-26-0-154-121:0]:06/21/2023 17:20:32 [INFO|DP=0|PP=0|TP=0]: iteration: 506 / 150000 | consumed_samples: 32384 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789818E+05 | tokens_per_sec_per_gpu: 9.046591E+03 | global_batch_size: 64 | lm_loss: 1.108473E+01 | lr: 7.590E-05 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 3544418.500 [ip-26-0-154-121:0]:06/21/2023 17:20:33 [INFO|DP=0|PP=0|TP=0]: iteration: 507 / 150000 | consumed_samples: 32448 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780422E+05 | tokens_per_sec_per_gpu: 9.031909E+03 | global_batch_size: 64 | lm_loss: 1.107931E+01 | lr: 7.605E-05 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3539931.750 [ip-26-0-154-121:0]:06/21/2023 17:20:34 [INFO|DP=0|PP=0|TP=0]: iteration: 508 / 150000 | consumed_samples: 32512 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782626E+05 | tokens_per_sec_per_gpu: 9.035353E+03 | global_batch_size: 64 | lm_loss: 1.099059E+01 | lr: 7.620E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3538525.500 [ip-26-0-154-121:0]:06/21/2023 17:20:35 [INFO|DP=0|PP=0|TP=0]: iteration: 509 / 150000 | consumed_samples: 32576 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778650E+05 | tokens_per_sec_per_gpu: 9.029141E+03 | global_batch_size: 64 | lm_loss: 1.109897E+01 | lr: 7.635E-05 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3533959.500 [ip-26-0-154-121:0]:06/21/2023 17:20:36 [INFO|DP=0|PP=0|TP=0]: iteration: 510 / 150000 | consumed_samples: 32640 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784568E+05 | tokens_per_sec_per_gpu: 9.038388E+03 | global_batch_size: 64 | lm_loss: 1.113114E+01 | lr: 7.650E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3529631.000 [ip-26-0-154-121:0]:06/21/2023 17:20:37 [INFO|DP=0|PP=0|TP=0]: iteration: 511 / 150000 | consumed_samples: 32704 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787752E+05 | tokens_per_sec_per_gpu: 9.043362E+03 | global_batch_size: 64 | lm_loss: 1.097615E+01 | lr: 7.665E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3533870.000 [ip-26-0-154-121:0]:06/21/2023 17:20:38 [INFO|DP=0|PP=0|TP=0]: iteration: 512 / 150000 | consumed_samples: 32768 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782747E+05 | tokens_per_sec_per_gpu: 9.035543E+03 | global_batch_size: 64 | lm_loss: 1.100149E+01 | lr: 7.680E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3546009.500 [ip-26-0-154-121:0]:06/21/2023 17:20:39 [INFO|DP=0|PP=0|TP=0]: iteration: 513 / 150000 | consumed_samples: 32832 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775368E+05 | tokens_per_sec_per_gpu: 9.024012E+03 | global_batch_size: 64 | lm_loss: 1.095181E+01 | lr: 7.695E-05 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 3538146.250 [ip-26-0-154-121:0]:06/21/2023 17:20:40 [INFO|DP=0|PP=0|TP=0]: iteration: 514 / 150000 | consumed_samples: 32896 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777908E+05 | tokens_per_sec_per_gpu: 9.027981E+03 | global_batch_size: 64 | lm_loss: 1.105748E+01 | lr: 7.710E-05 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 3524216.250 [ip-26-0-154-121:0]:06/21/2023 17:20:41 [INFO|DP=0|PP=0|TP=0]: iteration: 515 / 150000 | consumed_samples: 32960 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778409E+05 | tokens_per_sec_per_gpu: 9.028764E+03 | global_batch_size: 64 | lm_loss: 1.109083E+01 | lr: 7.725E-05 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 3547124.750 [ip-26-0-154-121:0]:06/21/2023 17:20:42 [INFO|DP=0|PP=0|TP=0]: iteration: 516 / 150000 | consumed_samples: 33024 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776817E+05 | tokens_per_sec_per_gpu: 9.026276E+03 | global_batch_size: 64 | lm_loss: 1.109105E+01 | lr: 7.740E-05 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 3614089.000 [ip-26-0-154-121:0]:06/21/2023 17:20:42 [INFO|DP=0|PP=0|TP=0]: iteration: 517 / 150000 | consumed_samples: 33088 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775353E+05 | tokens_per_sec_per_gpu: 9.023988E+03 | global_batch_size: 64 | lm_loss: 1.107150E+01 | lr: 7.755E-05 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 3564581.750 [ip-26-0-154-121:0]:06/21/2023 17:20:43 [INFO|DP=0|PP=0|TP=0]: iteration: 518 / 150000 | consumed_samples: 33152 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782554E+05 | tokens_per_sec_per_gpu: 9.035241E+03 | global_batch_size: 64 | lm_loss: 1.106899E+01 | lr: 7.770E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3530156.000 [ip-26-0-154-121:0]:06/21/2023 17:20:44 [INFO|DP=0|PP=0|TP=0]: iteration: 519 / 150000 | consumed_samples: 33216 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779057E+05 | tokens_per_sec_per_gpu: 9.029777E+03 | global_batch_size: 64 | lm_loss: 1.109651E+01 | lr: 7.785E-05 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3601729.500 [ip-26-0-154-121:0]:06/21/2023 17:20:45 [INFO|DP=0|PP=0|TP=0]: iteration: 520 / 150000 | consumed_samples: 33280 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784503E+05 | tokens_per_sec_per_gpu: 9.038286E+03 | global_batch_size: 64 | lm_loss: 1.107807E+01 | lr: 7.800E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3509222.000 [ip-26-0-154-121:0]:06/21/2023 17:20:46 [INFO|DP=0|PP=0|TP=0]: iteration: 521 / 150000 | consumed_samples: 33344 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779805E+05 | tokens_per_sec_per_gpu: 9.030945E+03 | global_batch_size: 64 | lm_loss: 1.113223E+01 | lr: 7.815E-05 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 3552363.250 [ip-26-0-154-121:0]:06/21/2023 17:20:47 [INFO|DP=0|PP=0|TP=0]: iteration: 522 / 150000 | consumed_samples: 33408 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788047E+05 | tokens_per_sec_per_gpu: 9.043824E+03 | global_batch_size: 64 | lm_loss: 1.108398E+01 | lr: 7.830E-05 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 3521064.750 [ip-26-0-154-121:0]:06/21/2023 17:20:48 [INFO|DP=0|PP=0|TP=0]: iteration: 523 / 150000 | consumed_samples: 33472 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788704E+05 | tokens_per_sec_per_gpu: 9.044850E+03 | global_batch_size: 64 | lm_loss: 1.108275E+01 | lr: 7.845E-05 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 3548656.750 [ip-26-0-154-121:0]:06/21/2023 17:20:49 [INFO|DP=0|PP=0|TP=0]: iteration: 524 / 150000 | consumed_samples: 33536 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786265E+05 | tokens_per_sec_per_gpu: 9.041040E+03 | global_batch_size: 64 | lm_loss: 1.107209E+01 | lr: 7.860E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3566227.250 [ip-26-0-154-121:0]:06/21/2023 17:20:50 [INFO|DP=0|PP=0|TP=0]: iteration: 525 / 150000 | consumed_samples: 33600 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777415E+05 | tokens_per_sec_per_gpu: 9.027210E+03 | global_batch_size: 64 | lm_loss: 1.111090E+01 | lr: 7.875E-05 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 3628150.000 [ip-26-0-154-121:0]:06/21/2023 17:20:51 [INFO|DP=0|PP=0|TP=0]: iteration: 526 / 150000 | consumed_samples: 33664 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778836E+05 | tokens_per_sec_per_gpu: 9.029431E+03 | global_batch_size: 64 | lm_loss: 1.109342E+01 | lr: 7.890E-05 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3616783.000 [ip-26-0-154-121:0]:06/21/2023 17:20:52 [INFO|DP=0|PP=0|TP=0]: iteration: 527 / 150000 | consumed_samples: 33728 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781675E+05 | tokens_per_sec_per_gpu: 9.033868E+03 | global_batch_size: 64 | lm_loss: 1.109966E+01 | lr: 7.905E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3697477.000 [ip-26-0-154-121:0]:06/21/2023 17:20:52 [INFO|DP=0|PP=0|TP=0]: iteration: 528 / 150000 | consumed_samples: 33792 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781768E+05 | tokens_per_sec_per_gpu: 9.034013E+03 | global_batch_size: 64 | lm_loss: 1.106793E+01 | lr: 7.920E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3700622.000 [ip-26-0-154-121:0]:06/21/2023 17:20:53 [INFO|DP=0|PP=0|TP=0]: iteration: 529 / 150000 | consumed_samples: 33856 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780185E+05 | tokens_per_sec_per_gpu: 9.031538E+03 | global_batch_size: 64 | lm_loss: 1.107123E+01 | lr: 7.935E-05 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 3752058.250 [ip-26-0-154-121:0]:06/21/2023 17:20:54 [INFO|DP=0|PP=0|TP=0]: iteration: 530 / 150000 | consumed_samples: 33920 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787732E+05 | tokens_per_sec_per_gpu: 9.043331E+03 | global_batch_size: 64 | lm_loss: 1.111411E+01 | lr: 7.950E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3733446.500 [ip-26-0-154-121:0]:06/21/2023 17:20:55 [INFO|DP=0|PP=0|TP=0]: iteration: 531 / 150000 | consumed_samples: 33984 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782974E+05 | tokens_per_sec_per_gpu: 9.035897E+03 | global_batch_size: 64 | lm_loss: 1.108722E+01 | lr: 7.965E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3785981.500 [ip-26-0-154-121:0]:06/21/2023 17:20:56 [INFO|DP=0|PP=0|TP=0]: iteration: 532 / 150000 | consumed_samples: 34048 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786451E+05 | tokens_per_sec_per_gpu: 9.041330E+03 | global_batch_size: 64 | lm_loss: 1.105804E+01 | lr: 7.980E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3738523.250 [ip-26-0-154-121:0]:06/21/2023 17:20:57 [INFO|DP=0|PP=0|TP=0]: iteration: 533 / 150000 | consumed_samples: 34112 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783427E+05 | tokens_per_sec_per_gpu: 9.036605E+03 | global_batch_size: 64 | lm_loss: 1.109982E+01 | lr: 7.995E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3738996.500 [ip-26-0-154-121:0]:06/21/2023 17:20:58 [INFO|DP=0|PP=0|TP=0]: iteration: 534 / 150000 | consumed_samples: 34176 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783859E+05 | tokens_per_sec_per_gpu: 9.037280E+03 | global_batch_size: 64 | lm_loss: 1.107212E+01 | lr: 8.010E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3701967.750 [ip-26-0-154-121:0]:06/21/2023 17:20:59 [INFO|DP=0|PP=0|TP=0]: iteration: 535 / 150000 | consumed_samples: 34240 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789984E+05 | tokens_per_sec_per_gpu: 9.046850E+03 | global_batch_size: 64 | lm_loss: 1.103881E+01 | lr: 8.025E-05 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 3695550.250 [ip-26-0-154-121:0]:06/21/2023 17:21:00 [INFO|DP=0|PP=0|TP=0]: iteration: 536 / 150000 | consumed_samples: 34304 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785839E+05 | tokens_per_sec_per_gpu: 9.040373E+03 | global_batch_size: 64 | lm_loss: 1.107540E+01 | lr: 8.040E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3669814.750 [ip-26-0-154-121:0]:06/21/2023 17:21:01 [INFO|DP=0|PP=0|TP=0]: iteration: 537 / 150000 | consumed_samples: 34368 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775436E+05 | tokens_per_sec_per_gpu: 9.024119E+03 | global_batch_size: 64 | lm_loss: 1.107972E+01 | lr: 8.055E-05 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 3685767.750 [ip-26-0-154-121:0]:06/21/2023 17:21:02 [INFO|DP=0|PP=0|TP=0]: iteration: 538 / 150000 | consumed_samples: 34432 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789658E+05 | tokens_per_sec_per_gpu: 9.046341E+03 | global_batch_size: 64 | lm_loss: 1.108093E+01 | lr: 8.070E-05 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 3655360.750 [ip-26-0-154-121:0]:06/21/2023 17:21:02 [INFO|DP=0|PP=0|TP=0]: iteration: 539 / 150000 | consumed_samples: 34496 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787337E+05 | tokens_per_sec_per_gpu: 9.042715E+03 | global_batch_size: 64 | lm_loss: 1.105397E+01 | lr: 8.085E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3635375.000 [ip-26-0-154-121:0]:06/21/2023 17:21:03 [INFO|DP=0|PP=0|TP=0]: iteration: 540 / 150000 | consumed_samples: 34560 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.775786E+05 | tokens_per_sec_per_gpu: 9.024666E+03 | global_batch_size: 64 | lm_loss: 1.076354E+01 | lr: 8.100E-05 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 3673333.750 [ip-26-0-154-121:0]:06/21/2023 17:21:04 [INFO|DP=0|PP=0|TP=0]: iteration: 541 / 150000 | consumed_samples: 34624 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782153E+05 | tokens_per_sec_per_gpu: 9.034614E+03 | global_batch_size: 64 | lm_loss: 1.104841E+01 | lr: 8.115E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3599461.750 [ip-26-0-154-121:0]:06/21/2023 17:21:05 [INFO|DP=0|PP=0|TP=0]: iteration: 542 / 150000 | consumed_samples: 34688 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784634E+05 | tokens_per_sec_per_gpu: 9.038490E+03 | global_batch_size: 64 | lm_loss: 1.115346E+01 | lr: 8.130E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3650157.250 [ip-26-0-154-121:0]:06/21/2023 17:21:06 [INFO|DP=0|PP=0|TP=0]: iteration: 543 / 150000 | consumed_samples: 34752 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.778994E+05 | tokens_per_sec_per_gpu: 9.029678E+03 | global_batch_size: 64 | lm_loss: 1.069722E+01 | lr: 8.145E-05 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3607279.500 [ip-26-0-154-121:0]:06/21/2023 17:21:07 [INFO|DP=0|PP=0|TP=0]: iteration: 544 / 150000 | consumed_samples: 34816 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783042E+05 | tokens_per_sec_per_gpu: 9.036004E+03 | global_batch_size: 64 | lm_loss: 1.107737E+01 | lr: 8.160E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3584099.250 [ip-26-0-154-121:0]:06/21/2023 17:21:08 [INFO|DP=0|PP=0|TP=0]: iteration: 545 / 150000 | consumed_samples: 34880 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780809E+05 | tokens_per_sec_per_gpu: 9.032514E+03 | global_batch_size: 64 | lm_loss: 1.106436E+01 | lr: 8.175E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3573437.000 [ip-26-0-154-121:0]:06/21/2023 17:21:09 [INFO|DP=0|PP=0|TP=0]: iteration: 546 / 150000 | consumed_samples: 34944 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780984E+05 | tokens_per_sec_per_gpu: 9.032787E+03 | global_batch_size: 64 | lm_loss: 1.097960E+01 | lr: 8.190E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3573421.750 [ip-26-0-154-121:0]:06/21/2023 17:21:10 [INFO|DP=0|PP=0|TP=0]: iteration: 547 / 150000 | consumed_samples: 35008 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778060E+05 | tokens_per_sec_per_gpu: 9.028218E+03 | global_batch_size: 64 | lm_loss: 1.102507E+01 | lr: 8.205E-05 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 3565623.750 [ip-26-0-154-121:0]:06/21/2023 17:21:11 [INFO|DP=0|PP=0|TP=0]: iteration: 548 / 150000 | consumed_samples: 35072 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785705E+05 | tokens_per_sec_per_gpu: 9.040164E+03 | global_batch_size: 64 | lm_loss: 1.107471E+01 | lr: 8.220E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3571258.000 [ip-26-0-154-121:0]:06/21/2023 17:21:12 [INFO|DP=0|PP=0|TP=0]: iteration: 549 / 150000 | consumed_samples: 35136 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786873E+05 | tokens_per_sec_per_gpu: 9.041989E+03 | global_batch_size: 64 | lm_loss: 1.107401E+01 | lr: 8.235E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3567670.750 [ip-26-0-154-121:0]:06/21/2023 17:21:12 [INFO|DP=0|PP=0|TP=0]: iteration: 550 / 150000 | consumed_samples: 35200 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784002E+05 | tokens_per_sec_per_gpu: 9.037503E+03 | global_batch_size: 64 | lm_loss: 1.106692E+01 | lr: 8.250E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3592423.750 [ip-26-0-154-121:0]:06/21/2023 17:21:13 [INFO|DP=0|PP=0|TP=0]: iteration: 551 / 150000 | consumed_samples: 35264 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778685E+05 | tokens_per_sec_per_gpu: 9.029196E+03 | global_batch_size: 64 | lm_loss: 1.097740E+01 | lr: 8.265E-05 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3565911.750 [ip-26-0-154-121:0]:06/21/2023 17:21:14 [INFO|DP=0|PP=0|TP=0]: iteration: 552 / 150000 | consumed_samples: 35328 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780748E+05 | tokens_per_sec_per_gpu: 9.032419E+03 | global_batch_size: 64 | lm_loss: 1.102697E+01 | lr: 8.280E-05 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3653400.750 [ip-26-0-154-121:0]:06/21/2023 17:21:15 [INFO|DP=0|PP=0|TP=0]: iteration: 553 / 150000 | consumed_samples: 35392 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778483E+05 | tokens_per_sec_per_gpu: 9.028880E+03 | global_batch_size: 64 | lm_loss: 1.107294E+01 | lr: 8.295E-05 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 3532957.250 [ip-26-0-154-121:0]:06/21/2023 17:21:16 [INFO|DP=0|PP=0|TP=0]: iteration: 554 / 150000 | consumed_samples: 35456 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776524E+05 | tokens_per_sec_per_gpu: 9.025818E+03 | global_batch_size: 64 | lm_loss: 1.096312E+01 | lr: 8.310E-05 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 3616115.250 [ip-26-0-154-121:0]:06/21/2023 17:21:17 [INFO|DP=0|PP=0|TP=0]: iteration: 555 / 150000 | consumed_samples: 35520 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779261E+05 | tokens_per_sec_per_gpu: 9.030095E+03 | global_batch_size: 64 | lm_loss: 1.105420E+01 | lr: 8.325E-05 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 3534997.000 [ip-26-0-154-121:0]:06/21/2023 17:21:18 [INFO|DP=0|PP=0|TP=0]: iteration: 556 / 150000 | consumed_samples: 35584 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777925E+05 | tokens_per_sec_per_gpu: 9.028007E+03 | global_batch_size: 64 | lm_loss: 1.103912E+01 | lr: 8.340E-05 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 3582863.750 [ip-26-0-154-121:0]:06/21/2023 17:21:19 [INFO|DP=0|PP=0|TP=0]: iteration: 557 / 150000 | consumed_samples: 35648 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780779E+05 | tokens_per_sec_per_gpu: 9.032467E+03 | global_batch_size: 64 | lm_loss: 1.079240E+01 | lr: 8.355E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3519004.500 [ip-26-0-154-121:0]:06/21/2023 17:21:20 [INFO|DP=0|PP=0|TP=0]: iteration: 558 / 150000 | consumed_samples: 35712 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784923E+05 | tokens_per_sec_per_gpu: 9.038942E+03 | global_batch_size: 64 | lm_loss: 1.103958E+01 | lr: 8.370E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3557732.500 [ip-26-0-154-121:0]:06/21/2023 17:21:21 [INFO|DP=0|PP=0|TP=0]: iteration: 559 / 150000 | consumed_samples: 35776 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781417E+05 | tokens_per_sec_per_gpu: 9.033464E+03 | global_batch_size: 64 | lm_loss: 1.108321E+01 | lr: 8.385E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3510979.000 [ip-26-0-154-121:0]:06/21/2023 17:21:22 [INFO|DP=0|PP=0|TP=0]: iteration: 560 / 150000 | consumed_samples: 35840 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778670E+05 | tokens_per_sec_per_gpu: 9.029172E+03 | global_batch_size: 64 | lm_loss: 1.106245E+01 | lr: 8.400E-05 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3551228.250 [ip-26-0-154-121:0]:06/21/2023 17:21:22 [INFO|DP=0|PP=0|TP=0]: iteration: 561 / 150000 | consumed_samples: 35904 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775453E+05 | tokens_per_sec_per_gpu: 9.024145E+03 | global_batch_size: 64 | lm_loss: 1.107804E+01 | lr: 8.415E-05 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 3512567.750 [ip-26-0-154-121:0]:06/21/2023 17:21:23 [INFO|DP=0|PP=0|TP=0]: iteration: 562 / 150000 | consumed_samples: 35968 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779428E+05 | tokens_per_sec_per_gpu: 9.030356E+03 | global_batch_size: 64 | lm_loss: 1.106982E+01 | lr: 8.430E-05 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 3556373.750 [ip-26-0-154-121:0]:06/21/2023 17:21:24 [INFO|DP=0|PP=0|TP=0]: iteration: 563 / 150000 | consumed_samples: 36032 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785810E+05 | tokens_per_sec_per_gpu: 9.040328E+03 | global_batch_size: 64 | lm_loss: 1.109247E+01 | lr: 8.445E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3514844.500 [ip-26-0-154-121:0]:06/21/2023 17:21:25 [INFO|DP=0|PP=0|TP=0]: iteration: 564 / 150000 | consumed_samples: 36096 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778755E+05 | tokens_per_sec_per_gpu: 9.029305E+03 | global_batch_size: 64 | lm_loss: 1.109175E+01 | lr: 8.460E-05 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3525605.000 [ip-26-0-154-121:0]:06/21/2023 17:21:26 [INFO|DP=0|PP=0|TP=0]: iteration: 565 / 150000 | consumed_samples: 36160 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775324E+05 | tokens_per_sec_per_gpu: 9.023943E+03 | global_batch_size: 64 | lm_loss: 1.107769E+01 | lr: 8.475E-05 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 3534965.750 [ip-26-0-154-121:0]:06/21/2023 17:21:27 [INFO|DP=0|PP=0|TP=0]: iteration: 566 / 150000 | consumed_samples: 36224 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777679E+05 | tokens_per_sec_per_gpu: 9.027623E+03 | global_batch_size: 64 | lm_loss: 1.111176E+01 | lr: 8.490E-05 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 3560672.750 [ip-26-0-154-121:0]:06/21/2023 17:21:28 [INFO|DP=0|PP=0|TP=0]: iteration: 567 / 150000 | consumed_samples: 36288 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776553E+05 | tokens_per_sec_per_gpu: 9.025863E+03 | global_batch_size: 64 | lm_loss: 1.090502E+01 | lr: 8.505E-05 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 3529395.250 [ip-26-0-154-121:0]:06/21/2023 17:21:29 [INFO|DP=0|PP=0|TP=0]: iteration: 568 / 150000 | consumed_samples: 36352 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777541E+05 | tokens_per_sec_per_gpu: 9.027407E+03 | global_batch_size: 64 | lm_loss: 1.111850E+01 | lr: 8.520E-05 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 3544334.000 [ip-26-0-154-121:0]:06/21/2023 17:21:30 [INFO|DP=0|PP=0|TP=0]: iteration: 569 / 150000 | consumed_samples: 36416 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775136E+05 | tokens_per_sec_per_gpu: 9.023650E+03 | global_batch_size: 64 | lm_loss: 1.112594E+01 | lr: 8.535E-05 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 3532374.250 [ip-26-0-154-121:0]:06/21/2023 17:21:31 [INFO|DP=0|PP=0|TP=0]: iteration: 570 / 150000 | consumed_samples: 36480 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.775692E+05 | tokens_per_sec_per_gpu: 9.024519E+03 | global_batch_size: 64 | lm_loss: 1.108320E+01 | lr: 8.550E-05 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 3550770.750 [ip-26-0-154-121:0]:06/21/2023 17:21:32 [INFO|DP=0|PP=0|TP=0]: iteration: 571 / 150000 | consumed_samples: 36544 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782208E+05 | tokens_per_sec_per_gpu: 9.034699E+03 | global_batch_size: 64 | lm_loss: 1.108412E+01 | lr: 8.565E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3589822.250 [ip-26-0-154-121:0]:06/21/2023 17:21:32 [INFO|DP=0|PP=0|TP=0]: iteration: 572 / 150000 | consumed_samples: 36608 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774625E+05 | tokens_per_sec_per_gpu: 9.022851E+03 | global_batch_size: 64 | lm_loss: 1.105868E+01 | lr: 8.580E-05 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 3624626.500 [ip-26-0-154-121:0]:06/21/2023 17:21:33 [INFO|DP=0|PP=0|TP=0]: iteration: 573 / 150000 | consumed_samples: 36672 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783508E+05 | tokens_per_sec_per_gpu: 9.036731E+03 | global_batch_size: 64 | lm_loss: 1.110272E+01 | lr: 8.595E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3609383.250 [ip-26-0-154-121:0]:06/21/2023 17:21:34 [INFO|DP=0|PP=0|TP=0]: iteration: 574 / 150000 | consumed_samples: 36736 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783486E+05 | tokens_per_sec_per_gpu: 9.036698E+03 | global_batch_size: 64 | lm_loss: 1.110076E+01 | lr: 8.610E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3612613.000 [ip-26-0-154-121:0]:06/21/2023 17:21:35 [INFO|DP=0|PP=0|TP=0]: iteration: 575 / 150000 | consumed_samples: 36800 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774431E+05 | tokens_per_sec_per_gpu: 9.022548E+03 | global_batch_size: 64 | lm_loss: 1.109857E+01 | lr: 8.625E-05 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 3625147.750 [ip-26-0-154-121:0]:06/21/2023 17:21:36 [INFO|DP=0|PP=0|TP=0]: iteration: 576 / 150000 | consumed_samples: 36864 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776542E+05 | tokens_per_sec_per_gpu: 9.025847E+03 | global_batch_size: 64 | lm_loss: 1.110097E+01 | lr: 8.640E-05 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 3614908.750 [ip-26-0-154-121:0]:06/21/2023 17:21:37 [INFO|DP=0|PP=0|TP=0]: iteration: 577 / 150000 | consumed_samples: 36928 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779130E+05 | tokens_per_sec_per_gpu: 9.029891E+03 | global_batch_size: 64 | lm_loss: 1.105712E+01 | lr: 8.655E-05 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 3711663.750 [ip-26-0-154-121:0]:06/21/2023 17:21:38 [INFO|DP=0|PP=0|TP=0]: iteration: 578 / 150000 | consumed_samples: 36992 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777518E+05 | tokens_per_sec_per_gpu: 9.027372E+03 | global_batch_size: 64 | lm_loss: 1.107661E+01 | lr: 8.670E-05 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 3654108.750 [ip-26-0-154-121:0]:06/21/2023 17:21:39 [INFO|DP=0|PP=0|TP=0]: iteration: 579 / 150000 | consumed_samples: 37056 | elapsed_time_per_iteration_ms: 908.3 | tokens_per_sec: 5.772102E+05 | tokens_per_sec_per_gpu: 9.018910E+03 | global_batch_size: 64 | lm_loss: 1.106706E+01 | lr: 8.685E-05 | model_tflops_per_gpu: 104.18 | hardware_tflops_per_gpu: 104.18 | grad_norm: 3595210.250 [ip-26-0-154-121:0]:06/21/2023 17:21:40 [INFO|DP=0|PP=0|TP=0]: iteration: 580 / 150000 | consumed_samples: 37120 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780195E+05 | tokens_per_sec_per_gpu: 9.031555E+03 | global_batch_size: 64 | lm_loss: 1.109125E+01 | lr: 8.700E-05 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 3598883.250 [ip-26-0-154-121:0]:06/21/2023 17:21:41 [INFO|DP=0|PP=0|TP=0]: iteration: 581 / 150000 | consumed_samples: 37184 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.774227E+05 | tokens_per_sec_per_gpu: 9.022230E+03 | global_batch_size: 64 | lm_loss: 1.111784E+01 | lr: 8.715E-05 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 3589061.750 [ip-26-0-154-121:0]:06/21/2023 17:21:42 [INFO|DP=0|PP=0|TP=0]: iteration: 582 / 150000 | consumed_samples: 37248 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779791E+05 | tokens_per_sec_per_gpu: 9.030924E+03 | global_batch_size: 64 | lm_loss: 1.103015E+01 | lr: 8.730E-05 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 3592627.500 [ip-26-0-154-121:0]:06/21/2023 17:21:42 [INFO|DP=0|PP=0|TP=0]: iteration: 583 / 150000 | consumed_samples: 37312 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778614E+05 | tokens_per_sec_per_gpu: 9.029084E+03 | global_batch_size: 64 | lm_loss: 1.109442E+01 | lr: 8.745E-05 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3562752.750 [ip-26-0-154-121:0]:06/21/2023 17:21:43 [INFO|DP=0|PP=0|TP=0]: iteration: 584 / 150000 | consumed_samples: 37376 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781511E+05 | tokens_per_sec_per_gpu: 9.033611E+03 | global_batch_size: 64 | lm_loss: 1.107740E+01 | lr: 8.760E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3545707.750 [ip-26-0-154-121:0]:06/21/2023 17:21:44 [INFO|DP=0|PP=0|TP=0]: iteration: 585 / 150000 | consumed_samples: 37440 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777928E+05 | tokens_per_sec_per_gpu: 9.028012E+03 | global_batch_size: 64 | lm_loss: 1.111386E+01 | lr: 8.775E-05 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 3535753.250 [ip-26-0-154-121:0]:06/21/2023 17:21:45 [INFO|DP=0|PP=0|TP=0]: iteration: 586 / 150000 | consumed_samples: 37504 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776877E+05 | tokens_per_sec_per_gpu: 9.026371E+03 | global_batch_size: 64 | lm_loss: 1.107577E+01 | lr: 8.790E-05 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 3543712.000 [ip-26-0-154-121:0]:06/21/2023 17:21:46 [INFO|DP=0|PP=0|TP=0]: iteration: 587 / 150000 | consumed_samples: 37568 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777428E+05 | tokens_per_sec_per_gpu: 9.027232E+03 | global_batch_size: 64 | lm_loss: 1.109133E+01 | lr: 8.805E-05 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 3529835.750 [ip-26-0-154-121:0]:06/21/2023 17:21:47 [INFO|DP=0|PP=0|TP=0]: iteration: 588 / 150000 | consumed_samples: 37632 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.774288E+05 | tokens_per_sec_per_gpu: 9.022325E+03 | global_batch_size: 64 | lm_loss: 1.110733E+01 | lr: 8.820E-05 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 3537814.250 [ip-26-0-154-121:0]:06/21/2023 17:21:48 [INFO|DP=0|PP=0|TP=0]: iteration: 589 / 150000 | consumed_samples: 37696 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777345E+05 | tokens_per_sec_per_gpu: 9.027101E+03 | global_batch_size: 64 | lm_loss: 1.107494E+01 | lr: 8.835E-05 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 3644242.500 [ip-26-0-154-121:0]:06/21/2023 17:21:49 [INFO|DP=0|PP=0|TP=0]: iteration: 590 / 150000 | consumed_samples: 37760 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786221E+05 | tokens_per_sec_per_gpu: 9.040971E+03 | global_batch_size: 64 | lm_loss: 1.107202E+01 | lr: 8.850E-05 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3599440.250 [ip-26-0-154-121:0]:06/21/2023 17:21:50 [INFO|DP=0|PP=0|TP=0]: iteration: 591 / 150000 | consumed_samples: 37824 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779449E+05 | tokens_per_sec_per_gpu: 9.030390E+03 | global_batch_size: 64 | lm_loss: 1.107368E+01 | lr: 8.865E-05 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 3524587.000 [ip-26-0-154-121:0]:06/21/2023 17:21:51 [INFO|DP=0|PP=0|TP=0]: iteration: 592 / 150000 | consumed_samples: 37888 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.773863E+05 | tokens_per_sec_per_gpu: 9.021662E+03 | global_batch_size: 64 | lm_loss: 1.106677E+01 | lr: 8.880E-05 | model_tflops_per_gpu: 104.21 | hardware_tflops_per_gpu: 104.21 | grad_norm: 3535144.000 [ip-26-0-154-121:0]:06/21/2023 17:21:52 [INFO|DP=0|PP=0|TP=0]: iteration: 593 / 150000 | consumed_samples: 37952 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779519E+05 | tokens_per_sec_per_gpu: 9.030499E+03 | global_batch_size: 64 | lm_loss: 1.108866E+01 | lr: 8.895E-05 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 3519379.500 [ip-26-0-154-121:0]:06/21/2023 17:21:52 [INFO|DP=0|PP=0|TP=0]: iteration: 594 / 150000 | consumed_samples: 38016 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780812E+05 | tokens_per_sec_per_gpu: 9.032519E+03 | global_batch_size: 64 | lm_loss: 1.110180E+01 | lr: 8.910E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3528682.250 [ip-26-0-154-121:0]:06/21/2023 17:21:53 [INFO|DP=0|PP=0|TP=0]: iteration: 595 / 150000 | consumed_samples: 38080 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782220E+05 | tokens_per_sec_per_gpu: 9.034718E+03 | global_batch_size: 64 | lm_loss: 1.112619E+01 | lr: 8.925E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3559430.000 [ip-26-0-154-121:0]:06/21/2023 17:21:54 [INFO|DP=0|PP=0|TP=0]: iteration: 596 / 150000 | consumed_samples: 38144 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781876E+05 | tokens_per_sec_per_gpu: 9.034181E+03 | global_batch_size: 64 | lm_loss: 1.108679E+01 | lr: 8.940E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3518065.250 [ip-26-0-154-121:0]:06/21/2023 17:21:55 [INFO|DP=0|PP=0|TP=0]: iteration: 597 / 150000 | consumed_samples: 38208 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780270E+05 | tokens_per_sec_per_gpu: 9.031671E+03 | global_batch_size: 64 | lm_loss: 1.106461E+01 | lr: 8.955E-05 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3596020.500 [ip-26-0-154-121:0]:06/21/2023 17:21:56 [INFO|DP=0|PP=0|TP=0]: iteration: 598 / 150000 | consumed_samples: 38272 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781943E+05 | tokens_per_sec_per_gpu: 9.034286E+03 | global_batch_size: 64 | lm_loss: 1.106026E+01 | lr: 8.970E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3540419.500 [ip-26-0-154-121:0]:06/21/2023 17:21:57 [INFO|DP=0|PP=0|TP=0]: iteration: 599 / 150000 | consumed_samples: 38336 | elapsed_time_per_iteration_ms: 912.2 | tokens_per_sec: 5.747641E+05 | tokens_per_sec_per_gpu: 8.980689E+03 | global_batch_size: 64 | lm_loss: 1.106584E+01 | lr: 8.985E-05 | model_tflops_per_gpu: 103.74 | hardware_tflops_per_gpu: 103.74 | grad_norm: 3549695.500 [ip-26-0-154-121:0]:06/21/2023 17:21:58 [INFO|DP=0|PP=0|TP=0]: iteration: 600 / 150000 | consumed_samples: 38400 | elapsed_time_per_iteration_ms: 917.5 | tokens_per_sec: 5.714071E+05 | tokens_per_sec_per_gpu: 8.928237E+03 | global_batch_size: 64 | lm_loss: 1.109719E+01 | lr: 9.000E-05 | model_tflops_per_gpu: 103.13 | hardware_tflops_per_gpu: 103.13 | grad_norm: 3555801.000 [ip-26-0-154-121:0]:06/21/2023 17:21:59 [INFO|DP=0|PP=0|TP=0]: iteration: 601 / 150000 | consumed_samples: 38464 | elapsed_time_per_iteration_ms: 914.2 | tokens_per_sec: 5.734765E+05 | tokens_per_sec_per_gpu: 8.960571E+03 | global_batch_size: 64 | lm_loss: 1.111749E+01 | lr: 9.015E-05 | model_tflops_per_gpu: 103.50 | hardware_tflops_per_gpu: 103.50 | grad_norm: 3506789.750 [ip-26-0-154-121:0]:06/21/2023 17:22:00 [INFO|DP=0|PP=0|TP=0]: iteration: 602 / 150000 | consumed_samples: 38528 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779502E+05 | tokens_per_sec_per_gpu: 9.030473E+03 | global_batch_size: 64 | lm_loss: 1.108887E+01 | lr: 9.030E-05 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 3516512.500 [ip-26-0-154-121:0]:06/21/2023 17:22:01 [INFO|DP=0|PP=0|TP=0]: iteration: 603 / 150000 | consumed_samples: 38592 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781651E+05 | tokens_per_sec_per_gpu: 9.033830E+03 | global_batch_size: 64 | lm_loss: 1.109511E+01 | lr: 9.045E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3515370.750 [ip-26-0-154-121:0]:06/21/2023 17:22:02 [INFO|DP=0|PP=0|TP=0]: iteration: 604 / 150000 | consumed_samples: 38656 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784714E+05 | tokens_per_sec_per_gpu: 9.038616E+03 | global_batch_size: 64 | lm_loss: 1.108645E+01 | lr: 9.060E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3546518.750 [ip-26-0-154-121:0]:06/21/2023 17:22:02 [INFO|DP=0|PP=0|TP=0]: iteration: 605 / 150000 | consumed_samples: 38720 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777205E+05 | tokens_per_sec_per_gpu: 9.026883E+03 | global_batch_size: 64 | lm_loss: 1.108350E+01 | lr: 9.075E-05 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 3565730.250 [ip-26-0-154-121:0]:06/21/2023 17:22:03 [INFO|DP=0|PP=0|TP=0]: iteration: 606 / 150000 | consumed_samples: 38784 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779951E+05 | tokens_per_sec_per_gpu: 9.031173E+03 | global_batch_size: 64 | lm_loss: 1.110876E+01 | lr: 9.090E-05 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 3488669.750 [ip-26-0-154-121:0]:06/21/2023 17:22:04 [INFO|DP=0|PP=0|TP=0]: iteration: 607 / 150000 | consumed_samples: 38848 | elapsed_time_per_iteration_ms: 920.3 | tokens_per_sec: 5.696993E+05 | tokens_per_sec_per_gpu: 8.901551E+03 | global_batch_size: 64 | lm_loss: 1.113654E+01 | lr: 9.105E-05 | model_tflops_per_gpu: 102.82 | hardware_tflops_per_gpu: 102.82 | grad_norm: 3514584.250 [ip-26-0-154-121:0]:06/21/2023 17:22:05 [INFO|DP=0|PP=0|TP=0]: iteration: 608 / 150000 | consumed_samples: 38912 | elapsed_time_per_iteration_ms: 918.6 | tokens_per_sec: 5.707257E+05 | tokens_per_sec_per_gpu: 8.917589E+03 | global_batch_size: 64 | lm_loss: 1.108390E+01 | lr: 9.120E-05 | model_tflops_per_gpu: 103.01 | hardware_tflops_per_gpu: 103.01 | grad_norm: 3519884.500 [ip-26-0-154-121:0]:06/21/2023 17:22:06 [INFO|DP=0|PP=0|TP=0]: iteration: 609 / 150000 | consumed_samples: 38976 | elapsed_time_per_iteration_ms: 904.3 | tokens_per_sec: 5.797546E+05 | tokens_per_sec_per_gpu: 9.058666E+03 | global_batch_size: 64 | lm_loss: 1.108333E+01 | lr: 9.135E-05 | model_tflops_per_gpu: 104.64 | hardware_tflops_per_gpu: 104.64 | grad_norm: 3493914.000 [ip-26-0-154-121:0]:06/21/2023 17:22:07 [INFO|DP=0|PP=0|TP=0]: iteration: 610 / 150000 | consumed_samples: 39040 | elapsed_time_per_iteration_ms: 911.0 | tokens_per_sec: 5.755186E+05 | tokens_per_sec_per_gpu: 8.992478E+03 | global_batch_size: 64 | lm_loss: 1.109016E+01 | lr: 9.150E-05 | model_tflops_per_gpu: 103.87 | hardware_tflops_per_gpu: 103.87 | grad_norm: 3510714.250 [ip-26-0-154-121:0]:06/21/2023 17:22:08 [INFO|DP=0|PP=0|TP=0]: iteration: 611 / 150000 | consumed_samples: 39104 | elapsed_time_per_iteration_ms: 913.2 | tokens_per_sec: 5.741011E+05 | tokens_per_sec_per_gpu: 8.970330E+03 | global_batch_size: 64 | lm_loss: 1.109237E+01 | lr: 9.165E-05 | model_tflops_per_gpu: 103.62 | hardware_tflops_per_gpu: 103.62 | grad_norm: 3632360.000 [ip-26-0-154-121:0]:06/21/2023 17:22:09 [INFO|DP=0|PP=0|TP=0]: iteration: 612 / 150000 | consumed_samples: 39168 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777307E+05 | tokens_per_sec_per_gpu: 9.027042E+03 | global_batch_size: 64 | lm_loss: 1.108479E+01 | lr: 9.180E-05 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 3661247.000 [ip-26-0-154-121:0]:06/21/2023 17:22:10 [INFO|DP=0|PP=0|TP=0]: iteration: 613 / 150000 | consumed_samples: 39232 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780533E+05 | tokens_per_sec_per_gpu: 9.032082E+03 | global_batch_size: 64 | lm_loss: 1.109965E+01 | lr: 9.195E-05 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3549353.750 [ip-26-0-154-121:0]:06/21/2023 17:22:11 [INFO|DP=0|PP=0|TP=0]: iteration: 614 / 150000 | consumed_samples: 39296 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782747E+05 | tokens_per_sec_per_gpu: 9.035543E+03 | global_batch_size: 64 | lm_loss: 1.108619E+01 | lr: 9.210E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3617576.000 [ip-26-0-154-121:0]:06/21/2023 17:22:12 [INFO|DP=0|PP=0|TP=0]: iteration: 615 / 150000 | consumed_samples: 39360 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782781E+05 | tokens_per_sec_per_gpu: 9.035595E+03 | global_batch_size: 64 | lm_loss: 1.110246E+01 | lr: 9.225E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3567470.750 [ip-26-0-154-121:0]:06/21/2023 17:22:12 [INFO|DP=0|PP=0|TP=0]: iteration: 616 / 150000 | consumed_samples: 39424 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784116E+05 | tokens_per_sec_per_gpu: 9.037682E+03 | global_batch_size: 64 | lm_loss: 1.109771E+01 | lr: 9.240E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3576363.750 [ip-26-0-154-121:0]:06/21/2023 17:22:13 [INFO|DP=0|PP=0|TP=0]: iteration: 617 / 150000 | consumed_samples: 39488 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784892E+05 | tokens_per_sec_per_gpu: 9.038894E+03 | global_batch_size: 64 | lm_loss: 1.108693E+01 | lr: 9.255E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3607688.250 [ip-26-0-154-121:0]:06/21/2023 17:22:14 [INFO|DP=0|PP=0|TP=0]: iteration: 618 / 150000 | consumed_samples: 39552 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787276E+05 | tokens_per_sec_per_gpu: 9.042619E+03 | global_batch_size: 64 | lm_loss: 1.107385E+01 | lr: 9.270E-05 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3601065.250 [ip-26-0-154-121:0]:06/21/2023 17:22:15 [INFO|DP=0|PP=0|TP=0]: iteration: 619 / 150000 | consumed_samples: 39616 | elapsed_time_per_iteration_ms: 911.0 | tokens_per_sec: 5.755258E+05 | tokens_per_sec_per_gpu: 8.992591E+03 | global_batch_size: 64 | lm_loss: 1.108222E+01 | lr: 9.285E-05 | model_tflops_per_gpu: 103.87 | hardware_tflops_per_gpu: 103.87 | grad_norm: 3610490.500 [ip-26-0-154-121:0]:06/21/2023 17:22:16 [INFO|DP=0|PP=0|TP=0]: iteration: 620 / 150000 | consumed_samples: 39680 | elapsed_time_per_iteration_ms: 908.4 | tokens_per_sec: 5.771799E+05 | tokens_per_sec_per_gpu: 9.018437E+03 | global_batch_size: 64 | lm_loss: 1.110232E+01 | lr: 9.300E-05 | model_tflops_per_gpu: 104.17 | hardware_tflops_per_gpu: 104.17 | grad_norm: 3658185.250 [ip-26-0-154-121:0]:06/21/2023 17:22:17 [INFO|DP=0|PP=0|TP=0]: iteration: 621 / 150000 | consumed_samples: 39744 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782866E+05 | tokens_per_sec_per_gpu: 9.035728E+03 | global_batch_size: 64 | lm_loss: 1.108944E+01 | lr: 9.315E-05 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3662555.250 [ip-26-0-154-121:0]:06/21/2023 17:22:18 [INFO|DP=0|PP=0|TP=0]: iteration: 622 / 150000 | consumed_samples: 39808 | elapsed_time_per_iteration_ms: 925.2 | tokens_per_sec: 5.666499E+05 | tokens_per_sec_per_gpu: 8.853905E+03 | global_batch_size: 64 | lm_loss: 1.111703E+01 | lr: 9.330E-05 | model_tflops_per_gpu: 102.27 | hardware_tflops_per_gpu: 102.27 | grad_norm: 3606191.000 [ip-26-0-154-121:0]:06/21/2023 17:22:19 [INFO|DP=0|PP=0|TP=0]: iteration: 623 / 150000 | consumed_samples: 39872 | elapsed_time_per_iteration_ms: 916.7 | tokens_per_sec: 5.719334E+05 | tokens_per_sec_per_gpu: 8.936459E+03 | global_batch_size: 64 | lm_loss: 1.108616E+01 | lr: 9.345E-05 | model_tflops_per_gpu: 103.23 | hardware_tflops_per_gpu: 103.23 | grad_norm: 3636357.000 [ip-26-0-154-121:0]:06/21/2023 17:22:20 [INFO|DP=0|PP=0|TP=0]: iteration: 624 / 150000 | consumed_samples: 39936 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781052E+05 | tokens_per_sec_per_gpu: 9.032894E+03 | global_batch_size: 64 | lm_loss: 1.107927E+01 | lr: 9.360E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3600432.750 [ip-26-0-154-121:0]:06/21/2023 17:22:21 [INFO|DP=0|PP=0|TP=0]: iteration: 625 / 150000 | consumed_samples: 40000 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787439E+05 | tokens_per_sec_per_gpu: 9.042874E+03 | global_batch_size: 64 | lm_loss: 1.108906E+01 | lr: 9.375E-05 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3602223.000 [ip-26-0-154-121:0]:06/21/2023 17:22:22 [INFO|DP=0|PP=0|TP=0]: iteration: 626 / 150000 | consumed_samples: 40064 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782373E+05 | tokens_per_sec_per_gpu: 9.034958E+03 | global_batch_size: 64 | lm_loss: 1.107282E+01 | lr: 9.390E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3614254.500 [ip-26-0-154-121:0]:06/21/2023 17:22:23 [INFO|DP=0|PP=0|TP=0]: iteration: 627 / 150000 | consumed_samples: 40128 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785197E+05 | tokens_per_sec_per_gpu: 9.039370E+03 | global_batch_size: 64 | lm_loss: 1.111668E+01 | lr: 9.405E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3540991.750 [ip-26-0-154-121:0]:06/21/2023 17:22:23 [INFO|DP=0|PP=0|TP=0]: iteration: 628 / 150000 | consumed_samples: 40192 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777630E+05 | tokens_per_sec_per_gpu: 9.027547E+03 | global_batch_size: 64 | lm_loss: 1.110419E+01 | lr: 9.420E-05 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 3693948.000 [ip-26-0-154-121:0]:06/21/2023 17:22:24 [INFO|DP=0|PP=0|TP=0]: iteration: 629 / 150000 | consumed_samples: 40256 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777916E+05 | tokens_per_sec_per_gpu: 9.027993E+03 | global_batch_size: 64 | lm_loss: 1.110279E+01 | lr: 9.435E-05 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 3616216.750 [ip-26-0-154-121:0]:06/21/2023 17:22:25 [INFO|DP=0|PP=0|TP=0]: iteration: 630 / 150000 | consumed_samples: 40320 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783398E+05 | tokens_per_sec_per_gpu: 9.036560E+03 | global_batch_size: 64 | lm_loss: 1.091060E+01 | lr: 9.450E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3620037.000 [ip-26-0-154-121:0]:06/21/2023 17:22:26 [INFO|DP=0|PP=0|TP=0]: iteration: 631 / 150000 | consumed_samples: 40384 | elapsed_time_per_iteration_ms: 916.6 | tokens_per_sec: 5.719628E+05 | tokens_per_sec_per_gpu: 8.936919E+03 | global_batch_size: 64 | lm_loss: 1.109251E+01 | lr: 9.465E-05 | model_tflops_per_gpu: 103.23 | hardware_tflops_per_gpu: 103.23 | grad_norm: 3666388.000 [ip-26-0-154-121:0]:06/21/2023 17:22:27 [INFO|DP=0|PP=0|TP=0]: iteration: 632 / 150000 | consumed_samples: 40448 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781753E+05 | tokens_per_sec_per_gpu: 9.033989E+03 | global_batch_size: 64 | lm_loss: 1.104989E+01 | lr: 9.480E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3584765.250 [ip-26-0-154-121:0]:06/21/2023 17:22:28 [INFO|DP=0|PP=0|TP=0]: iteration: 633 / 150000 | consumed_samples: 40512 | elapsed_time_per_iteration_ms: 910.3 | tokens_per_sec: 5.759627E+05 | tokens_per_sec_per_gpu: 8.999417E+03 | global_batch_size: 64 | lm_loss: 1.108565E+01 | lr: 9.495E-05 | model_tflops_per_gpu: 103.95 | hardware_tflops_per_gpu: 103.95 | grad_norm: 3553874.500 [ip-26-0-154-121:0]:06/21/2023 17:22:29 [INFO|DP=0|PP=0|TP=0]: iteration: 634 / 150000 | consumed_samples: 40576 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779881E+05 | tokens_per_sec_per_gpu: 9.031064E+03 | global_batch_size: 64 | lm_loss: 1.109985E+01 | lr: 9.510E-05 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 3574942.000 [ip-26-0-154-121:0]:06/21/2023 17:22:30 [INFO|DP=0|PP=0|TP=0]: iteration: 635 / 150000 | consumed_samples: 40640 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783829E+05 | tokens_per_sec_per_gpu: 9.037232E+03 | global_batch_size: 64 | lm_loss: 1.108382E+01 | lr: 9.525E-05 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3514928.000 [ip-26-0-154-121:0]:06/21/2023 17:22:31 [INFO|DP=0|PP=0|TP=0]: iteration: 636 / 150000 | consumed_samples: 40704 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.782042E+05 | tokens_per_sec_per_gpu: 9.034440E+03 | global_batch_size: 64 | lm_loss: 1.109305E+01 | lr: 9.540E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3518143.250 [ip-26-0-154-121:0]:06/21/2023 17:22:32 [INFO|DP=0|PP=0|TP=0]: iteration: 637 / 150000 | consumed_samples: 40768 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782198E+05 | tokens_per_sec_per_gpu: 9.034685E+03 | global_batch_size: 64 | lm_loss: 1.107097E+01 | lr: 9.555E-05 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3539095.750 [ip-26-0-154-121:0]:06/21/2023 17:22:33 [INFO|DP=0|PP=0|TP=0]: iteration: 638 / 150000 | consumed_samples: 40832 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785571E+05 | tokens_per_sec_per_gpu: 9.039955E+03 | global_batch_size: 64 | lm_loss: 1.107558E+01 | lr: 9.570E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3530357.250 [ip-26-0-154-121:0]:06/21/2023 17:22:33 [INFO|DP=0|PP=0|TP=0]: iteration: 639 / 150000 | consumed_samples: 40896 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780794E+05 | tokens_per_sec_per_gpu: 9.032490E+03 | global_batch_size: 64 | lm_loss: 1.107858E+01 | lr: 9.585E-05 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3527339.500 [ip-26-0-154-121:0]:06/21/2023 17:22:34 [INFO|DP=0|PP=0|TP=0]: iteration: 640 / 150000 | consumed_samples: 40960 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784618E+05 | tokens_per_sec_per_gpu: 9.038466E+03 | global_batch_size: 64 | lm_loss: 1.110132E+01 | lr: 9.600E-05 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3538478.750 [ip-26-0-154-121:0]:06/21/2023 17:22:35 [INFO|DP=0|PP=0|TP=0]: iteration: 641 / 150000 | consumed_samples: 41024 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780311E+05 | tokens_per_sec_per_gpu: 9.031735E+03 | global_batch_size: 64 | lm_loss: 1.107809E+01 | lr: 9.615E-05 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3523026.750 [ip-26-0-154-121:0]:06/21/2023 17:22:36 [INFO|DP=0|PP=0|TP=0]: iteration: 642 / 150000 | consumed_samples: 41088 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778708E+05 | tokens_per_sec_per_gpu: 9.029232E+03 | global_batch_size: 64 | lm_loss: 1.104985E+01 | lr: 9.630E-05 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3649695.500 [ip-26-0-154-121:0]:06/21/2023 17:22:37 [INFO|DP=0|PP=0|TP=0]: iteration: 643 / 150000 | consumed_samples: 41152 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780122E+05 | tokens_per_sec_per_gpu: 9.031441E+03 | global_batch_size: 64 | lm_loss: 1.103742E+01 | lr: 9.645E-05 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 3554099.750 [ip-26-0-154-121:0]:06/21/2023 17:22:38 [INFO|DP=0|PP=0|TP=0]: iteration: 644 / 150000 | consumed_samples: 41216 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777504E+05 | tokens_per_sec_per_gpu: 9.027350E+03 | global_batch_size: 64 | lm_loss: 1.094028E+01 | lr: 9.660E-05 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 3582766.250 [ip-26-0-154-121:0]:06/21/2023 17:22:39 [INFO|DP=0|PP=0|TP=0]: iteration: 645 / 150000 | consumed_samples: 41280 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.776962E+05 | tokens_per_sec_per_gpu: 9.026504E+03 | global_batch_size: 64 | lm_loss: 1.110799E+01 | lr: 9.675E-05 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 3576611.500 [ip-26-0-154-121:0]:06/21/2023 17:22:40 [INFO|DP=0|PP=0|TP=0]: iteration: 646 / 150000 | consumed_samples: 41344 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783383E+05 | tokens_per_sec_per_gpu: 9.036536E+03 | global_batch_size: 64 | lm_loss: 1.099893E+01 | lr: 9.690E-05 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3573269.750 [ip-26-0-154-121:0]:06/21/2023 17:22:41 [INFO|DP=0|PP=0|TP=0]: iteration: 647 / 150000 | consumed_samples: 41408 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780505E+05 | tokens_per_sec_per_gpu: 9.032039E+03 | global_batch_size: 64 | lm_loss: 1.109057E+01 | lr: 9.705E-05 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3557030.000 [ip-26-0-154-121:0]:06/21/2023 17:22:42 [INFO|DP=0|PP=0|TP=0]: iteration: 648 / 150000 | consumed_samples: 41472 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777478E+05 | tokens_per_sec_per_gpu: 9.027310E+03 | global_batch_size: 64 | lm_loss: 1.109124E+01 | lr: 9.720E-05 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 3556211.000 [ip-26-0-154-121:0]:06/21/2023 17:22:42 [INFO|DP=0|PP=0|TP=0]: iteration: 649 / 150000 | consumed_samples: 41536 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780131E+05 | tokens_per_sec_per_gpu: 9.031455E+03 | global_batch_size: 64 | lm_loss: 1.108321E+01 | lr: 9.735E-05 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 3617594.750 [ip-26-0-154-121:0]:06/21/2023 17:22:43 [INFO|DP=0|PP=0|TP=0]: iteration: 650 / 150000 | consumed_samples: 41600 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780403E+05 | tokens_per_sec_per_gpu: 9.031880E+03 | global_batch_size: 64 | lm_loss: 1.109142E+01 | lr: 9.750E-05 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3554052.250 [ip-26-0-154-121:0]:06/21/2023 17:22:44 [INFO|DP=0|PP=0|TP=0]: iteration: 651 / 150000 | consumed_samples: 41664 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777172E+05 | tokens_per_sec_per_gpu: 9.026831E+03 | global_batch_size: 64 | lm_loss: 1.095473E+01 | lr: 9.765E-05 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 3580531.500 [ip-26-0-154-121:0]:06/21/2023 17:22:45 [INFO|DP=0|PP=0|TP=0]: iteration: 652 / 150000 | consumed_samples: 41728 | elapsed_time_per_iteration_ms: 917.9 | tokens_per_sec: 5.711599E+05 | tokens_per_sec_per_gpu: 8.924373E+03 | global_batch_size: 64 | lm_loss: 1.109680E+01 | lr: 9.780E-05 | model_tflops_per_gpu: 103.09 | hardware_tflops_per_gpu: 103.09 | grad_norm: 3619483.000 [ip-26-0-154-121:0]:06/21/2023 17:22:46 [INFO|DP=0|PP=0|TP=0]: iteration: 653 / 150000 | consumed_samples: 41792 | elapsed_time_per_iteration_ms: 908.1 | tokens_per_sec: 5.773566E+05 | tokens_per_sec_per_gpu: 9.021197E+03 | global_batch_size: 64 | lm_loss: 1.105252E+01 | lr: 9.795E-05 | model_tflops_per_gpu: 104.20 | hardware_tflops_per_gpu: 104.20 | grad_norm: 3594834.250 [ip-26-0-154-121:0]:06/21/2023 17:22:47 [INFO|DP=0|PP=0|TP=0]: iteration: 654 / 150000 | consumed_samples: 41856 | elapsed_time_per_iteration_ms: 908.1 | tokens_per_sec: 5.773469E+05 | tokens_per_sec_per_gpu: 9.021046E+03 | global_batch_size: 64 | lm_loss: 1.110745E+01 | lr: 9.810E-05 | model_tflops_per_gpu: 104.20 | hardware_tflops_per_gpu: 104.20 | grad_norm: 3674643.000 [ip-26-0-154-121:0]:06/21/2023 17:22:48 [INFO|DP=0|PP=0|TP=0]: iteration: 655 / 150000 | consumed_samples: 41920 | elapsed_time_per_iteration_ms: 910.4 | tokens_per_sec: 5.759036E+05 | tokens_per_sec_per_gpu: 8.998493E+03 | global_batch_size: 64 | lm_loss: 1.109351E+01 | lr: 9.825E-05 | model_tflops_per_gpu: 103.94 | hardware_tflops_per_gpu: 103.94 | grad_norm: 3605420.000 [ip-26-0-154-121:0]:06/21/2023 17:22:49 [INFO|DP=0|PP=0|TP=0]: iteration: 656 / 150000 | consumed_samples: 41984 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784935E+05 | tokens_per_sec_per_gpu: 9.038961E+03 | global_batch_size: 64 | lm_loss: 1.112818E+01 | lr: 9.840E-05 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3667794.000 [ip-26-0-154-121:0]:06/21/2023 17:22:50 [INFO|DP=0|PP=0|TP=0]: iteration: 657 / 150000 | consumed_samples: 42048 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.774300E+05 | tokens_per_sec_per_gpu: 9.022344E+03 | global_batch_size: 64 | lm_loss: 1.106839E+01 | lr: 9.855E-05 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 3681895.750 [ip-26-0-154-121:0]:06/21/2023 17:22:51 [INFO|DP=0|PP=0|TP=0]: iteration: 658 / 150000 | consumed_samples: 42112 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785332E+05 | tokens_per_sec_per_gpu: 9.039581E+03 | global_batch_size: 64 | lm_loss: 1.110120E+01 | lr: 9.870E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3692331.250 [ip-26-0-154-121:0]:06/21/2023 17:22:52 [INFO|DP=0|PP=0|TP=0]: iteration: 659 / 150000 | consumed_samples: 42176 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786800E+05 | tokens_per_sec_per_gpu: 9.041875E+03 | global_batch_size: 64 | lm_loss: 1.098596E+01 | lr: 9.885E-05 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3678858.000 [ip-26-0-154-121:0]:06/21/2023 17:22:53 [INFO|DP=0|PP=0|TP=0]: iteration: 660 / 150000 | consumed_samples: 42240 | elapsed_time_per_iteration_ms: 910.5 | tokens_per_sec: 5.758333E+05 | tokens_per_sec_per_gpu: 8.997395E+03 | global_batch_size: 64 | lm_loss: 1.110950E+01 | lr: 9.900E-05 | model_tflops_per_gpu: 103.93 | hardware_tflops_per_gpu: 103.93 | grad_norm: 3656000.750 [ip-26-0-154-121:0]:06/21/2023 17:22:53 [INFO|DP=0|PP=0|TP=0]: iteration: 661 / 150000 | consumed_samples: 42304 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781549E+05 | tokens_per_sec_per_gpu: 9.033671E+03 | global_batch_size: 64 | lm_loss: 1.111410E+01 | lr: 9.915E-05 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3610132.250 [ip-26-0-154-121:0]:06/21/2023 17:22:54 [INFO|DP=0|PP=0|TP=0]: iteration: 662 / 150000 | consumed_samples: 42368 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785341E+05 | tokens_per_sec_per_gpu: 9.039596E+03 | global_batch_size: 64 | lm_loss: 1.111196E+01 | lr: 9.930E-05 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3601893.500 [ip-26-0-154-121:0]:06/21/2023 17:22:55 [INFO|DP=0|PP=0|TP=0]: iteration: 663 / 150000 | consumed_samples: 42432 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778731E+05 | tokens_per_sec_per_gpu: 9.029267E+03 | global_batch_size: 64 | lm_loss: 1.111975E+01 | lr: 9.945E-05 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3689909.750 [ip-26-0-154-121:0]:06/21/2023 17:22:57 [INFO|DP=0|PP=0|TP=0]: iteration: 664 / 150000 | consumed_samples: 42496 | elapsed_time_per_iteration_ms: 1818.5 | tokens_per_sec: 2.883081E+05 | tokens_per_sec_per_gpu: 4.504815E+03 | global_batch_size: 64 | lm_loss: 1.108529E+01 | lr: 9.960E-05 | model_tflops_per_gpu: 52.04 | hardware_tflops_per_gpu: 52.04 | grad_norm: 3905128.500 [ip-26-0-154-121:0]:06/21/2023 17:22:58 [INFO|DP=0|PP=0|TP=0]: iteration: 665 / 150000 | consumed_samples: 42560 | elapsed_time_per_iteration_ms: 1023.5 | tokens_per_sec: 5.122684E+05 | tokens_per_sec_per_gpu: 8.004194E+03 | global_batch_size: 64 | lm_loss: 1.109853E+01 | lr: 9.975E-05 | model_tflops_per_gpu: 92.46 | hardware_tflops_per_gpu: 92.46 | grad_norm: 3853215.250 [ip-26-0-154-121:0]:06/21/2023 17:22:59 [INFO|DP=0|PP=0|TP=0]: iteration: 666 / 150000 | consumed_samples: 42624 | elapsed_time_per_iteration_ms: 982.6 | tokens_per_sec: 5.335909E+05 | tokens_per_sec_per_gpu: 8.337358E+03 | global_batch_size: 64 | lm_loss: 1.108358E+01 | lr: 9.990E-05 | model_tflops_per_gpu: 96.31 | hardware_tflops_per_gpu: 96.31 | grad_norm: 3678439.250 [ip-26-0-154-121:0]:06/21/2023 17:23:00 [INFO|DP=0|PP=0|TP=0]: iteration: 667 / 150000 | consumed_samples: 42688 | elapsed_time_per_iteration_ms: 931.0 | tokens_per_sec: 5.631574E+05 | tokens_per_sec_per_gpu: 8.799335E+03 | global_batch_size: 64 | lm_loss: 1.103178E+01 | lr: 1.000E-04 | model_tflops_per_gpu: 101.64 | hardware_tflops_per_gpu: 101.64 | grad_norm: 3580925.000 [ip-26-0-154-121:0]:06/21/2023 17:23:01 [INFO|DP=0|PP=0|TP=0]: iteration: 668 / 150000 | consumed_samples: 42752 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777056E+05 | tokens_per_sec_per_gpu: 9.026651E+03 | global_batch_size: 64 | lm_loss: 1.106989E+01 | lr: 1.002E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 3549337.750 [ip-26-0-154-121:0]:06/21/2023 17:23:02 [INFO|DP=0|PP=0|TP=0]: iteration: 669 / 150000 | consumed_samples: 42816 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781758E+05 | tokens_per_sec_per_gpu: 9.033996E+03 | global_batch_size: 64 | lm_loss: 1.109905E+01 | lr: 1.003E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3578439.500 [ip-26-0-154-121:0]:06/21/2023 17:23:03 [INFO|DP=0|PP=0|TP=0]: iteration: 670 / 150000 | consumed_samples: 42880 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775310E+05 | tokens_per_sec_per_gpu: 9.023922E+03 | global_batch_size: 64 | lm_loss: 1.110056E+01 | lr: 1.005E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 3557226.500 [ip-26-0-154-121:0]:06/21/2023 17:23:04 [INFO|DP=0|PP=0|TP=0]: iteration: 671 / 150000 | consumed_samples: 42944 | elapsed_time_per_iteration_ms: 1147.4 | tokens_per_sec: 4.569180E+05 | tokens_per_sec_per_gpu: 7.139343E+03 | global_batch_size: 64 | lm_loss: 1.110895E+01 | lr: 1.006E-04 | model_tflops_per_gpu: 82.47 | hardware_tflops_per_gpu: 82.47 | grad_norm: 3656438.500 [ip-26-0-154-121:0]:06/21/2023 17:23:05 [INFO|DP=0|PP=0|TP=0]: iteration: 672 / 150000 | consumed_samples: 43008 | elapsed_time_per_iteration_ms: 935.7 | tokens_per_sec: 5.602904E+05 | tokens_per_sec_per_gpu: 8.754537E+03 | global_batch_size: 64 | lm_loss: 1.109327E+01 | lr: 1.008E-04 | model_tflops_per_gpu: 101.12 | hardware_tflops_per_gpu: 101.12 | grad_norm: 3577363.500 [ip-26-0-154-121:0]:06/21/2023 17:23:06 [INFO|DP=0|PP=0|TP=0]: iteration: 673 / 150000 | consumed_samples: 43072 | elapsed_time_per_iteration_ms: 940.3 | tokens_per_sec: 5.575734E+05 | tokens_per_sec_per_gpu: 8.712085E+03 | global_batch_size: 64 | lm_loss: 1.110886E+01 | lr: 1.010E-04 | model_tflops_per_gpu: 100.63 | hardware_tflops_per_gpu: 100.63 | grad_norm: 3673652.750 [ip-26-0-154-121:0]:06/21/2023 17:23:07 [INFO|DP=0|PP=0|TP=0]: iteration: 674 / 150000 | consumed_samples: 43136 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778669E+05 | tokens_per_sec_per_gpu: 9.029170E+03 | global_batch_size: 64 | lm_loss: 1.111822E+01 | lr: 1.011E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3610437.750 [ip-26-0-154-121:0]:06/21/2023 17:23:08 [INFO|DP=0|PP=0|TP=0]: iteration: 675 / 150000 | consumed_samples: 43200 | elapsed_time_per_iteration_ms: 993.4 | tokens_per_sec: 5.277536E+05 | tokens_per_sec_per_gpu: 8.246151E+03 | global_batch_size: 64 | lm_loss: 1.108838E+01 | lr: 1.012E-04 | model_tflops_per_gpu: 95.25 | hardware_tflops_per_gpu: 95.25 | grad_norm: 3555817.750 [ip-26-0-154-121:0]:06/21/2023 17:23:09 [INFO|DP=0|PP=0|TP=0]: iteration: 676 / 150000 | consumed_samples: 43264 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778365E+05 | tokens_per_sec_per_gpu: 9.028695E+03 | global_batch_size: 64 | lm_loss: 1.107522E+01 | lr: 1.014E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 3685013.750 [ip-26-0-154-121:0]:06/21/2023 17:23:09 [INFO|DP=0|PP=0|TP=0]: iteration: 677 / 150000 | consumed_samples: 43328 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783167E+05 | tokens_per_sec_per_gpu: 9.036199E+03 | global_batch_size: 64 | lm_loss: 1.109940E+01 | lr: 1.015E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3736245.250 [ip-26-0-154-121:0]:06/21/2023 17:23:10 [INFO|DP=0|PP=0|TP=0]: iteration: 678 / 150000 | consumed_samples: 43392 | elapsed_time_per_iteration_ms: 1001.9 | tokens_per_sec: 5.232878E+05 | tokens_per_sec_per_gpu: 8.176372E+03 | global_batch_size: 64 | lm_loss: 1.108553E+01 | lr: 1.017E-04 | model_tflops_per_gpu: 94.45 | hardware_tflops_per_gpu: 94.45 | grad_norm: 3727254.750 [ip-26-0-154-121:0]:06/21/2023 17:23:11 [INFO|DP=0|PP=0|TP=0]: iteration: 679 / 150000 | consumed_samples: 43456 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778052E+05 | tokens_per_sec_per_gpu: 9.028207E+03 | global_batch_size: 64 | lm_loss: 1.107653E+01 | lr: 1.018E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 3822557.750 [ip-26-0-154-121:0]:06/21/2023 17:23:12 [INFO|DP=0|PP=0|TP=0]: iteration: 680 / 150000 | consumed_samples: 43520 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778212E+05 | tokens_per_sec_per_gpu: 9.028456E+03 | global_batch_size: 64 | lm_loss: 1.106832E+01 | lr: 1.020E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 3637171.500 [ip-26-0-154-121:0]:06/21/2023 17:23:13 [INFO|DP=0|PP=0|TP=0]: iteration: 681 / 150000 | consumed_samples: 43584 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786393E+05 | tokens_per_sec_per_gpu: 9.041239E+03 | global_batch_size: 64 | lm_loss: 1.107951E+01 | lr: 1.021E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3809838.750 [ip-26-0-154-121:0]:06/21/2023 17:23:14 [INFO|DP=0|PP=0|TP=0]: iteration: 682 / 150000 | consumed_samples: 43648 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783236E+05 | tokens_per_sec_per_gpu: 9.036305E+03 | global_batch_size: 64 | lm_loss: 1.105644E+01 | lr: 1.023E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3592643.500 [ip-26-0-154-121:0]:06/21/2023 17:23:15 [INFO|DP=0|PP=0|TP=0]: iteration: 683 / 150000 | consumed_samples: 43712 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783512E+05 | tokens_per_sec_per_gpu: 9.036738E+03 | global_batch_size: 64 | lm_loss: 1.109287E+01 | lr: 1.024E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3702140.500 [ip-26-0-154-121:0]:06/21/2023 17:23:16 [INFO|DP=0|PP=0|TP=0]: iteration: 684 / 150000 | consumed_samples: 43776 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787598E+05 | tokens_per_sec_per_gpu: 9.043122E+03 | global_batch_size: 64 | lm_loss: 1.110416E+01 | lr: 1.026E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3588281.500 [ip-26-0-154-121:0]:06/21/2023 17:23:17 [INFO|DP=0|PP=0|TP=0]: iteration: 685 / 150000 | consumed_samples: 43840 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777975E+05 | tokens_per_sec_per_gpu: 9.028086E+03 | global_batch_size: 64 | lm_loss: 1.105649E+01 | lr: 1.027E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 3822601.500 [ip-26-0-154-121:0]:06/21/2023 17:23:18 [INFO|DP=0|PP=0|TP=0]: iteration: 686 / 150000 | consumed_samples: 43904 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783699E+05 | tokens_per_sec_per_gpu: 9.037030E+03 | global_batch_size: 64 | lm_loss: 1.108399E+01 | lr: 1.029E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3663542.250 [ip-26-0-154-121:0]:06/21/2023 17:23:19 [INFO|DP=0|PP=0|TP=0]: iteration: 687 / 150000 | consumed_samples: 43968 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784848E+05 | tokens_per_sec_per_gpu: 9.038825E+03 | global_batch_size: 64 | lm_loss: 1.109124E+01 | lr: 1.030E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3815387.000 [ip-26-0-154-121:0]:06/21/2023 17:23:20 [INFO|DP=0|PP=0|TP=0]: iteration: 688 / 150000 | consumed_samples: 44032 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786491E+05 | tokens_per_sec_per_gpu: 9.041392E+03 | global_batch_size: 64 | lm_loss: 1.108035E+01 | lr: 1.032E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3641270.500 [ip-26-0-154-121:0]:06/21/2023 17:23:20 [INFO|DP=0|PP=0|TP=0]: iteration: 689 / 150000 | consumed_samples: 44096 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785743E+05 | tokens_per_sec_per_gpu: 9.040224E+03 | global_batch_size: 64 | lm_loss: 1.109662E+01 | lr: 1.034E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3769980.750 [ip-26-0-154-121:0]:06/21/2023 17:23:21 [INFO|DP=0|PP=0|TP=0]: iteration: 690 / 150000 | consumed_samples: 44160 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784535E+05 | tokens_per_sec_per_gpu: 9.038335E+03 | global_batch_size: 64 | lm_loss: 1.107958E+01 | lr: 1.035E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3663481.500 [ip-26-0-154-121:0]:06/21/2023 17:23:22 [INFO|DP=0|PP=0|TP=0]: iteration: 691 / 150000 | consumed_samples: 44224 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784831E+05 | tokens_per_sec_per_gpu: 9.038799E+03 | global_batch_size: 64 | lm_loss: 1.103913E+01 | lr: 1.036E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3691815.750 [ip-26-0-154-121:0]:06/21/2023 17:23:23 [INFO|DP=0|PP=0|TP=0]: iteration: 692 / 150000 | consumed_samples: 44288 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776789E+05 | tokens_per_sec_per_gpu: 9.026233E+03 | global_batch_size: 64 | lm_loss: 1.110032E+01 | lr: 1.038E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 3587945.250 [ip-26-0-154-121:0]:06/21/2023 17:23:24 [INFO|DP=0|PP=0|TP=0]: iteration: 693 / 150000 | consumed_samples: 44352 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784033E+05 | tokens_per_sec_per_gpu: 9.037551E+03 | global_batch_size: 64 | lm_loss: 1.110171E+01 | lr: 1.040E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3861112.000 [ip-26-0-154-121:0]:06/21/2023 17:23:25 [INFO|DP=0|PP=0|TP=0]: iteration: 694 / 150000 | consumed_samples: 44416 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781586E+05 | tokens_per_sec_per_gpu: 9.033728E+03 | global_batch_size: 64 | lm_loss: 1.110130E+01 | lr: 1.041E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3618318.000 [ip-26-0-154-121:0]:06/21/2023 17:23:26 [INFO|DP=0|PP=0|TP=0]: iteration: 695 / 150000 | consumed_samples: 44480 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775629E+05 | tokens_per_sec_per_gpu: 9.024420E+03 | global_batch_size: 64 | lm_loss: 1.109254E+01 | lr: 1.042E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 3659033.000 [ip-26-0-154-121:0]:06/21/2023 17:23:27 [INFO|DP=0|PP=0|TP=0]: iteration: 696 / 150000 | consumed_samples: 44544 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777862E+05 | tokens_per_sec_per_gpu: 9.027910E+03 | global_batch_size: 64 | lm_loss: 1.105753E+01 | lr: 1.044E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 3634251.250 [ip-26-0-154-121:0]:06/21/2023 17:23:28 [INFO|DP=0|PP=0|TP=0]: iteration: 697 / 150000 | consumed_samples: 44608 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774426E+05 | tokens_per_sec_per_gpu: 9.022541E+03 | global_batch_size: 64 | lm_loss: 1.109876E+01 | lr: 1.046E-04 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 3591893.000 [ip-26-0-154-121:0]:06/21/2023 17:23:29 [INFO|DP=0|PP=0|TP=0]: iteration: 698 / 150000 | consumed_samples: 44672 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.776996E+05 | tokens_per_sec_per_gpu: 9.026556E+03 | global_batch_size: 64 | lm_loss: 1.111010E+01 | lr: 1.047E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 3679212.250 [ip-26-0-154-121:0]:06/21/2023 17:23:30 [INFO|DP=0|PP=0|TP=0]: iteration: 699 / 150000 | consumed_samples: 44736 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776410E+05 | tokens_per_sec_per_gpu: 9.025641E+03 | global_batch_size: 64 | lm_loss: 1.102230E+01 | lr: 1.048E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 3737498.250 [ip-26-0-154-121:0]:06/21/2023 17:23:30 [INFO|DP=0|PP=0|TP=0]: iteration: 700 / 150000 | consumed_samples: 44800 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776463E+05 | tokens_per_sec_per_gpu: 9.025724E+03 | global_batch_size: 64 | lm_loss: 1.107558E+01 | lr: 1.050E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 3570762.250 [ip-26-0-154-121:0]:06/21/2023 17:23:31 [INFO|DP=0|PP=0|TP=0]: iteration: 701 / 150000 | consumed_samples: 44864 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776632E+05 | tokens_per_sec_per_gpu: 9.025987E+03 | global_batch_size: 64 | lm_loss: 1.109988E+01 | lr: 1.051E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 3779440.500 [ip-26-0-154-121:0]:06/21/2023 17:23:32 [INFO|DP=0|PP=0|TP=0]: iteration: 702 / 150000 | consumed_samples: 44928 | elapsed_time_per_iteration_ms: 908.2 | tokens_per_sec: 5.772719E+05 | tokens_per_sec_per_gpu: 9.019874E+03 | global_batch_size: 64 | lm_loss: 1.107829E+01 | lr: 1.053E-04 | model_tflops_per_gpu: 104.19 | hardware_tflops_per_gpu: 104.19 | grad_norm: 3574624.750 [ip-26-0-154-121:0]:06/21/2023 17:23:33 [INFO|DP=0|PP=0|TP=0]: iteration: 703 / 150000 | consumed_samples: 44992 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777371E+05 | tokens_per_sec_per_gpu: 9.027142E+03 | global_batch_size: 64 | lm_loss: 1.109000E+01 | lr: 1.054E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 3769992.250 [ip-26-0-154-121:0]:06/21/2023 17:23:34 [INFO|DP=0|PP=0|TP=0]: iteration: 704 / 150000 | consumed_samples: 45056 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775454E+05 | tokens_per_sec_per_gpu: 9.024147E+03 | global_batch_size: 64 | lm_loss: 1.109669E+01 | lr: 1.056E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 3718429.500 [ip-26-0-154-121:0]:06/21/2023 17:23:35 [INFO|DP=0|PP=0|TP=0]: iteration: 705 / 150000 | consumed_samples: 45120 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780165E+05 | tokens_per_sec_per_gpu: 9.031508E+03 | global_batch_size: 64 | lm_loss: 1.107875E+01 | lr: 1.057E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 3625833.000 [ip-26-0-154-121:0]:06/21/2023 17:23:36 [INFO|DP=0|PP=0|TP=0]: iteration: 706 / 150000 | consumed_samples: 45184 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.775931E+05 | tokens_per_sec_per_gpu: 9.024891E+03 | global_batch_size: 64 | lm_loss: 1.109205E+01 | lr: 1.059E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 3682763.500 [ip-26-0-154-121:0]:06/21/2023 17:23:37 [INFO|DP=0|PP=0|TP=0]: iteration: 707 / 150000 | consumed_samples: 45248 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779799E+05 | tokens_per_sec_per_gpu: 9.030935E+03 | global_batch_size: 64 | lm_loss: 1.108808E+01 | lr: 1.060E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4568144.500 [ip-26-0-154-121:0]:06/21/2023 17:23:38 [INFO|DP=0|PP=0|TP=0]: iteration: 708 / 150000 | consumed_samples: 45312 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782439E+05 | tokens_per_sec_per_gpu: 9.035060E+03 | global_batch_size: 64 | lm_loss: 1.109854E+01 | lr: 1.062E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3838020.250 [ip-26-0-154-121:0]:06/21/2023 17:23:39 [INFO|DP=0|PP=0|TP=0]: iteration: 709 / 150000 | consumed_samples: 45376 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775577E+05 | tokens_per_sec_per_gpu: 9.024339E+03 | global_batch_size: 64 | lm_loss: 1.110167E+01 | lr: 1.063E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 4071766.250 [ip-26-0-154-121:0]:06/21/2023 17:23:40 [INFO|DP=0|PP=0|TP=0]: iteration: 710 / 150000 | consumed_samples: 45440 | elapsed_time_per_iteration_ms: 908.4 | tokens_per_sec: 5.771652E+05 | tokens_per_sec_per_gpu: 9.018207E+03 | global_batch_size: 64 | lm_loss: 1.108278E+01 | lr: 1.065E-04 | model_tflops_per_gpu: 104.17 | hardware_tflops_per_gpu: 104.17 | grad_norm: 3967295.250 [ip-26-0-154-121:0]:06/21/2023 17:23:40 [INFO|DP=0|PP=0|TP=0]: iteration: 711 / 150000 | consumed_samples: 45504 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778256E+05 | tokens_per_sec_per_gpu: 9.028525E+03 | global_batch_size: 64 | lm_loss: 1.109066E+01 | lr: 1.066E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 3896013.750 [ip-26-0-154-121:0]:06/21/2023 17:23:41 [INFO|DP=0|PP=0|TP=0]: iteration: 712 / 150000 | consumed_samples: 45568 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777090E+05 | tokens_per_sec_per_gpu: 9.026703E+03 | global_batch_size: 64 | lm_loss: 1.107145E+01 | lr: 1.068E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 4121748.000 [ip-26-0-154-121:0]:06/21/2023 17:23:42 [INFO|DP=0|PP=0|TP=0]: iteration: 713 / 150000 | consumed_samples: 45632 | elapsed_time_per_iteration_ms: 908.2 | tokens_per_sec: 5.773113E+05 | tokens_per_sec_per_gpu: 9.020489E+03 | global_batch_size: 64 | lm_loss: 1.109591E+01 | lr: 1.069E-04 | model_tflops_per_gpu: 104.20 | hardware_tflops_per_gpu: 104.20 | grad_norm: 3771720.000 [ip-26-0-154-121:0]:06/21/2023 17:23:43 [INFO|DP=0|PP=0|TP=0]: iteration: 714 / 150000 | consumed_samples: 45696 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775410E+05 | tokens_per_sec_per_gpu: 9.024078E+03 | global_batch_size: 64 | lm_loss: 1.111696E+01 | lr: 1.071E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 3797792.000 [ip-26-0-154-121:0]:06/21/2023 17:23:44 [INFO|DP=0|PP=0|TP=0]: iteration: 715 / 150000 | consumed_samples: 45760 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779323E+05 | tokens_per_sec_per_gpu: 9.030193E+03 | global_batch_size: 64 | lm_loss: 1.106164E+01 | lr: 1.072E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 3887300.500 [ip-26-0-154-121:0]:06/21/2023 17:23:45 [INFO|DP=0|PP=0|TP=0]: iteration: 716 / 150000 | consumed_samples: 45824 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775647E+05 | tokens_per_sec_per_gpu: 9.024448E+03 | global_batch_size: 64 | lm_loss: 1.109777E+01 | lr: 1.074E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 3747159.500 [ip-26-0-154-121:0]:06/21/2023 17:23:46 [INFO|DP=0|PP=0|TP=0]: iteration: 717 / 150000 | consumed_samples: 45888 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.774326E+05 | tokens_per_sec_per_gpu: 9.022384E+03 | global_batch_size: 64 | lm_loss: 1.110777E+01 | lr: 1.075E-04 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 4008965.000 [ip-26-0-154-121:0]:06/21/2023 17:23:47 [INFO|DP=0|PP=0|TP=0]: iteration: 718 / 150000 | consumed_samples: 45952 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777944E+05 | tokens_per_sec_per_gpu: 9.028038E+03 | global_batch_size: 64 | lm_loss: 1.110583E+01 | lr: 1.077E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 3908136.750 [ip-26-0-154-121:0]:06/21/2023 17:23:48 [INFO|DP=0|PP=0|TP=0]: iteration: 719 / 150000 | consumed_samples: 46016 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774879E+05 | tokens_per_sec_per_gpu: 9.023249E+03 | global_batch_size: 64 | lm_loss: 1.108098E+01 | lr: 1.078E-04 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 4166184.000 [ip-26-0-154-121:0]:06/21/2023 17:23:49 [INFO|DP=0|PP=0|TP=0]: iteration: 720 / 150000 | consumed_samples: 46080 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778606E+05 | tokens_per_sec_per_gpu: 9.029073E+03 | global_batch_size: 64 | lm_loss: 1.109817E+01 | lr: 1.080E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3915765.250 [ip-26-0-154-121:0]:06/21/2023 17:23:50 [INFO|DP=0|PP=0|TP=0]: iteration: 721 / 150000 | consumed_samples: 46144 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779910E+05 | tokens_per_sec_per_gpu: 9.031109E+03 | global_batch_size: 64 | lm_loss: 1.110149E+01 | lr: 1.081E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4305641.000 [ip-26-0-154-121:0]:06/21/2023 17:23:50 [INFO|DP=0|PP=0|TP=0]: iteration: 722 / 150000 | consumed_samples: 46208 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776407E+05 | tokens_per_sec_per_gpu: 9.025636E+03 | global_batch_size: 64 | lm_loss: 1.111046E+01 | lr: 1.083E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 3777326.750 [ip-26-0-154-121:0]:06/21/2023 17:23:51 [INFO|DP=0|PP=0|TP=0]: iteration: 723 / 150000 | consumed_samples: 46272 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.773951E+05 | tokens_per_sec_per_gpu: 9.021799E+03 | global_batch_size: 64 | lm_loss: 1.108093E+01 | lr: 1.084E-04 | model_tflops_per_gpu: 104.21 | hardware_tflops_per_gpu: 104.21 | grad_norm: 4162983.250 [ip-26-0-154-121:0]:06/21/2023 17:23:52 [INFO|DP=0|PP=0|TP=0]: iteration: 724 / 150000 | consumed_samples: 46336 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.774017E+05 | tokens_per_sec_per_gpu: 9.021901E+03 | global_batch_size: 64 | lm_loss: 1.108838E+01 | lr: 1.086E-04 | model_tflops_per_gpu: 104.21 | hardware_tflops_per_gpu: 104.21 | grad_norm: 3778886.000 [ip-26-0-154-121:0]:06/21/2023 17:23:53 [INFO|DP=0|PP=0|TP=0]: iteration: 725 / 150000 | consumed_samples: 46400 | elapsed_time_per_iteration_ms: 908.2 | tokens_per_sec: 5.773040E+05 | tokens_per_sec_per_gpu: 9.020376E+03 | global_batch_size: 64 | lm_loss: 1.104909E+01 | lr: 1.087E-04 | model_tflops_per_gpu: 104.20 | hardware_tflops_per_gpu: 104.20 | grad_norm: 4454791.500 [ip-26-0-154-121:0]:06/21/2023 17:23:54 [INFO|DP=0|PP=0|TP=0]: iteration: 726 / 150000 | consumed_samples: 46464 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778118E+05 | tokens_per_sec_per_gpu: 9.028309E+03 | global_batch_size: 64 | lm_loss: 1.110290E+01 | lr: 1.089E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 3849366.500 [ip-26-0-154-121:0]:06/21/2023 17:23:55 [INFO|DP=0|PP=0|TP=0]: iteration: 727 / 150000 | consumed_samples: 46528 | elapsed_time_per_iteration_ms: 908.1 | tokens_per_sec: 5.773530E+05 | tokens_per_sec_per_gpu: 9.021141E+03 | global_batch_size: 64 | lm_loss: 1.111288E+01 | lr: 1.090E-04 | model_tflops_per_gpu: 104.20 | hardware_tflops_per_gpu: 104.20 | grad_norm: 3980340.750 [ip-26-0-154-121:0]:06/21/2023 17:23:56 [INFO|DP=0|PP=0|TP=0]: iteration: 728 / 150000 | consumed_samples: 46592 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782483E+05 | tokens_per_sec_per_gpu: 9.035129E+03 | global_batch_size: 64 | lm_loss: 1.107723E+01 | lr: 1.092E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3871933.500 [ip-26-0-154-121:0]:06/21/2023 17:23:57 [INFO|DP=0|PP=0|TP=0]: iteration: 729 / 150000 | consumed_samples: 46656 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780324E+05 | tokens_per_sec_per_gpu: 9.031757E+03 | global_batch_size: 64 | lm_loss: 1.109761E+01 | lr: 1.093E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3918499.500 [ip-26-0-154-121:0]:06/21/2023 17:23:58 [INFO|DP=0|PP=0|TP=0]: iteration: 730 / 150000 | consumed_samples: 46720 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781954E+05 | tokens_per_sec_per_gpu: 9.034303E+03 | global_batch_size: 64 | lm_loss: 1.107211E+01 | lr: 1.095E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3916758.750 [ip-26-0-154-121:0]:06/21/2023 17:23:59 [INFO|DP=0|PP=0|TP=0]: iteration: 731 / 150000 | consumed_samples: 46784 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780359E+05 | tokens_per_sec_per_gpu: 9.031811E+03 | global_batch_size: 64 | lm_loss: 1.100621E+01 | lr: 1.096E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3907729.750 [ip-26-0-154-121:0]:06/21/2023 17:24:00 [INFO|DP=0|PP=0|TP=0]: iteration: 732 / 150000 | consumed_samples: 46848 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778187E+05 | tokens_per_sec_per_gpu: 9.028418E+03 | global_batch_size: 64 | lm_loss: 1.110414E+01 | lr: 1.098E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 3973497.500 [ip-26-0-154-121:0]:06/21/2023 17:24:00 [INFO|DP=0|PP=0|TP=0]: iteration: 733 / 150000 | consumed_samples: 46912 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774940E+05 | tokens_per_sec_per_gpu: 9.023344E+03 | global_batch_size: 64 | lm_loss: 1.108995E+01 | lr: 1.099E-04 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 3839399.500 [ip-26-0-154-121:0]:06/21/2023 17:24:01 [INFO|DP=0|PP=0|TP=0]: iteration: 734 / 150000 | consumed_samples: 46976 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779487E+05 | tokens_per_sec_per_gpu: 9.030449E+03 | global_batch_size: 64 | lm_loss: 1.109313E+01 | lr: 1.101E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 3933928.250 [ip-26-0-154-121:0]:06/21/2023 17:24:02 [INFO|DP=0|PP=0|TP=0]: iteration: 735 / 150000 | consumed_samples: 47040 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781080E+05 | tokens_per_sec_per_gpu: 9.032937E+03 | global_batch_size: 64 | lm_loss: 1.108115E+01 | lr: 1.102E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3932885.500 [ip-26-0-154-121:0]:06/21/2023 17:24:03 [INFO|DP=0|PP=0|TP=0]: iteration: 736 / 150000 | consumed_samples: 47104 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.776293E+05 | tokens_per_sec_per_gpu: 9.025458E+03 | global_batch_size: 64 | lm_loss: 1.109029E+01 | lr: 1.104E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 3939604.000 [ip-26-0-154-121:0]:06/21/2023 17:24:04 [INFO|DP=0|PP=0|TP=0]: iteration: 737 / 150000 | consumed_samples: 47168 | elapsed_time_per_iteration_ms: 908.2 | tokens_per_sec: 5.772624E+05 | tokens_per_sec_per_gpu: 9.019724E+03 | global_batch_size: 64 | lm_loss: 1.108768E+01 | lr: 1.105E-04 | model_tflops_per_gpu: 104.19 | hardware_tflops_per_gpu: 104.19 | grad_norm: 3987626.750 [ip-26-0-154-121:0]:06/21/2023 17:24:05 [INFO|DP=0|PP=0|TP=0]: iteration: 738 / 150000 | consumed_samples: 47232 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779524E+05 | tokens_per_sec_per_gpu: 9.030506E+03 | global_batch_size: 64 | lm_loss: 1.109751E+01 | lr: 1.107E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4074457.000 [ip-26-0-154-121:0]:06/21/2023 17:24:06 [INFO|DP=0|PP=0|TP=0]: iteration: 739 / 150000 | consumed_samples: 47296 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.773974E+05 | tokens_per_sec_per_gpu: 9.021835E+03 | global_batch_size: 64 | lm_loss: 1.109399E+01 | lr: 1.108E-04 | model_tflops_per_gpu: 104.21 | hardware_tflops_per_gpu: 104.21 | grad_norm: 3731679.750 [ip-26-0-154-121:0]:06/21/2023 17:24:07 [INFO|DP=0|PP=0|TP=0]: iteration: 740 / 150000 | consumed_samples: 47360 | elapsed_time_per_iteration_ms: 908.1 | tokens_per_sec: 5.773618E+05 | tokens_per_sec_per_gpu: 9.021278E+03 | global_batch_size: 64 | lm_loss: 1.109159E+01 | lr: 1.110E-04 | model_tflops_per_gpu: 104.21 | hardware_tflops_per_gpu: 104.21 | grad_norm: 3959319.750 [ip-26-0-154-121:0]:06/21/2023 17:24:08 [INFO|DP=0|PP=0|TP=0]: iteration: 741 / 150000 | consumed_samples: 47424 | elapsed_time_per_iteration_ms: 908.8 | tokens_per_sec: 5.769242E+05 | tokens_per_sec_per_gpu: 9.014440E+03 | global_batch_size: 64 | lm_loss: 1.109534E+01 | lr: 1.111E-04 | model_tflops_per_gpu: 104.13 | hardware_tflops_per_gpu: 104.13 | grad_norm: 3818150.000 [ip-26-0-154-121:0]:06/21/2023 17:24:09 [INFO|DP=0|PP=0|TP=0]: iteration: 742 / 150000 | consumed_samples: 47488 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783118E+05 | tokens_per_sec_per_gpu: 9.036123E+03 | global_batch_size: 64 | lm_loss: 1.107724E+01 | lr: 1.113E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3862773.500 [ip-26-0-154-121:0]:06/21/2023 17:24:10 [INFO|DP=0|PP=0|TP=0]: iteration: 743 / 150000 | consumed_samples: 47552 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781753E+05 | tokens_per_sec_per_gpu: 9.033989E+03 | global_batch_size: 64 | lm_loss: 1.109611E+01 | lr: 1.115E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 4011143.500 [ip-26-0-154-121:0]:06/21/2023 17:24:10 [INFO|DP=0|PP=0|TP=0]: iteration: 744 / 150000 | consumed_samples: 47616 | elapsed_time_per_iteration_ms: 908.4 | tokens_per_sec: 5.771709E+05 | tokens_per_sec_per_gpu: 9.018295E+03 | global_batch_size: 64 | lm_loss: 1.105441E+01 | lr: 1.116E-04 | model_tflops_per_gpu: 104.17 | hardware_tflops_per_gpu: 104.17 | grad_norm: 4004824.500 [ip-26-0-154-121:0]:06/21/2023 17:24:11 [INFO|DP=0|PP=0|TP=0]: iteration: 745 / 150000 | consumed_samples: 47680 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780653E+05 | tokens_per_sec_per_gpu: 9.032270E+03 | global_batch_size: 64 | lm_loss: 1.109491E+01 | lr: 1.117E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3830064.500 [ip-26-0-154-121:0]:06/21/2023 17:24:12 [INFO|DP=0|PP=0|TP=0]: iteration: 746 / 150000 | consumed_samples: 47744 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776428E+05 | tokens_per_sec_per_gpu: 9.025669E+03 | global_batch_size: 64 | lm_loss: 1.109558E+01 | lr: 1.119E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 4020871.000 [ip-26-0-154-121:0]:06/21/2023 17:24:13 [INFO|DP=0|PP=0|TP=0]: iteration: 747 / 150000 | consumed_samples: 47808 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778600E+05 | tokens_per_sec_per_gpu: 9.029063E+03 | global_batch_size: 64 | lm_loss: 1.109016E+01 | lr: 1.121E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3976394.500 [ip-26-0-154-121:0]:06/21/2023 17:24:14 [INFO|DP=0|PP=0|TP=0]: iteration: 748 / 150000 | consumed_samples: 47872 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782644E+05 | tokens_per_sec_per_gpu: 9.035381E+03 | global_batch_size: 64 | lm_loss: 1.108413E+01 | lr: 1.122E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 4173653.000 [ip-26-0-154-121:0]:06/21/2023 17:24:15 [INFO|DP=0|PP=0|TP=0]: iteration: 749 / 150000 | consumed_samples: 47936 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780847E+05 | tokens_per_sec_per_gpu: 9.032574E+03 | global_batch_size: 64 | lm_loss: 1.110302E+01 | lr: 1.123E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3944510.750 [ip-26-0-154-121:0]:06/21/2023 17:24:16 [INFO|DP=0|PP=0|TP=0]: iteration: 750 / 150000 | consumed_samples: 48000 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777491E+05 | tokens_per_sec_per_gpu: 9.027329E+03 | global_batch_size: 64 | lm_loss: 1.109949E+01 | lr: 1.125E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 3876116.000 [ip-26-0-154-121:0]:06/21/2023 17:24:17 [INFO|DP=0|PP=0|TP=0]: iteration: 751 / 150000 | consumed_samples: 48064 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775485E+05 | tokens_per_sec_per_gpu: 9.024195E+03 | global_batch_size: 64 | lm_loss: 1.106991E+01 | lr: 1.126E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 3810517.250 [ip-26-0-154-121:0]:06/21/2023 17:24:18 [INFO|DP=0|PP=0|TP=0]: iteration: 752 / 150000 | consumed_samples: 48128 | elapsed_time_per_iteration_ms: 908.1 | tokens_per_sec: 5.773636E+05 | tokens_per_sec_per_gpu: 9.021306E+03 | global_batch_size: 64 | lm_loss: 1.105516E+01 | lr: 1.128E-04 | model_tflops_per_gpu: 104.21 | hardware_tflops_per_gpu: 104.21 | grad_norm: 4008916.250 [ip-26-0-154-121:0]:06/21/2023 17:24:19 [INFO|DP=0|PP=0|TP=0]: iteration: 753 / 150000 | consumed_samples: 48192 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779910E+05 | tokens_per_sec_per_gpu: 9.031109E+03 | global_batch_size: 64 | lm_loss: 1.109049E+01 | lr: 1.129E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4244825.000 [ip-26-0-154-121:0]:06/21/2023 17:24:20 [INFO|DP=0|PP=0|TP=0]: iteration: 754 / 150000 | consumed_samples: 48256 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.775899E+05 | tokens_per_sec_per_gpu: 9.024842E+03 | global_batch_size: 64 | lm_loss: 1.109335E+01 | lr: 1.131E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 4610187.000 [ip-26-0-154-121:0]:06/21/2023 17:24:20 [INFO|DP=0|PP=0|TP=0]: iteration: 755 / 150000 | consumed_samples: 48320 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782957E+05 | tokens_per_sec_per_gpu: 9.035871E+03 | global_batch_size: 64 | lm_loss: 1.106893E+01 | lr: 1.132E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 4039714.500 [ip-26-0-154-121:0]:06/21/2023 17:24:21 [INFO|DP=0|PP=0|TP=0]: iteration: 756 / 150000 | consumed_samples: 48384 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780546E+05 | tokens_per_sec_per_gpu: 9.032103E+03 | global_batch_size: 64 | lm_loss: 1.108271E+01 | lr: 1.134E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 4475271.500 [ip-26-0-154-121:0]:06/21/2023 17:24:22 [INFO|DP=0|PP=0|TP=0]: iteration: 757 / 150000 | consumed_samples: 48448 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.776085E+05 | tokens_per_sec_per_gpu: 9.025133E+03 | global_batch_size: 64 | lm_loss: 1.110515E+01 | lr: 1.135E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 4134833.750 [ip-26-0-154-121:0]:06/21/2023 17:24:23 [INFO|DP=0|PP=0|TP=0]: iteration: 758 / 150000 | consumed_samples: 48512 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780496E+05 | tokens_per_sec_per_gpu: 9.032025E+03 | global_batch_size: 64 | lm_loss: 1.108112E+01 | lr: 1.137E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 4569862.000 [ip-26-0-154-121:0]:06/21/2023 17:24:24 [INFO|DP=0|PP=0|TP=0]: iteration: 759 / 150000 | consumed_samples: 48576 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788768E+05 | tokens_per_sec_per_gpu: 9.044950E+03 | global_batch_size: 64 | lm_loss: 1.108553E+01 | lr: 1.138E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 4378849.500 [ip-26-0-154-121:0]:06/21/2023 17:24:25 [INFO|DP=0|PP=0|TP=0]: iteration: 760 / 150000 | consumed_samples: 48640 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782615E+05 | tokens_per_sec_per_gpu: 9.035336E+03 | global_batch_size: 64 | lm_loss: 1.110464E+01 | lr: 1.140E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 4041052.250 [ip-26-0-154-121:0]:06/21/2023 17:24:26 [INFO|DP=0|PP=0|TP=0]: iteration: 761 / 150000 | consumed_samples: 48704 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783249E+05 | tokens_per_sec_per_gpu: 9.036327E+03 | global_batch_size: 64 | lm_loss: 1.107922E+01 | lr: 1.141E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 4417244.500 [ip-26-0-154-121:0]:06/21/2023 17:24:27 [INFO|DP=0|PP=0|TP=0]: iteration: 762 / 150000 | consumed_samples: 48768 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785845E+05 | tokens_per_sec_per_gpu: 9.040383E+03 | global_batch_size: 64 | lm_loss: 1.108273E+01 | lr: 1.143E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4333617.500 [ip-26-0-154-121:0]:06/21/2023 17:24:28 [INFO|DP=0|PP=0|TP=0]: iteration: 763 / 150000 | consumed_samples: 48832 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779923E+05 | tokens_per_sec_per_gpu: 9.031130E+03 | global_batch_size: 64 | lm_loss: 1.109311E+01 | lr: 1.144E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4142705.750 [ip-26-0-154-121:0]:06/21/2023 17:24:29 [INFO|DP=0|PP=0|TP=0]: iteration: 764 / 150000 | consumed_samples: 48896 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779767E+05 | tokens_per_sec_per_gpu: 9.030886E+03 | global_batch_size: 64 | lm_loss: 1.109052E+01 | lr: 1.146E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4077684.250 [ip-26-0-154-121:0]:06/21/2023 17:24:30 [INFO|DP=0|PP=0|TP=0]: iteration: 765 / 150000 | consumed_samples: 48960 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777835E+05 | tokens_per_sec_per_gpu: 9.027867E+03 | global_batch_size: 64 | lm_loss: 1.110302E+01 | lr: 1.147E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 4480385.000 [ip-26-0-154-121:0]:06/21/2023 17:24:30 [INFO|DP=0|PP=0|TP=0]: iteration: 766 / 150000 | consumed_samples: 49024 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777281E+05 | tokens_per_sec_per_gpu: 9.027002E+03 | global_batch_size: 64 | lm_loss: 1.109377E+01 | lr: 1.149E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 4203083.000 [ip-26-0-154-121:0]:06/21/2023 17:24:31 [INFO|DP=0|PP=0|TP=0]: iteration: 767 / 150000 | consumed_samples: 49088 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781498E+05 | tokens_per_sec_per_gpu: 9.033590E+03 | global_batch_size: 64 | lm_loss: 1.105225E+01 | lr: 1.150E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3966660.500 [ip-26-0-154-121:0]:06/21/2023 17:24:32 [INFO|DP=0|PP=0|TP=0]: iteration: 768 / 150000 | consumed_samples: 49152 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780151E+05 | tokens_per_sec_per_gpu: 9.031486E+03 | global_batch_size: 64 | lm_loss: 1.108308E+01 | lr: 1.152E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4380076.000 [ip-26-0-154-121:0]:06/21/2023 17:24:33 [INFO|DP=0|PP=0|TP=0]: iteration: 769 / 150000 | consumed_samples: 49216 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778547E+05 | tokens_per_sec_per_gpu: 9.028980E+03 | global_batch_size: 64 | lm_loss: 1.108920E+01 | lr: 1.153E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 4452287.000 [ip-26-0-154-121:0]:06/21/2023 17:24:34 [INFO|DP=0|PP=0|TP=0]: iteration: 770 / 150000 | consumed_samples: 49280 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778660E+05 | tokens_per_sec_per_gpu: 9.029156E+03 | global_batch_size: 64 | lm_loss: 1.108355E+01 | lr: 1.155E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 4277091.500 [ip-26-0-154-121:0]:06/21/2023 17:24:35 [INFO|DP=0|PP=0|TP=0]: iteration: 771 / 150000 | consumed_samples: 49344 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784007E+05 | tokens_per_sec_per_gpu: 9.037511E+03 | global_batch_size: 64 | lm_loss: 1.105180E+01 | lr: 1.156E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 4960097.000 [ip-26-0-154-121:0]:06/21/2023 17:24:36 [INFO|DP=0|PP=0|TP=0]: iteration: 772 / 150000 | consumed_samples: 49408 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781064E+05 | tokens_per_sec_per_gpu: 9.032913E+03 | global_batch_size: 64 | lm_loss: 1.108527E+01 | lr: 1.158E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 4040125.500 [ip-26-0-154-121:0]:06/21/2023 17:24:37 [INFO|DP=0|PP=0|TP=0]: iteration: 773 / 150000 | consumed_samples: 49472 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781391E+05 | tokens_per_sec_per_gpu: 9.033424E+03 | global_batch_size: 64 | lm_loss: 1.106184E+01 | lr: 1.159E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 4819213.500 [ip-26-0-154-121:0]:06/21/2023 17:24:38 [INFO|DP=0|PP=0|TP=0]: iteration: 774 / 150000 | consumed_samples: 49536 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784526E+05 | tokens_per_sec_per_gpu: 9.038321E+03 | global_batch_size: 64 | lm_loss: 1.107761E+01 | lr: 1.161E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 4091580.500 [ip-26-0-154-121:0]:06/21/2023 17:24:39 [INFO|DP=0|PP=0|TP=0]: iteration: 775 / 150000 | consumed_samples: 49600 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780802E+05 | tokens_per_sec_per_gpu: 9.032502E+03 | global_batch_size: 64 | lm_loss: 1.109332E+01 | lr: 1.162E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 4272882.500 [ip-26-0-154-121:0]:06/21/2023 17:24:40 [INFO|DP=0|PP=0|TP=0]: iteration: 776 / 150000 | consumed_samples: 49664 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785121E+05 | tokens_per_sec_per_gpu: 9.039251E+03 | global_batch_size: 64 | lm_loss: 1.108263E+01 | lr: 1.164E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 4456539.000 [ip-26-0-154-121:0]:06/21/2023 17:24:40 [INFO|DP=0|PP=0|TP=0]: iteration: 777 / 150000 | consumed_samples: 49728 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782471E+05 | tokens_per_sec_per_gpu: 9.035110E+03 | global_batch_size: 64 | lm_loss: 1.108579E+01 | lr: 1.165E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 4088657.000 [ip-26-0-154-121:0]:06/21/2023 17:24:41 [INFO|DP=0|PP=0|TP=0]: iteration: 778 / 150000 | consumed_samples: 49792 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779158E+05 | tokens_per_sec_per_gpu: 9.029934E+03 | global_batch_size: 64 | lm_loss: 1.110459E+01 | lr: 1.167E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4452438.500 [ip-26-0-154-121:0]:06/21/2023 17:24:42 [INFO|DP=0|PP=0|TP=0]: iteration: 779 / 150000 | consumed_samples: 49856 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779822E+05 | tokens_per_sec_per_gpu: 9.030971E+03 | global_batch_size: 64 | lm_loss: 1.102673E+01 | lr: 1.168E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4123323.500 [ip-26-0-154-121:0]:06/21/2023 17:24:43 [INFO|DP=0|PP=0|TP=0]: iteration: 780 / 150000 | consumed_samples: 49920 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783932E+05 | tokens_per_sec_per_gpu: 9.037394E+03 | global_batch_size: 64 | lm_loss: 1.109817E+01 | lr: 1.170E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 4192115.500 [ip-26-0-154-121:0]:06/21/2023 17:24:44 [INFO|DP=0|PP=0|TP=0]: iteration: 781 / 150000 | consumed_samples: 49984 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.782005E+05 | tokens_per_sec_per_gpu: 9.034383E+03 | global_batch_size: 64 | lm_loss: 1.108394E+01 | lr: 1.171E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 4069746.500 [ip-26-0-154-121:0]:06/21/2023 17:24:45 [INFO|DP=0|PP=0|TP=0]: iteration: 782 / 150000 | consumed_samples: 50048 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780134E+05 | tokens_per_sec_per_gpu: 9.031460E+03 | global_batch_size: 64 | lm_loss: 1.108480E+01 | lr: 1.173E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4135895.250 [ip-26-0-154-121:0]:06/21/2023 17:24:46 [INFO|DP=0|PP=0|TP=0]: iteration: 783 / 150000 | consumed_samples: 50112 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781373E+05 | tokens_per_sec_per_gpu: 9.033395E+03 | global_batch_size: 64 | lm_loss: 1.108204E+01 | lr: 1.174E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 4174408.000 [ip-26-0-154-121:0]:06/21/2023 17:24:47 [INFO|DP=0|PP=0|TP=0]: iteration: 784 / 150000 | consumed_samples: 50176 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781791E+05 | tokens_per_sec_per_gpu: 9.034048E+03 | global_batch_size: 64 | lm_loss: 1.108550E+01 | lr: 1.176E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 4339045.500 [ip-26-0-154-121:0]:06/21/2023 17:24:48 [INFO|DP=0|PP=0|TP=0]: iteration: 785 / 150000 | consumed_samples: 50240 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780557E+05 | tokens_per_sec_per_gpu: 9.032120E+03 | global_batch_size: 64 | lm_loss: 1.107742E+01 | lr: 1.177E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 4476835.000 [ip-26-0-154-121:0]:06/21/2023 17:24:49 [INFO|DP=0|PP=0|TP=0]: iteration: 786 / 150000 | consumed_samples: 50304 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782062E+05 | tokens_per_sec_per_gpu: 9.034471E+03 | global_batch_size: 64 | lm_loss: 1.109797E+01 | lr: 1.179E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 4388520.500 [ip-26-0-154-121:0]:06/21/2023 17:24:50 [INFO|DP=0|PP=0|TP=0]: iteration: 787 / 150000 | consumed_samples: 50368 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781028E+05 | tokens_per_sec_per_gpu: 9.032856E+03 | global_batch_size: 64 | lm_loss: 1.102637E+01 | lr: 1.180E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 4531573.500 [ip-26-0-154-121:0]:06/21/2023 17:24:50 [INFO|DP=0|PP=0|TP=0]: iteration: 788 / 150000 | consumed_samples: 50432 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784157E+05 | tokens_per_sec_per_gpu: 9.037746E+03 | global_batch_size: 64 | lm_loss: 1.108338E+01 | lr: 1.182E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 4865619.000 [ip-26-0-154-121:0]:06/21/2023 17:24:51 [INFO|DP=0|PP=0|TP=0]: iteration: 789 / 150000 | consumed_samples: 50496 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779961E+05 | tokens_per_sec_per_gpu: 9.031189E+03 | global_batch_size: 64 | lm_loss: 1.107617E+01 | lr: 1.183E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4801824.000 [ip-26-0-154-121:0]:06/21/2023 17:24:52 [INFO|DP=0|PP=0|TP=0]: iteration: 790 / 150000 | consumed_samples: 50560 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781875E+05 | tokens_per_sec_per_gpu: 9.034179E+03 | global_batch_size: 64 | lm_loss: 1.108653E+01 | lr: 1.185E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 4507030.500 [ip-26-0-154-121:0]:06/21/2023 17:24:53 [INFO|DP=0|PP=0|TP=0]: iteration: 791 / 150000 | consumed_samples: 50624 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779917E+05 | tokens_per_sec_per_gpu: 9.031121E+03 | global_batch_size: 64 | lm_loss: 1.105796E+01 | lr: 1.186E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4603554.500 [ip-26-0-154-121:0]:06/21/2023 17:24:54 [INFO|DP=0|PP=0|TP=0]: iteration: 792 / 150000 | consumed_samples: 50688 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782972E+05 | tokens_per_sec_per_gpu: 9.035894E+03 | global_batch_size: 64 | lm_loss: 1.107866E+01 | lr: 1.188E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 4557695.500 [ip-26-0-154-121:0]:06/21/2023 17:24:55 [INFO|DP=0|PP=0|TP=0]: iteration: 793 / 150000 | consumed_samples: 50752 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782060E+05 | tokens_per_sec_per_gpu: 9.034469E+03 | global_batch_size: 64 | lm_loss: 1.098693E+01 | lr: 1.189E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 4444105.500 [ip-26-0-154-121:0]:06/21/2023 17:24:56 [INFO|DP=0|PP=0|TP=0]: iteration: 794 / 150000 | consumed_samples: 50816 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786089E+05 | tokens_per_sec_per_gpu: 9.040764E+03 | global_batch_size: 64 | lm_loss: 1.108875E+01 | lr: 1.191E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4692739.500 [ip-26-0-154-121:0]:06/21/2023 17:24:57 [INFO|DP=0|PP=0|TP=0]: iteration: 795 / 150000 | consumed_samples: 50880 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787534E+05 | tokens_per_sec_per_gpu: 9.043022E+03 | global_batch_size: 64 | lm_loss: 1.104127E+01 | lr: 1.192E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 4440605.500 [ip-26-0-154-121:0]:06/21/2023 17:24:58 [INFO|DP=0|PP=0|TP=0]: iteration: 796 / 150000 | consumed_samples: 50944 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787097E+05 | tokens_per_sec_per_gpu: 9.042339E+03 | global_batch_size: 64 | lm_loss: 1.109834E+01 | lr: 1.194E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 4372424.500 [ip-26-0-154-121:0]:06/21/2023 17:24:59 [INFO|DP=0|PP=0|TP=0]: iteration: 797 / 150000 | consumed_samples: 51008 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781464E+05 | tokens_per_sec_per_gpu: 9.033538E+03 | global_batch_size: 64 | lm_loss: 1.108408E+01 | lr: 1.195E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 4412336.000 [ip-26-0-154-121:0]:06/21/2023 17:24:59 [INFO|DP=0|PP=0|TP=0]: iteration: 798 / 150000 | consumed_samples: 51072 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782125E+05 | tokens_per_sec_per_gpu: 9.034571E+03 | global_batch_size: 64 | lm_loss: 1.109574E+01 | lr: 1.197E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 4285849.500 [ip-26-0-154-121:0]:06/21/2023 17:25:00 [INFO|DP=0|PP=0|TP=0]: iteration: 799 / 150000 | consumed_samples: 51136 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779439E+05 | tokens_per_sec_per_gpu: 9.030373E+03 | global_batch_size: 64 | lm_loss: 1.107949E+01 | lr: 1.198E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4375242.500 [ip-26-0-154-121:0]:06/21/2023 17:25:01 [INFO|DP=0|PP=0|TP=0]: iteration: 800 / 150000 | consumed_samples: 51200 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784491E+05 | tokens_per_sec_per_gpu: 9.038267E+03 | global_batch_size: 64 | lm_loss: 1.110254E+01 | lr: 1.200E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 4402053.500 [ip-26-0-154-121:0]:06/21/2023 17:25:02 [INFO|DP=0|PP=0|TP=0]: iteration: 801 / 150000 | consumed_samples: 51264 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780311E+05 | tokens_per_sec_per_gpu: 9.031735E+03 | global_batch_size: 64 | lm_loss: 1.106985E+01 | lr: 1.201E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 4422496.000 [ip-26-0-154-121:0]:06/21/2023 17:25:03 [INFO|DP=0|PP=0|TP=0]: iteration: 802 / 150000 | consumed_samples: 51328 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778365E+05 | tokens_per_sec_per_gpu: 9.028695E+03 | global_batch_size: 64 | lm_loss: 1.107518E+01 | lr: 1.203E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 4144988.250 [ip-26-0-154-121:0]:06/21/2023 17:25:04 [INFO|DP=0|PP=0|TP=0]: iteration: 803 / 150000 | consumed_samples: 51392 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781054E+05 | tokens_per_sec_per_gpu: 9.032897E+03 | global_batch_size: 64 | lm_loss: 1.108328E+01 | lr: 1.204E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 4503145.500 [ip-26-0-154-121:0]:06/21/2023 17:25:05 [INFO|DP=0|PP=0|TP=0]: iteration: 804 / 150000 | consumed_samples: 51456 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784096E+05 | tokens_per_sec_per_gpu: 9.037651E+03 | global_batch_size: 64 | lm_loss: 1.109089E+01 | lr: 1.206E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 4520949.000 [ip-26-0-154-121:0]:06/21/2023 17:25:06 [INFO|DP=0|PP=0|TP=0]: iteration: 805 / 150000 | consumed_samples: 51520 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778635E+05 | tokens_per_sec_per_gpu: 9.029118E+03 | global_batch_size: 64 | lm_loss: 1.108093E+01 | lr: 1.207E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 4208819.000 [ip-26-0-154-121:0]:06/21/2023 17:25:07 [INFO|DP=0|PP=0|TP=0]: iteration: 806 / 150000 | consumed_samples: 51584 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779560E+05 | tokens_per_sec_per_gpu: 9.030563E+03 | global_batch_size: 64 | lm_loss: 1.107684E+01 | lr: 1.209E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4375925.500 [ip-26-0-154-121:0]:06/21/2023 17:25:08 [INFO|DP=0|PP=0|TP=0]: iteration: 807 / 150000 | consumed_samples: 51648 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778658E+05 | tokens_per_sec_per_gpu: 9.029153E+03 | global_batch_size: 64 | lm_loss: 1.105963E+01 | lr: 1.210E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 4451671.000 [ip-26-0-154-121:0]:06/21/2023 17:25:09 [INFO|DP=0|PP=0|TP=0]: iteration: 808 / 150000 | consumed_samples: 51712 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780049E+05 | tokens_per_sec_per_gpu: 9.031327E+03 | global_batch_size: 64 | lm_loss: 1.106300E+01 | lr: 1.212E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4484642.500 [ip-26-0-154-121:0]:06/21/2023 17:25:09 [INFO|DP=0|PP=0|TP=0]: iteration: 809 / 150000 | consumed_samples: 51776 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779389E+05 | tokens_per_sec_per_gpu: 9.030295E+03 | global_batch_size: 64 | lm_loss: 1.107083E+01 | lr: 1.213E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4200702.000 [ip-26-0-154-121:0]:06/21/2023 17:25:10 [INFO|DP=0|PP=0|TP=0]: iteration: 810 / 150000 | consumed_samples: 51840 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781572E+05 | tokens_per_sec_per_gpu: 9.033706E+03 | global_batch_size: 64 | lm_loss: 1.108468E+01 | lr: 1.215E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 4226787.000 [ip-26-0-154-121:0]:06/21/2023 17:25:11 [INFO|DP=0|PP=0|TP=0]: iteration: 811 / 150000 | consumed_samples: 51904 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778298E+05 | tokens_per_sec_per_gpu: 9.028591E+03 | global_batch_size: 64 | lm_loss: 1.105741E+01 | lr: 1.216E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 4497419.500 [ip-26-0-154-121:0]:06/21/2023 17:25:12 [INFO|DP=0|PP=0|TP=0]: iteration: 812 / 150000 | consumed_samples: 51968 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776618E+05 | tokens_per_sec_per_gpu: 9.025965E+03 | global_batch_size: 64 | lm_loss: 1.108637E+01 | lr: 1.218E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 4349615.500 [ip-26-0-154-121:0]:06/21/2023 17:25:13 [INFO|DP=0|PP=0|TP=0]: iteration: 813 / 150000 | consumed_samples: 52032 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783105E+05 | tokens_per_sec_per_gpu: 9.036101E+03 | global_batch_size: 64 | lm_loss: 1.107430E+01 | lr: 1.219E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 4390760.000 [ip-26-0-154-121:0]:06/21/2023 17:25:14 [INFO|DP=0|PP=0|TP=0]: iteration: 814 / 150000 | consumed_samples: 52096 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784320E+05 | tokens_per_sec_per_gpu: 9.038000E+03 | global_batch_size: 64 | lm_loss: 1.108160E+01 | lr: 1.221E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 4508034.000 [ip-26-0-154-121:0]:06/21/2023 17:25:15 [INFO|DP=0|PP=0|TP=0]: iteration: 815 / 150000 | consumed_samples: 52160 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787088E+05 | tokens_per_sec_per_gpu: 9.042324E+03 | global_batch_size: 64 | lm_loss: 1.108835E+01 | lr: 1.222E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 4494183.500 [ip-26-0-154-121:0]:06/21/2023 17:25:16 [INFO|DP=0|PP=0|TP=0]: iteration: 816 / 150000 | consumed_samples: 52224 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787720E+05 | tokens_per_sec_per_gpu: 9.043312E+03 | global_batch_size: 64 | lm_loss: 1.107460E+01 | lr: 1.224E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 4399013.500 [ip-26-0-154-121:0]:06/21/2023 17:25:17 [INFO|DP=0|PP=0|TP=0]: iteration: 817 / 150000 | consumed_samples: 52288 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788765E+05 | tokens_per_sec_per_gpu: 9.044945E+03 | global_batch_size: 64 | lm_loss: 1.109002E+01 | lr: 1.225E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 4333888.500 [ip-26-0-154-121:0]:06/21/2023 17:25:18 [INFO|DP=0|PP=0|TP=0]: iteration: 818 / 150000 | consumed_samples: 52352 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788937E+05 | tokens_per_sec_per_gpu: 9.045214E+03 | global_batch_size: 64 | lm_loss: 1.105940E+01 | lr: 1.227E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 4291319.000 [ip-26-0-154-121:0]:06/21/2023 17:25:19 [INFO|DP=0|PP=0|TP=0]: iteration: 819 / 150000 | consumed_samples: 52416 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785216E+05 | tokens_per_sec_per_gpu: 9.039401E+03 | global_batch_size: 64 | lm_loss: 1.109349E+01 | lr: 1.228E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 4207173.500 [ip-26-0-154-121:0]:06/21/2023 17:25:19 [INFO|DP=0|PP=0|TP=0]: iteration: 820 / 150000 | consumed_samples: 52480 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779986E+05 | tokens_per_sec_per_gpu: 9.031227E+03 | global_batch_size: 64 | lm_loss: 1.108241E+01 | lr: 1.230E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4392546.500 [ip-26-0-154-121:0]:06/21/2023 17:25:20 [INFO|DP=0|PP=0|TP=0]: iteration: 821 / 150000 | consumed_samples: 52544 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784830E+05 | tokens_per_sec_per_gpu: 9.038797E+03 | global_batch_size: 64 | lm_loss: 1.108219E+01 | lr: 1.231E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 4211894.000 [ip-26-0-154-121:0]:06/21/2023 17:25:21 [INFO|DP=0|PP=0|TP=0]: iteration: 822 / 150000 | consumed_samples: 52608 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784924E+05 | tokens_per_sec_per_gpu: 9.038944E+03 | global_batch_size: 64 | lm_loss: 1.107593E+01 | lr: 1.233E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 4490309.000 [ip-26-0-154-121:0]:06/21/2023 17:25:22 [INFO|DP=0|PP=0|TP=0]: iteration: 823 / 150000 | consumed_samples: 52672 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781976E+05 | tokens_per_sec_per_gpu: 9.034338E+03 | global_batch_size: 64 | lm_loss: 1.108171E+01 | lr: 1.234E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 4239643.500 [ip-26-0-154-121:0]:06/21/2023 17:25:23 [INFO|DP=0|PP=0|TP=0]: iteration: 824 / 150000 | consumed_samples: 52736 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.775961E+05 | tokens_per_sec_per_gpu: 9.024939E+03 | global_batch_size: 64 | lm_loss: 1.108640E+01 | lr: 1.236E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 4915997.000 [ip-26-0-154-121:0]:06/21/2023 17:25:24 [INFO|DP=0|PP=0|TP=0]: iteration: 825 / 150000 | consumed_samples: 52800 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777553E+05 | tokens_per_sec_per_gpu: 9.027426E+03 | global_batch_size: 64 | lm_loss: 1.108975E+01 | lr: 1.237E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 4280338.000 [ip-26-0-154-121:0]:06/21/2023 17:25:25 [INFO|DP=0|PP=0|TP=0]: iteration: 826 / 150000 | consumed_samples: 52864 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781153E+05 | tokens_per_sec_per_gpu: 9.033051E+03 | global_batch_size: 64 | lm_loss: 1.108895E+01 | lr: 1.239E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 4478551.000 [ip-26-0-154-121:0]:06/21/2023 17:25:26 [INFO|DP=0|PP=0|TP=0]: iteration: 827 / 150000 | consumed_samples: 52928 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776685E+05 | tokens_per_sec_per_gpu: 9.026070E+03 | global_batch_size: 64 | lm_loss: 1.107594E+01 | lr: 1.241E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 4420768.000 [ip-26-0-154-121:0]:06/21/2023 17:25:27 [INFO|DP=0|PP=0|TP=0]: iteration: 828 / 150000 | consumed_samples: 52992 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776380E+05 | tokens_per_sec_per_gpu: 9.025593E+03 | global_batch_size: 64 | lm_loss: 1.107679E+01 | lr: 1.242E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 4162234.000 [ip-26-0-154-121:0]:06/21/2023 17:25:28 [INFO|DP=0|PP=0|TP=0]: iteration: 829 / 150000 | consumed_samples: 53056 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778272E+05 | tokens_per_sec_per_gpu: 9.028551E+03 | global_batch_size: 64 | lm_loss: 1.107382E+01 | lr: 1.243E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 5175934.000 [ip-26-0-154-121:0]:06/21/2023 17:25:29 [INFO|DP=0|PP=0|TP=0]: iteration: 830 / 150000 | consumed_samples: 53120 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778813E+05 | tokens_per_sec_per_gpu: 9.029395E+03 | global_batch_size: 64 | lm_loss: 1.108215E+01 | lr: 1.245E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 4586069.500 [ip-26-0-154-121:0]:06/21/2023 17:25:29 [INFO|DP=0|PP=0|TP=0]: iteration: 831 / 150000 | consumed_samples: 53184 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779502E+05 | tokens_per_sec_per_gpu: 9.030473E+03 | global_batch_size: 64 | lm_loss: 1.108727E+01 | lr: 1.246E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4851493.500 [ip-26-0-154-121:0]:06/21/2023 17:25:30 [INFO|DP=0|PP=0|TP=0]: iteration: 832 / 150000 | consumed_samples: 53248 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779475E+05 | tokens_per_sec_per_gpu: 9.030430E+03 | global_batch_size: 64 | lm_loss: 1.108348E+01 | lr: 1.248E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4946854.500 [ip-26-0-154-121:0]:06/21/2023 17:25:31 [INFO|DP=0|PP=0|TP=0]: iteration: 833 / 150000 | consumed_samples: 53312 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779695E+05 | tokens_per_sec_per_gpu: 9.030774E+03 | global_batch_size: 64 | lm_loss: 1.107746E+01 | lr: 1.250E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 5439503.000 [ip-26-0-154-121:0]:06/21/2023 17:25:32 [INFO|DP=0|PP=0|TP=0]: iteration: 834 / 150000 | consumed_samples: 53376 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775413E+05 | tokens_per_sec_per_gpu: 9.024083E+03 | global_batch_size: 64 | lm_loss: 1.106514E+01 | lr: 1.251E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 5566776.500 [ip-26-0-154-121:0]:06/21/2023 17:25:33 [INFO|DP=0|PP=0|TP=0]: iteration: 835 / 150000 | consumed_samples: 53440 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.773964E+05 | tokens_per_sec_per_gpu: 9.021818E+03 | global_batch_size: 64 | lm_loss: 1.106485E+01 | lr: 1.253E-04 | model_tflops_per_gpu: 104.21 | hardware_tflops_per_gpu: 104.21 | grad_norm: 4940359.500 [ip-26-0-154-121:0]:06/21/2023 17:25:34 [INFO|DP=0|PP=0|TP=0]: iteration: 836 / 150000 | consumed_samples: 53504 | elapsed_time_per_iteration_ms: 908.2 | tokens_per_sec: 5.772915E+05 | tokens_per_sec_per_gpu: 9.020179E+03 | global_batch_size: 64 | lm_loss: 1.107392E+01 | lr: 1.254E-04 | model_tflops_per_gpu: 104.19 | hardware_tflops_per_gpu: 104.19 | grad_norm: 4687260.000 [ip-26-0-154-121:0]:06/21/2023 17:25:35 [INFO|DP=0|PP=0|TP=0]: iteration: 837 / 150000 | consumed_samples: 53568 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776392E+05 | tokens_per_sec_per_gpu: 9.025612E+03 | global_batch_size: 64 | lm_loss: 1.106736E+01 | lr: 1.255E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 4945640.500 [ip-26-0-154-121:0]:06/21/2023 17:25:36 [INFO|DP=0|PP=0|TP=0]: iteration: 838 / 150000 | consumed_samples: 53632 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777465E+05 | tokens_per_sec_per_gpu: 9.027289E+03 | global_batch_size: 64 | lm_loss: 1.105689E+01 | lr: 1.257E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 4673061.500 [ip-26-0-154-121:0]:06/21/2023 17:25:37 [INFO|DP=0|PP=0|TP=0]: iteration: 839 / 150000 | consumed_samples: 53696 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776427E+05 | tokens_per_sec_per_gpu: 9.025667E+03 | global_batch_size: 64 | lm_loss: 1.106086E+01 | lr: 1.258E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 4964081.500 [ip-26-0-154-121:0]:06/21/2023 17:25:38 [INFO|DP=0|PP=0|TP=0]: iteration: 840 / 150000 | consumed_samples: 53760 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777410E+05 | tokens_per_sec_per_gpu: 9.027203E+03 | global_batch_size: 64 | lm_loss: 1.106793E+01 | lr: 1.260E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 4581067.000 [ip-26-0-154-121:0]:06/21/2023 17:25:39 [INFO|DP=0|PP=0|TP=0]: iteration: 841 / 150000 | consumed_samples: 53824 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.775802E+05 | tokens_per_sec_per_gpu: 9.024690E+03 | global_batch_size: 64 | lm_loss: 1.106957E+01 | lr: 1.261E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 4348064.000 [ip-26-0-154-121:0]:06/21/2023 17:25:39 [INFO|DP=0|PP=0|TP=0]: iteration: 842 / 150000 | consumed_samples: 53888 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777620E+05 | tokens_per_sec_per_gpu: 9.027531E+03 | global_batch_size: 64 | lm_loss: 1.106756E+01 | lr: 1.263E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 4927729.000 [ip-26-0-154-121:0]:06/21/2023 17:25:40 [INFO|DP=0|PP=0|TP=0]: iteration: 843 / 150000 | consumed_samples: 53952 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782978E+05 | tokens_per_sec_per_gpu: 9.035904E+03 | global_batch_size: 64 | lm_loss: 1.107554E+01 | lr: 1.264E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 5073017.000 [ip-26-0-154-121:0]:06/21/2023 17:25:41 [INFO|DP=0|PP=0|TP=0]: iteration: 844 / 150000 | consumed_samples: 54016 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779510E+05 | tokens_per_sec_per_gpu: 9.030485E+03 | global_batch_size: 64 | lm_loss: 1.108278E+01 | lr: 1.266E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4993174.500 [ip-26-0-154-121:0]:06/21/2023 17:25:42 [INFO|DP=0|PP=0|TP=0]: iteration: 845 / 150000 | consumed_samples: 54080 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775233E+05 | tokens_per_sec_per_gpu: 9.023801E+03 | global_batch_size: 64 | lm_loss: 1.107355E+01 | lr: 1.267E-04 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 4826978.000 [ip-26-0-154-121:0]:06/21/2023 17:25:43 [INFO|DP=0|PP=0|TP=0]: iteration: 846 / 150000 | consumed_samples: 54144 | elapsed_time_per_iteration_ms: 908.2 | tokens_per_sec: 5.772565E+05 | tokens_per_sec_per_gpu: 9.019632E+03 | global_batch_size: 64 | lm_loss: 1.106353E+01 | lr: 1.269E-04 | model_tflops_per_gpu: 104.19 | hardware_tflops_per_gpu: 104.19 | grad_norm: 5503846.000 [ip-26-0-154-121:0]:06/21/2023 17:25:44 [INFO|DP=0|PP=0|TP=0]: iteration: 847 / 150000 | consumed_samples: 54208 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774646E+05 | tokens_per_sec_per_gpu: 9.022884E+03 | global_batch_size: 64 | lm_loss: 1.106437E+01 | lr: 1.270E-04 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 5458714.000 [ip-26-0-154-121:0]:06/21/2023 17:25:45 [INFO|DP=0|PP=0|TP=0]: iteration: 848 / 150000 | consumed_samples: 54272 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774523E+05 | tokens_per_sec_per_gpu: 9.022692E+03 | global_batch_size: 64 | lm_loss: 1.106674E+01 | lr: 1.272E-04 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 5149203.000 [ip-26-0-154-121:0]:06/21/2023 17:25:46 [INFO|DP=0|PP=0|TP=0]: iteration: 849 / 150000 | consumed_samples: 54336 | elapsed_time_per_iteration_ms: 908.3 | tokens_per_sec: 5.772325E+05 | tokens_per_sec_per_gpu: 9.019258E+03 | global_batch_size: 64 | lm_loss: 1.102006E+01 | lr: 1.273E-04 | model_tflops_per_gpu: 104.18 | hardware_tflops_per_gpu: 104.18 | grad_norm: 5045661.500 [ip-26-0-154-121:0]:06/21/2023 17:25:47 [INFO|DP=0|PP=0|TP=0]: iteration: 850 / 150000 | consumed_samples: 54400 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782183E+05 | tokens_per_sec_per_gpu: 9.034661E+03 | global_batch_size: 64 | lm_loss: 1.100624E+01 | lr: 1.275E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 5343421.500 [ip-26-0-154-121:0]:06/21/2023 17:25:48 [INFO|DP=0|PP=0|TP=0]: iteration: 851 / 150000 | consumed_samples: 54464 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779158E+05 | tokens_per_sec_per_gpu: 9.029934E+03 | global_batch_size: 64 | lm_loss: 1.105698E+01 | lr: 1.276E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 5286958.000 [ip-26-0-154-121:0]:06/21/2023 17:25:49 [INFO|DP=0|PP=0|TP=0]: iteration: 852 / 150000 | consumed_samples: 54528 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780391E+05 | tokens_per_sec_per_gpu: 9.031861E+03 | global_batch_size: 64 | lm_loss: 1.105851E+01 | lr: 1.278E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 5099182.000 [ip-26-0-154-121:0]:06/21/2023 17:25:49 [INFO|DP=0|PP=0|TP=0]: iteration: 853 / 150000 | consumed_samples: 54592 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782518E+05 | tokens_per_sec_per_gpu: 9.035184E+03 | global_batch_size: 64 | lm_loss: 1.108663E+01 | lr: 1.279E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 4856174.500 [ip-26-0-154-121:0]:06/21/2023 17:25:50 [INFO|DP=0|PP=0|TP=0]: iteration: 854 / 150000 | consumed_samples: 54656 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778790E+05 | tokens_per_sec_per_gpu: 9.029360E+03 | global_batch_size: 64 | lm_loss: 1.106349E+01 | lr: 1.281E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 4623857.000 [ip-26-0-154-121:0]:06/21/2023 17:25:51 [INFO|DP=0|PP=0|TP=0]: iteration: 855 / 150000 | consumed_samples: 54720 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778579E+05 | tokens_per_sec_per_gpu: 9.029030E+03 | global_batch_size: 64 | lm_loss: 1.106553E+01 | lr: 1.282E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 4369893.000 [ip-26-0-154-121:0]:06/21/2023 17:25:52 [INFO|DP=0|PP=0|TP=0]: iteration: 856 / 150000 | consumed_samples: 54784 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779199E+05 | tokens_per_sec_per_gpu: 9.029998E+03 | global_batch_size: 64 | lm_loss: 1.109706E+01 | lr: 1.284E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4330146.000 [ip-26-0-154-121:0]:06/21/2023 17:25:53 [INFO|DP=0|PP=0|TP=0]: iteration: 857 / 150000 | consumed_samples: 54848 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775406E+05 | tokens_per_sec_per_gpu: 9.024071E+03 | global_batch_size: 64 | lm_loss: 1.106717E+01 | lr: 1.286E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 4604804.500 [ip-26-0-154-121:0]:06/21/2023 17:25:54 [INFO|DP=0|PP=0|TP=0]: iteration: 858 / 150000 | consumed_samples: 54912 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777527E+05 | tokens_per_sec_per_gpu: 9.027386E+03 | global_batch_size: 64 | lm_loss: 1.103514E+01 | lr: 1.287E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 4509264.500 [ip-26-0-154-121:0]:06/21/2023 17:25:55 [INFO|DP=0|PP=0|TP=0]: iteration: 859 / 150000 | consumed_samples: 54976 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777826E+05 | tokens_per_sec_per_gpu: 9.027853E+03 | global_batch_size: 64 | lm_loss: 1.107402E+01 | lr: 1.288E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 4451527.500 [ip-26-0-154-121:0]:06/21/2023 17:25:56 [INFO|DP=0|PP=0|TP=0]: iteration: 860 / 150000 | consumed_samples: 55040 | elapsed_time_per_iteration_ms: 908.3 | tokens_per_sec: 5.772490E+05 | tokens_per_sec_per_gpu: 9.019516E+03 | global_batch_size: 64 | lm_loss: 1.106558E+01 | lr: 1.290E-04 | model_tflops_per_gpu: 104.19 | hardware_tflops_per_gpu: 104.19 | grad_norm: 4547733.500 [ip-26-0-154-121:0]:06/21/2023 17:25:57 [INFO|DP=0|PP=0|TP=0]: iteration: 861 / 150000 | consumed_samples: 55104 | elapsed_time_per_iteration_ms: 908.4 | tokens_per_sec: 5.771374E+05 | tokens_per_sec_per_gpu: 9.017771E+03 | global_batch_size: 64 | lm_loss: 1.106435E+01 | lr: 1.291E-04 | model_tflops_per_gpu: 104.17 | hardware_tflops_per_gpu: 104.17 | grad_norm: 4350359.000 [ip-26-0-154-121:0]:06/21/2023 17:25:58 [INFO|DP=0|PP=0|TP=0]: iteration: 862 / 150000 | consumed_samples: 55168 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776547E+05 | tokens_per_sec_per_gpu: 9.025854E+03 | global_batch_size: 64 | lm_loss: 1.106973E+01 | lr: 1.293E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 4520912.000 [ip-26-0-154-121:0]:06/21/2023 17:25:59 [INFO|DP=0|PP=0|TP=0]: iteration: 863 / 150000 | consumed_samples: 55232 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775676E+05 | tokens_per_sec_per_gpu: 9.024493E+03 | global_batch_size: 64 | lm_loss: 1.106545E+01 | lr: 1.294E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 4484229.500 [ip-26-0-154-121:0]:06/21/2023 17:25:59 [INFO|DP=0|PP=0|TP=0]: iteration: 864 / 150000 | consumed_samples: 55296 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779109E+05 | tokens_per_sec_per_gpu: 9.029858E+03 | global_batch_size: 64 | lm_loss: 1.105872E+01 | lr: 1.296E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 4392137.500 [ip-26-0-154-121:0]:06/21/2023 17:26:00 [INFO|DP=0|PP=0|TP=0]: iteration: 865 / 150000 | consumed_samples: 55360 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.773788E+05 | tokens_per_sec_per_gpu: 9.021543E+03 | global_batch_size: 64 | lm_loss: 1.105324E+01 | lr: 1.297E-04 | model_tflops_per_gpu: 104.21 | hardware_tflops_per_gpu: 104.21 | grad_norm: 4671141.000 [ip-26-0-154-121:0]:06/21/2023 17:26:01 [INFO|DP=0|PP=0|TP=0]: iteration: 866 / 150000 | consumed_samples: 55424 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783210E+05 | tokens_per_sec_per_gpu: 9.036265E+03 | global_batch_size: 64 | lm_loss: 1.103864E+01 | lr: 1.299E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 4461153.000 [ip-26-0-154-121:0]:06/21/2023 17:26:02 [INFO|DP=0|PP=0|TP=0]: iteration: 867 / 150000 | consumed_samples: 55488 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783897E+05 | tokens_per_sec_per_gpu: 9.037339E+03 | global_batch_size: 64 | lm_loss: 1.106581E+01 | lr: 1.300E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 4396991.000 [ip-26-0-154-121:0]:06/21/2023 17:26:03 [INFO|DP=0|PP=0|TP=0]: iteration: 868 / 150000 | consumed_samples: 55552 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.778898E+05 | tokens_per_sec_per_gpu: 9.029528E+03 | global_batch_size: 64 | lm_loss: 1.105674E+01 | lr: 1.302E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 4455731.500 [ip-26-0-154-121:0]:06/21/2023 17:26:04 [INFO|DP=0|PP=0|TP=0]: iteration: 869 / 150000 | consumed_samples: 55616 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777539E+05 | tokens_per_sec_per_gpu: 9.027405E+03 | global_batch_size: 64 | lm_loss: 1.106370E+01 | lr: 1.303E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 4473328.500 [ip-26-0-154-121:0]:06/21/2023 17:26:05 [INFO|DP=0|PP=0|TP=0]: iteration: 870 / 150000 | consumed_samples: 55680 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780387E+05 | tokens_per_sec_per_gpu: 9.031854E+03 | global_batch_size: 64 | lm_loss: 1.105004E+01 | lr: 1.305E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 4297849.000 [ip-26-0-154-121:0]:06/21/2023 17:26:06 [INFO|DP=0|PP=0|TP=0]: iteration: 871 / 150000 | consumed_samples: 55744 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777914E+05 | tokens_per_sec_per_gpu: 9.027991E+03 | global_batch_size: 64 | lm_loss: 1.106757E+01 | lr: 1.306E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 4373966.500 [ip-26-0-154-121:0]:06/21/2023 17:26:07 [INFO|DP=0|PP=0|TP=0]: iteration: 872 / 150000 | consumed_samples: 55808 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.775735E+05 | tokens_per_sec_per_gpu: 9.024586E+03 | global_batch_size: 64 | lm_loss: 1.105502E+01 | lr: 1.308E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 4508467.000 [ip-26-0-154-121:0]:06/21/2023 17:26:08 [INFO|DP=0|PP=0|TP=0]: iteration: 873 / 150000 | consumed_samples: 55872 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.776129E+05 | tokens_per_sec_per_gpu: 9.025202E+03 | global_batch_size: 64 | lm_loss: 1.106075E+01 | lr: 1.309E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 4283097.000 [ip-26-0-154-121:0]:06/21/2023 17:26:09 [INFO|DP=0|PP=0|TP=0]: iteration: 874 / 150000 | consumed_samples: 55936 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.775826E+05 | tokens_per_sec_per_gpu: 9.024728E+03 | global_batch_size: 64 | lm_loss: 1.105968E+01 | lr: 1.311E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 4296675.500 [ip-26-0-154-121:0]:06/21/2023 17:26:09 [INFO|DP=0|PP=0|TP=0]: iteration: 875 / 150000 | consumed_samples: 56000 | elapsed_time_per_iteration_ms: 908.3 | tokens_per_sec: 5.772480E+05 | tokens_per_sec_per_gpu: 9.019500E+03 | global_batch_size: 64 | lm_loss: 1.105024E+01 | lr: 1.312E-04 | model_tflops_per_gpu: 104.19 | hardware_tflops_per_gpu: 104.19 | grad_norm: 4389974.500 [ip-26-0-154-121:0]:06/21/2023 17:26:10 [INFO|DP=0|PP=0|TP=0]: iteration: 876 / 150000 | consumed_samples: 56064 | elapsed_time_per_iteration_ms: 908.3 | tokens_per_sec: 5.772405E+05 | tokens_per_sec_per_gpu: 9.019384E+03 | global_batch_size: 64 | lm_loss: 1.107530E+01 | lr: 1.314E-04 | model_tflops_per_gpu: 104.18 | hardware_tflops_per_gpu: 104.18 | grad_norm: 4292000.000 [ip-26-0-154-121:0]:06/21/2023 17:26:11 [INFO|DP=0|PP=0|TP=0]: iteration: 877 / 150000 | consumed_samples: 56128 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.776072E+05 | tokens_per_sec_per_gpu: 9.025112E+03 | global_batch_size: 64 | lm_loss: 1.105552E+01 | lr: 1.315E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 3984520.500 [ip-26-0-154-121:0]:06/21/2023 17:26:12 [INFO|DP=0|PP=0|TP=0]: iteration: 878 / 150000 | consumed_samples: 56192 | elapsed_time_per_iteration_ms: 908.6 | tokens_per_sec: 5.770074E+05 | tokens_per_sec_per_gpu: 9.015741E+03 | global_batch_size: 64 | lm_loss: 1.107306E+01 | lr: 1.317E-04 | model_tflops_per_gpu: 104.14 | hardware_tflops_per_gpu: 104.14 | grad_norm: 4310663.000 [ip-26-0-154-121:0]:06/21/2023 17:26:13 [INFO|DP=0|PP=0|TP=0]: iteration: 879 / 150000 | consumed_samples: 56256 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774456E+05 | tokens_per_sec_per_gpu: 9.022588E+03 | global_batch_size: 64 | lm_loss: 1.106327E+01 | lr: 1.319E-04 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 4149975.500 [ip-26-0-154-121:0]:06/21/2023 17:26:14 [INFO|DP=0|PP=0|TP=0]: iteration: 880 / 150000 | consumed_samples: 56320 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777340E+05 | tokens_per_sec_per_gpu: 9.027094E+03 | global_batch_size: 64 | lm_loss: 1.105354E+01 | lr: 1.320E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 4461662.000 [ip-26-0-154-121:0]:06/21/2023 17:26:15 [INFO|DP=0|PP=0|TP=0]: iteration: 881 / 150000 | consumed_samples: 56384 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782610E+05 | tokens_per_sec_per_gpu: 9.035329E+03 | global_batch_size: 64 | lm_loss: 1.106728E+01 | lr: 1.321E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 4837137.500 [ip-26-0-154-121:0]:06/21/2023 17:26:16 [INFO|DP=0|PP=0|TP=0]: iteration: 882 / 150000 | consumed_samples: 56448 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786084E+05 | tokens_per_sec_per_gpu: 9.040756E+03 | global_batch_size: 64 | lm_loss: 1.106596E+01 | lr: 1.323E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4654994.000 [ip-26-0-154-121:0]:06/21/2023 17:26:17 [INFO|DP=0|PP=0|TP=0]: iteration: 883 / 150000 | consumed_samples: 56512 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784725E+05 | tokens_per_sec_per_gpu: 9.038633E+03 | global_batch_size: 64 | lm_loss: 1.106806E+01 | lr: 1.324E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 4259971.500 [ip-26-0-154-121:0]:06/21/2023 17:26:18 [INFO|DP=0|PP=0|TP=0]: iteration: 884 / 150000 | consumed_samples: 56576 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779917E+05 | tokens_per_sec_per_gpu: 9.031121E+03 | global_batch_size: 64 | lm_loss: 1.106659E+01 | lr: 1.326E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4223260.500 [ip-26-0-154-121:0]:06/21/2023 17:26:19 [INFO|DP=0|PP=0|TP=0]: iteration: 885 / 150000 | consumed_samples: 56640 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777659E+05 | tokens_per_sec_per_gpu: 9.027592E+03 | global_batch_size: 64 | lm_loss: 1.106067E+01 | lr: 1.327E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 4481063.500 [ip-26-0-154-121:0]:06/21/2023 17:26:19 [INFO|DP=0|PP=0|TP=0]: iteration: 886 / 150000 | consumed_samples: 56704 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779013E+05 | tokens_per_sec_per_gpu: 9.029708E+03 | global_batch_size: 64 | lm_loss: 1.107136E+01 | lr: 1.329E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 4424912.500 [ip-26-0-154-121:0]:06/21/2023 17:26:20 [INFO|DP=0|PP=0|TP=0]: iteration: 887 / 150000 | consumed_samples: 56768 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779979E+05 | tokens_per_sec_per_gpu: 9.031218E+03 | global_batch_size: 64 | lm_loss: 1.106697E+01 | lr: 1.331E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4185568.250 [ip-26-0-154-121:0]:06/21/2023 17:26:21 [INFO|DP=0|PP=0|TP=0]: iteration: 888 / 150000 | consumed_samples: 56832 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778114E+05 | tokens_per_sec_per_gpu: 9.028304E+03 | global_batch_size: 64 | lm_loss: 1.103541E+01 | lr: 1.332E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 4460850.000 [ip-26-0-154-121:0]:06/21/2023 17:26:22 [INFO|DP=0|PP=0|TP=0]: iteration: 889 / 150000 | consumed_samples: 56896 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777735E+05 | tokens_per_sec_per_gpu: 9.027711E+03 | global_batch_size: 64 | lm_loss: 1.105740E+01 | lr: 1.333E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 4529598.500 [ip-26-0-154-121:0]:06/21/2023 17:26:23 [INFO|DP=0|PP=0|TP=0]: iteration: 890 / 150000 | consumed_samples: 56960 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777506E+05 | tokens_per_sec_per_gpu: 9.027353E+03 | global_batch_size: 64 | lm_loss: 1.104977E+01 | lr: 1.335E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 4185730.500 [ip-26-0-154-121:0]:06/21/2023 17:26:24 [INFO|DP=0|PP=0|TP=0]: iteration: 891 / 150000 | consumed_samples: 57024 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774531E+05 | tokens_per_sec_per_gpu: 9.022704E+03 | global_batch_size: 64 | lm_loss: 1.105899E+01 | lr: 1.336E-04 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 4222483.500 [ip-26-0-154-121:0]:06/21/2023 17:26:25 [INFO|DP=0|PP=0|TP=0]: iteration: 892 / 150000 | consumed_samples: 57088 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785608E+05 | tokens_per_sec_per_gpu: 9.040012E+03 | global_batch_size: 64 | lm_loss: 1.103857E+01 | lr: 1.338E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 4348910.000 [ip-26-0-154-121:0]:06/21/2023 17:26:26 [INFO|DP=0|PP=0|TP=0]: iteration: 893 / 150000 | consumed_samples: 57152 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780975E+05 | tokens_per_sec_per_gpu: 9.032773E+03 | global_batch_size: 64 | lm_loss: 1.104062E+01 | lr: 1.339E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 4838228.000 [ip-26-0-154-121:0]:06/21/2023 17:26:27 [INFO|DP=0|PP=0|TP=0]: iteration: 894 / 150000 | consumed_samples: 57216 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777026E+05 | tokens_per_sec_per_gpu: 9.026603E+03 | global_batch_size: 64 | lm_loss: 1.104873E+01 | lr: 1.341E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 4721386.500 [ip-26-0-154-121:0]:06/21/2023 17:26:28 [INFO|DP=0|PP=0|TP=0]: iteration: 895 / 150000 | consumed_samples: 57280 | elapsed_time_per_iteration_ms: 908.2 | tokens_per_sec: 5.773021E+05 | tokens_per_sec_per_gpu: 9.020345E+03 | global_batch_size: 64 | lm_loss: 1.105358E+01 | lr: 1.342E-04 | model_tflops_per_gpu: 104.20 | hardware_tflops_per_gpu: 104.20 | grad_norm: 5302947.500 [ip-26-0-154-121:0]:06/21/2023 17:26:29 [INFO|DP=0|PP=0|TP=0]: iteration: 896 / 150000 | consumed_samples: 57344 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779472E+05 | tokens_per_sec_per_gpu: 9.030425E+03 | global_batch_size: 64 | lm_loss: 1.106304E+01 | lr: 1.344E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4583216.500 [ip-26-0-154-121:0]:06/21/2023 17:26:29 [INFO|DP=0|PP=0|TP=0]: iteration: 897 / 150000 | consumed_samples: 57408 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778130E+05 | tokens_per_sec_per_gpu: 9.028328E+03 | global_batch_size: 64 | lm_loss: 1.105454E+01 | lr: 1.345E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 5123755.500 [ip-26-0-154-121:0]:06/21/2023 17:26:30 [INFO|DP=0|PP=0|TP=0]: iteration: 898 / 150000 | consumed_samples: 57472 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782382E+05 | tokens_per_sec_per_gpu: 9.034972E+03 | global_batch_size: 64 | lm_loss: 1.105984E+01 | lr: 1.347E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 4760217.500 [ip-26-0-154-121:0]:06/21/2023 17:26:31 [INFO|DP=0|PP=0|TP=0]: iteration: 899 / 150000 | consumed_samples: 57536 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775196E+05 | tokens_per_sec_per_gpu: 9.023744E+03 | global_batch_size: 64 | lm_loss: 1.103402E+01 | lr: 1.348E-04 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 4598441.000 [ip-26-0-154-121:0]:06/21/2023 17:26:32 [INFO|DP=0|PP=0|TP=0]: iteration: 900 / 150000 | consumed_samples: 57600 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776577E+05 | tokens_per_sec_per_gpu: 9.025901E+03 | global_batch_size: 64 | lm_loss: 1.105634E+01 | lr: 1.350E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 4534850.000 [ip-26-0-154-121:0]:06/21/2023 17:26:33 [INFO|DP=0|PP=0|TP=0]: iteration: 901 / 150000 | consumed_samples: 57664 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776566E+05 | tokens_per_sec_per_gpu: 9.025885E+03 | global_batch_size: 64 | lm_loss: 1.105750E+01 | lr: 1.352E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 4447658.500 [ip-26-0-154-121:0]:06/21/2023 17:26:34 [INFO|DP=0|PP=0|TP=0]: iteration: 902 / 150000 | consumed_samples: 57728 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780365E+05 | tokens_per_sec_per_gpu: 9.031821E+03 | global_batch_size: 64 | lm_loss: 1.103120E+01 | lr: 1.353E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 4810054.000 [ip-26-0-154-121:0]:06/21/2023 17:26:35 [INFO|DP=0|PP=0|TP=0]: iteration: 903 / 150000 | consumed_samples: 57792 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775682E+05 | tokens_per_sec_per_gpu: 9.024503E+03 | global_batch_size: 64 | lm_loss: 1.105887E+01 | lr: 1.354E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 4637705.500 [ip-26-0-154-121:0]:06/21/2023 17:26:36 [INFO|DP=0|PP=0|TP=0]: iteration: 904 / 150000 | consumed_samples: 57856 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782940E+05 | tokens_per_sec_per_gpu: 9.035844E+03 | global_batch_size: 64 | lm_loss: 1.105625E+01 | lr: 1.356E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 4859013.500 [ip-26-0-154-121:0]:06/21/2023 17:26:37 [INFO|DP=0|PP=0|TP=0]: iteration: 905 / 150000 | consumed_samples: 57920 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781420E+05 | tokens_per_sec_per_gpu: 9.033469E+03 | global_batch_size: 64 | lm_loss: 1.106245E+01 | lr: 1.357E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 4762977.500 [ip-26-0-154-121:0]:06/21/2023 17:26:38 [INFO|DP=0|PP=0|TP=0]: iteration: 906 / 150000 | consumed_samples: 57984 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779859E+05 | tokens_per_sec_per_gpu: 9.031030E+03 | global_batch_size: 64 | lm_loss: 1.106107E+01 | lr: 1.359E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 5089684.500 [ip-26-0-154-121:0]:06/21/2023 17:26:39 [INFO|DP=0|PP=0|TP=0]: iteration: 907 / 150000 | consumed_samples: 58048 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774502E+05 | tokens_per_sec_per_gpu: 9.022659E+03 | global_batch_size: 64 | lm_loss: 1.104686E+01 | lr: 1.360E-04 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 4941005.000 [ip-26-0-154-121:0]:06/21/2023 17:26:39 [INFO|DP=0|PP=0|TP=0]: iteration: 908 / 150000 | consumed_samples: 58112 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774989E+05 | tokens_per_sec_per_gpu: 9.023420E+03 | global_batch_size: 64 | lm_loss: 1.105655E+01 | lr: 1.362E-04 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 4396268.500 [ip-26-0-154-121:0]:06/21/2023 17:26:40 [INFO|DP=0|PP=0|TP=0]: iteration: 909 / 150000 | consumed_samples: 58176 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.775749E+05 | tokens_per_sec_per_gpu: 9.024607E+03 | global_batch_size: 64 | lm_loss: 1.106489E+01 | lr: 1.364E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 4262868.500 [ip-26-0-154-121:0]:06/21/2023 17:26:41 [INFO|DP=0|PP=0|TP=0]: iteration: 910 / 150000 | consumed_samples: 58240 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775184E+05 | tokens_per_sec_per_gpu: 9.023725E+03 | global_batch_size: 64 | lm_loss: 1.106608E+01 | lr: 1.365E-04 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 5344465.500 [ip-26-0-154-121:0]:06/21/2023 17:26:42 [INFO|DP=0|PP=0|TP=0]: iteration: 911 / 150000 | consumed_samples: 58304 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779574E+05 | tokens_per_sec_per_gpu: 9.030584E+03 | global_batch_size: 64 | lm_loss: 1.106694E+01 | lr: 1.366E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4934296.500 [ip-26-0-154-121:0]:06/21/2023 17:26:43 [INFO|DP=0|PP=0|TP=0]: iteration: 912 / 150000 | consumed_samples: 58368 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778512E+05 | tokens_per_sec_per_gpu: 9.028925E+03 | global_batch_size: 64 | lm_loss: 1.105842E+01 | lr: 1.368E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 5778173.000 [ip-26-0-154-121:0]:06/21/2023 17:26:44 [INFO|DP=0|PP=0|TP=0]: iteration: 913 / 150000 | consumed_samples: 58432 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777460E+05 | tokens_per_sec_per_gpu: 9.027282E+03 | global_batch_size: 64 | lm_loss: 1.106963E+01 | lr: 1.369E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 5359275.500 [ip-26-0-154-121:0]:06/21/2023 17:26:45 [INFO|DP=0|PP=0|TP=0]: iteration: 914 / 150000 | consumed_samples: 58496 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775163E+05 | tokens_per_sec_per_gpu: 9.023692E+03 | global_batch_size: 64 | lm_loss: 1.105890E+01 | lr: 1.371E-04 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 6577899.000 [ip-26-0-154-121:0]:06/21/2023 17:26:46 [INFO|DP=0|PP=0|TP=0]: iteration: 915 / 150000 | consumed_samples: 58560 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.775812E+05 | tokens_per_sec_per_gpu: 9.024707E+03 | global_batch_size: 64 | lm_loss: 1.106331E+01 | lr: 1.372E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 6081236.500 [ip-26-0-154-121:0]:06/21/2023 17:26:47 [INFO|DP=0|PP=0|TP=0]: iteration: 916 / 150000 | consumed_samples: 58624 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775172E+05 | tokens_per_sec_per_gpu: 9.023706E+03 | global_batch_size: 64 | lm_loss: 1.106084E+01 | lr: 1.374E-04 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 5053407.000 [ip-26-0-154-121:0]:06/21/2023 17:26:48 [INFO|DP=0|PP=0|TP=0]: iteration: 917 / 150000 | consumed_samples: 58688 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774434E+05 | tokens_per_sec_per_gpu: 9.022552E+03 | global_batch_size: 64 | lm_loss: 1.106038E+01 | lr: 1.375E-04 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 5260723.000 [ip-26-0-154-121:0]:06/21/2023 17:26:49 [INFO|DP=0|PP=0|TP=0]: iteration: 918 / 150000 | consumed_samples: 58752 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780660E+05 | tokens_per_sec_per_gpu: 9.032282E+03 | global_batch_size: 64 | lm_loss: 1.103524E+01 | lr: 1.377E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 4823753.000 [ip-26-0-154-121:0]:06/21/2023 17:26:49 [INFO|DP=0|PP=0|TP=0]: iteration: 919 / 150000 | consumed_samples: 58816 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777642E+05 | tokens_per_sec_per_gpu: 9.027566E+03 | global_batch_size: 64 | lm_loss: 1.105474E+01 | lr: 1.378E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 5024362.000 [ip-26-0-154-121:0]:06/21/2023 17:26:50 [INFO|DP=0|PP=0|TP=0]: iteration: 920 / 150000 | consumed_samples: 58880 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.776286E+05 | tokens_per_sec_per_gpu: 9.025446E+03 | global_batch_size: 64 | lm_loss: 1.105677E+01 | lr: 1.380E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 4946365.500 [ip-26-0-154-121:0]:06/21/2023 17:26:51 [INFO|DP=0|PP=0|TP=0]: iteration: 921 / 150000 | consumed_samples: 58944 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776852E+05 | tokens_per_sec_per_gpu: 9.026331E+03 | global_batch_size: 64 | lm_loss: 1.105082E+01 | lr: 1.381E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 4652224.000 [ip-26-0-154-121:0]:06/21/2023 17:26:52 [INFO|DP=0|PP=0|TP=0]: iteration: 922 / 150000 | consumed_samples: 59008 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.774290E+05 | tokens_per_sec_per_gpu: 9.022327E+03 | global_batch_size: 64 | lm_loss: 1.105055E+01 | lr: 1.383E-04 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 4971201.000 [ip-26-0-154-121:0]:06/21/2023 17:26:53 [INFO|DP=0|PP=0|TP=0]: iteration: 923 / 150000 | consumed_samples: 59072 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774923E+05 | tokens_per_sec_per_gpu: 9.023318E+03 | global_batch_size: 64 | lm_loss: 1.105304E+01 | lr: 1.384E-04 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 4166789.750 [ip-26-0-154-121:0]:06/21/2023 17:26:54 [INFO|DP=0|PP=0|TP=0]: iteration: 924 / 150000 | consumed_samples: 59136 | elapsed_time_per_iteration_ms: 908.2 | tokens_per_sec: 5.772899E+05 | tokens_per_sec_per_gpu: 9.020155E+03 | global_batch_size: 64 | lm_loss: 1.104917E+01 | lr: 1.386E-04 | model_tflops_per_gpu: 104.19 | hardware_tflops_per_gpu: 104.19 | grad_norm: 4373370.000 [ip-26-0-154-121:0]:06/21/2023 17:26:55 [INFO|DP=0|PP=0|TP=0]: iteration: 925 / 150000 | consumed_samples: 59200 | elapsed_time_per_iteration_ms: 908.4 | tokens_per_sec: 5.771280E+05 | tokens_per_sec_per_gpu: 9.017625E+03 | global_batch_size: 64 | lm_loss: 1.105188E+01 | lr: 1.387E-04 | model_tflops_per_gpu: 104.16 | hardware_tflops_per_gpu: 104.16 | grad_norm: 4040915.500 [ip-26-0-154-121:0]:06/21/2023 17:26:56 [INFO|DP=0|PP=0|TP=0]: iteration: 926 / 150000 | consumed_samples: 59264 | elapsed_time_per_iteration_ms: 908.7 | tokens_per_sec: 5.769587E+05 | tokens_per_sec_per_gpu: 9.014980E+03 | global_batch_size: 64 | lm_loss: 1.105545E+01 | lr: 1.389E-04 | model_tflops_per_gpu: 104.13 | hardware_tflops_per_gpu: 104.13 | grad_norm: 4169251.750 [ip-26-0-154-121:0]:06/21/2023 17:26:57 [INFO|DP=0|PP=0|TP=0]: iteration: 927 / 150000 | consumed_samples: 59328 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774828E+05 | tokens_per_sec_per_gpu: 9.023168E+03 | global_batch_size: 64 | lm_loss: 1.104539E+01 | lr: 1.390E-04 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 4450439.000 [ip-26-0-154-121:0]:06/21/2023 17:26:58 [INFO|DP=0|PP=0|TP=0]: iteration: 928 / 150000 | consumed_samples: 59392 | elapsed_time_per_iteration_ms: 908.2 | tokens_per_sec: 5.773066E+05 | tokens_per_sec_per_gpu: 9.020416E+03 | global_batch_size: 64 | lm_loss: 1.105237E+01 | lr: 1.392E-04 | model_tflops_per_gpu: 104.20 | hardware_tflops_per_gpu: 104.20 | grad_norm: 5213552.000 [ip-26-0-154-121:0]:06/21/2023 17:26:59 [INFO|DP=0|PP=0|TP=0]: iteration: 929 / 150000 | consumed_samples: 59456 | elapsed_time_per_iteration_ms: 908.2 | tokens_per_sec: 5.772969E+05 | tokens_per_sec_per_gpu: 9.020264E+03 | global_batch_size: 64 | lm_loss: 1.104703E+01 | lr: 1.393E-04 | model_tflops_per_gpu: 104.19 | hardware_tflops_per_gpu: 104.19 | grad_norm: 5080251.500 [ip-26-0-154-121:0]:06/21/2023 17:26:59 [INFO|DP=0|PP=0|TP=0]: iteration: 930 / 150000 | consumed_samples: 59520 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778455E+05 | tokens_per_sec_per_gpu: 9.028835E+03 | global_batch_size: 64 | lm_loss: 1.103220E+01 | lr: 1.395E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 4668203.000 [ip-26-0-154-121:0]:06/21/2023 17:27:00 [INFO|DP=0|PP=0|TP=0]: iteration: 931 / 150000 | consumed_samples: 59584 | elapsed_time_per_iteration_ms: 908.4 | tokens_per_sec: 5.771339E+05 | tokens_per_sec_per_gpu: 9.017717E+03 | global_batch_size: 64 | lm_loss: 1.107732E+01 | lr: 1.397E-04 | model_tflops_per_gpu: 104.16 | hardware_tflops_per_gpu: 104.16 | grad_norm: 4499059.000 [ip-26-0-154-121:0]:06/21/2023 17:27:01 [INFO|DP=0|PP=0|TP=0]: iteration: 932 / 150000 | consumed_samples: 59648 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778034E+05 | tokens_per_sec_per_gpu: 9.028178E+03 | global_batch_size: 64 | lm_loss: 1.105481E+01 | lr: 1.398E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 4620935.500 [ip-26-0-154-121:0]:06/21/2023 17:27:02 [INFO|DP=0|PP=0|TP=0]: iteration: 933 / 150000 | consumed_samples: 59712 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779250E+05 | tokens_per_sec_per_gpu: 9.030079E+03 | global_batch_size: 64 | lm_loss: 1.106044E+01 | lr: 1.399E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 3991571.250 [ip-26-0-154-121:0]:06/21/2023 17:27:03 [INFO|DP=0|PP=0|TP=0]: iteration: 934 / 150000 | consumed_samples: 59776 | elapsed_time_per_iteration_ms: 908.5 | tokens_per_sec: 5.771018E+05 | tokens_per_sec_per_gpu: 9.017215E+03 | global_batch_size: 64 | lm_loss: 1.105123E+01 | lr: 1.401E-04 | model_tflops_per_gpu: 104.16 | hardware_tflops_per_gpu: 104.16 | grad_norm: 4390333.000 [ip-26-0-154-121:0]:06/21/2023 17:27:04 [INFO|DP=0|PP=0|TP=0]: iteration: 935 / 150000 | consumed_samples: 59840 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781510E+05 | tokens_per_sec_per_gpu: 9.033609E+03 | global_batch_size: 64 | lm_loss: 1.105937E+01 | lr: 1.402E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 4910827.500 [ip-26-0-154-121:0]:06/21/2023 17:27:05 [INFO|DP=0|PP=0|TP=0]: iteration: 936 / 150000 | consumed_samples: 59904 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779174E+05 | tokens_per_sec_per_gpu: 9.029960E+03 | global_batch_size: 64 | lm_loss: 1.106666E+01 | lr: 1.404E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4162210.500 [ip-26-0-154-121:0]:06/21/2023 17:27:06 [INFO|DP=0|PP=0|TP=0]: iteration: 937 / 150000 | consumed_samples: 59968 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775416E+05 | tokens_per_sec_per_gpu: 9.024088E+03 | global_batch_size: 64 | lm_loss: 1.103620E+01 | lr: 1.405E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 4681246.000 [ip-26-0-154-121:0]:06/21/2023 17:27:07 [INFO|DP=0|PP=0|TP=0]: iteration: 938 / 150000 | consumed_samples: 60032 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782214E+05 | tokens_per_sec_per_gpu: 9.034709E+03 | global_batch_size: 64 | lm_loss: 1.107635E+01 | lr: 1.407E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 4489976.500 [ip-26-0-154-121:0]:06/21/2023 17:27:08 [INFO|DP=0|PP=0|TP=0]: iteration: 939 / 150000 | consumed_samples: 60096 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776633E+05 | tokens_per_sec_per_gpu: 9.025989E+03 | global_batch_size: 64 | lm_loss: 1.105708E+01 | lr: 1.408E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 4138161.250 [ip-26-0-154-121:0]:06/21/2023 17:27:09 [INFO|DP=0|PP=0|TP=0]: iteration: 940 / 150000 | consumed_samples: 60160 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776753E+05 | tokens_per_sec_per_gpu: 9.026176E+03 | global_batch_size: 64 | lm_loss: 1.105845E+01 | lr: 1.410E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 4651311.000 [ip-26-0-154-121:0]:06/21/2023 17:27:09 [INFO|DP=0|PP=0|TP=0]: iteration: 941 / 150000 | consumed_samples: 60224 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781878E+05 | tokens_per_sec_per_gpu: 9.034184E+03 | global_batch_size: 64 | lm_loss: 1.103281E+01 | lr: 1.411E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 4399027.000 [ip-26-0-154-121:0]:06/21/2023 17:27:10 [INFO|DP=0|PP=0|TP=0]: iteration: 942 / 150000 | consumed_samples: 60288 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780232E+05 | tokens_per_sec_per_gpu: 9.031612E+03 | global_batch_size: 64 | lm_loss: 1.104703E+01 | lr: 1.413E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 5594255.500 [ip-26-0-154-121:0]:06/21/2023 17:27:11 [INFO|DP=0|PP=0|TP=0]: iteration: 943 / 150000 | consumed_samples: 60352 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776650E+05 | tokens_per_sec_per_gpu: 9.026015E+03 | global_batch_size: 64 | lm_loss: 1.105407E+01 | lr: 1.414E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 6137829.500 [ip-26-0-154-121:0]:06/21/2023 17:27:12 [INFO|DP=0|PP=0|TP=0]: iteration: 944 / 150000 | consumed_samples: 60416 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782732E+05 | tokens_per_sec_per_gpu: 9.035519E+03 | global_batch_size: 64 | lm_loss: 1.105122E+01 | lr: 1.416E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 5280642.500 [ip-26-0-154-121:0]:06/21/2023 17:27:13 [INFO|DP=0|PP=0|TP=0]: iteration: 945 / 150000 | consumed_samples: 60480 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784337E+05 | tokens_per_sec_per_gpu: 9.038026E+03 | global_batch_size: 64 | lm_loss: 1.103533E+01 | lr: 1.417E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 5999632.000 [ip-26-0-154-121:0]:06/21/2023 17:27:14 [INFO|DP=0|PP=0|TP=0]: iteration: 946 / 150000 | consumed_samples: 60544 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.778995E+05 | tokens_per_sec_per_gpu: 9.029680E+03 | global_batch_size: 64 | lm_loss: 1.102798E+01 | lr: 1.419E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 4949320.000 [ip-26-0-154-121:0]:06/21/2023 17:27:15 [INFO|DP=0|PP=0|TP=0]: iteration: 947 / 150000 | consumed_samples: 60608 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774634E+05 | tokens_per_sec_per_gpu: 9.022865E+03 | global_batch_size: 64 | lm_loss: 1.104420E+01 | lr: 1.420E-04 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 4216593.500 [ip-26-0-154-121:0]:06/21/2023 17:27:16 [INFO|DP=0|PP=0|TP=0]: iteration: 948 / 150000 | consumed_samples: 60672 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777182E+05 | tokens_per_sec_per_gpu: 9.026848E+03 | global_batch_size: 64 | lm_loss: 1.104531E+01 | lr: 1.422E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 4640843.500 [ip-26-0-154-121:0]:06/21/2023 17:27:17 [INFO|DP=0|PP=0|TP=0]: iteration: 949 / 150000 | consumed_samples: 60736 | elapsed_time_per_iteration_ms: 908.3 | tokens_per_sec: 5.772310E+05 | tokens_per_sec_per_gpu: 9.019234E+03 | global_batch_size: 64 | lm_loss: 1.105121E+01 | lr: 1.423E-04 | model_tflops_per_gpu: 104.18 | hardware_tflops_per_gpu: 104.18 | grad_norm: 4200136.000 [ip-26-0-154-121:0]:06/21/2023 17:27:18 [INFO|DP=0|PP=0|TP=0]: iteration: 950 / 150000 | consumed_samples: 60800 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775111E+05 | tokens_per_sec_per_gpu: 9.023612E+03 | global_batch_size: 64 | lm_loss: 1.104668E+01 | lr: 1.425E-04 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 4306383.000 [ip-26-0-154-121:0]:06/21/2023 17:27:19 [INFO|DP=0|PP=0|TP=0]: iteration: 951 / 150000 | consumed_samples: 60864 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779815E+05 | tokens_per_sec_per_gpu: 9.030962E+03 | global_batch_size: 64 | lm_loss: 1.105769E+01 | lr: 1.426E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4146096.000 [ip-26-0-154-121:0]:06/21/2023 17:27:19 [INFO|DP=0|PP=0|TP=0]: iteration: 952 / 150000 | consumed_samples: 60928 | elapsed_time_per_iteration_ms: 908.4 | tokens_per_sec: 5.771701E+05 | tokens_per_sec_per_gpu: 9.018283E+03 | global_batch_size: 64 | lm_loss: 1.103989E+01 | lr: 1.428E-04 | model_tflops_per_gpu: 104.17 | hardware_tflops_per_gpu: 104.17 | grad_norm: 4925783.000 [ip-26-0-154-121:0]:06/21/2023 17:27:20 [INFO|DP=0|PP=0|TP=0]: iteration: 953 / 150000 | consumed_samples: 60992 | elapsed_time_per_iteration_ms: 908.2 | tokens_per_sec: 5.773074E+05 | tokens_per_sec_per_gpu: 9.020428E+03 | global_batch_size: 64 | lm_loss: 1.105637E+01 | lr: 1.430E-04 | model_tflops_per_gpu: 104.20 | hardware_tflops_per_gpu: 104.20 | grad_norm: 5057569.000 [ip-26-0-154-121:0]:06/21/2023 17:27:21 [INFO|DP=0|PP=0|TP=0]: iteration: 954 / 150000 | consumed_samples: 61056 | elapsed_time_per_iteration_ms: 908.6 | tokens_per_sec: 5.770500E+05 | tokens_per_sec_per_gpu: 9.016406E+03 | global_batch_size: 64 | lm_loss: 1.105390E+01 | lr: 1.431E-04 | model_tflops_per_gpu: 104.15 | hardware_tflops_per_gpu: 104.15 | grad_norm: 5615692.500 [ip-26-0-154-121:0]:06/21/2023 17:27:22 [INFO|DP=0|PP=0|TP=0]: iteration: 955 / 150000 | consumed_samples: 61120 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781584E+05 | tokens_per_sec_per_gpu: 9.033725E+03 | global_batch_size: 64 | lm_loss: 1.103418E+01 | lr: 1.432E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 4924055.500 [ip-26-0-154-121:0]:06/21/2023 17:27:23 [INFO|DP=0|PP=0|TP=0]: iteration: 956 / 150000 | consumed_samples: 61184 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778275E+05 | tokens_per_sec_per_gpu: 9.028555E+03 | global_batch_size: 64 | lm_loss: 1.105593E+01 | lr: 1.434E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 5538930.000 [ip-26-0-154-121:0]:06/21/2023 17:27:24 [INFO|DP=0|PP=0|TP=0]: iteration: 957 / 150000 | consumed_samples: 61248 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778244E+05 | tokens_per_sec_per_gpu: 9.028506E+03 | global_batch_size: 64 | lm_loss: 1.106048E+01 | lr: 1.435E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 4945081.000 [ip-26-0-154-121:0]:06/21/2023 17:27:25 [INFO|DP=0|PP=0|TP=0]: iteration: 958 / 150000 | consumed_samples: 61312 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777771E+05 | tokens_per_sec_per_gpu: 9.027768E+03 | global_batch_size: 64 | lm_loss: 1.105751E+01 | lr: 1.437E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 7082840.000 [ip-26-0-154-121:0]:06/21/2023 17:27:26 [INFO|DP=0|PP=0|TP=0]: iteration: 959 / 150000 | consumed_samples: 61376 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784620E+05 | tokens_per_sec_per_gpu: 9.038469E+03 | global_batch_size: 64 | lm_loss: 1.103638E+01 | lr: 1.438E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 7365710.000 [ip-26-0-154-121:0]:06/21/2023 17:27:27 [INFO|DP=0|PP=0|TP=0]: iteration: 960 / 150000 | consumed_samples: 61440 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780337E+05 | tokens_per_sec_per_gpu: 9.031776E+03 | global_batch_size: 64 | lm_loss: 1.105355E+01 | lr: 1.440E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 7810774.500 [ip-26-0-154-121:0]:06/21/2023 17:27:28 [INFO|DP=0|PP=0|TP=0]: iteration: 961 / 150000 | consumed_samples: 61504 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785457E+05 | tokens_per_sec_per_gpu: 9.039776E+03 | global_batch_size: 64 | lm_loss: 1.106382E+01 | lr: 1.442E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 7808570.000 [ip-26-0-154-121:0]:06/21/2023 17:27:29 [INFO|DP=0|PP=0|TP=0]: iteration: 962 / 150000 | consumed_samples: 61568 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776551E+05 | tokens_per_sec_per_gpu: 9.025861E+03 | global_batch_size: 64 | lm_loss: 1.106771E+01 | lr: 1.443E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 7147624.500 [ip-26-0-154-121:0]:06/21/2023 17:27:29 [INFO|DP=0|PP=0|TP=0]: iteration: 963 / 150000 | consumed_samples: 61632 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781869E+05 | tokens_per_sec_per_gpu: 9.034170E+03 | global_batch_size: 64 | lm_loss: 1.104662E+01 | lr: 1.444E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 6878478.000 [ip-26-0-154-121:0]:06/21/2023 17:27:30 [INFO|DP=0|PP=0|TP=0]: iteration: 964 / 150000 | consumed_samples: 61696 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782869E+05 | tokens_per_sec_per_gpu: 9.035733E+03 | global_batch_size: 64 | lm_loss: 1.105754E+01 | lr: 1.446E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 5873646.500 [ip-26-0-154-121:0]:06/21/2023 17:27:31 [INFO|DP=0|PP=0|TP=0]: iteration: 965 / 150000 | consumed_samples: 61760 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779386E+05 | tokens_per_sec_per_gpu: 9.030290E+03 | global_batch_size: 64 | lm_loss: 1.103216E+01 | lr: 1.447E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 5423716.500 [ip-26-0-154-121:0]:06/21/2023 17:27:32 [INFO|DP=0|PP=0|TP=0]: iteration: 966 / 150000 | consumed_samples: 61824 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782385E+05 | tokens_per_sec_per_gpu: 9.034977E+03 | global_batch_size: 64 | lm_loss: 1.106184E+01 | lr: 1.449E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 4414424.500 [ip-26-0-154-121:0]:06/21/2023 17:27:33 [INFO|DP=0|PP=0|TP=0]: iteration: 967 / 150000 | consumed_samples: 61888 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779367E+05 | tokens_per_sec_per_gpu: 9.030261E+03 | global_batch_size: 64 | lm_loss: 1.104792E+01 | lr: 1.450E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4793597.000 [ip-26-0-154-121:0]:06/21/2023 17:27:34 [INFO|DP=0|PP=0|TP=0]: iteration: 968 / 150000 | consumed_samples: 61952 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.782034E+05 | tokens_per_sec_per_gpu: 9.034428E+03 | global_batch_size: 64 | lm_loss: 1.102906E+01 | lr: 1.452E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 4934747.500 [ip-26-0-154-121:0]:06/21/2023 17:27:35 [INFO|DP=0|PP=0|TP=0]: iteration: 969 / 150000 | consumed_samples: 62016 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785789E+05 | tokens_per_sec_per_gpu: 9.040295E+03 | global_batch_size: 64 | lm_loss: 1.104741E+01 | lr: 1.453E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4893102.000 [ip-26-0-154-121:0]:06/21/2023 17:27:36 [INFO|DP=0|PP=0|TP=0]: iteration: 970 / 150000 | consumed_samples: 62080 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780013E+05 | tokens_per_sec_per_gpu: 9.031270E+03 | global_batch_size: 64 | lm_loss: 1.103464E+01 | lr: 1.455E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4462233.000 [ip-26-0-154-121:0]:06/21/2023 17:27:37 [INFO|DP=0|PP=0|TP=0]: iteration: 971 / 150000 | consumed_samples: 62144 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775607E+05 | tokens_per_sec_per_gpu: 9.024387E+03 | global_batch_size: 64 | lm_loss: 1.104934E+01 | lr: 1.456E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 4901434.500 [ip-26-0-154-121:0]:06/21/2023 17:27:38 [INFO|DP=0|PP=0|TP=0]: iteration: 972 / 150000 | consumed_samples: 62208 | elapsed_time_per_iteration_ms: 908.4 | tokens_per_sec: 5.771512E+05 | tokens_per_sec_per_gpu: 9.017987E+03 | global_batch_size: 64 | lm_loss: 1.104923E+01 | lr: 1.458E-04 | model_tflops_per_gpu: 104.17 | hardware_tflops_per_gpu: 104.17 | grad_norm: 4753201.000 [ip-26-0-154-121:0]:06/21/2023 17:27:38 [INFO|DP=0|PP=0|TP=0]: iteration: 973 / 150000 | consumed_samples: 62272 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782419E+05 | tokens_per_sec_per_gpu: 9.035030E+03 | global_batch_size: 64 | lm_loss: 1.104487E+01 | lr: 1.459E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 5842515.000 [ip-26-0-154-121:0]:06/21/2023 17:27:39 [INFO|DP=0|PP=0|TP=0]: iteration: 974 / 150000 | consumed_samples: 62336 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776503E+05 | tokens_per_sec_per_gpu: 9.025785E+03 | global_batch_size: 64 | lm_loss: 1.104468E+01 | lr: 1.461E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 5709235.500 [ip-26-0-154-121:0]:06/21/2023 17:27:40 [INFO|DP=0|PP=0|TP=0]: iteration: 975 / 150000 | consumed_samples: 62400 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778244E+05 | tokens_per_sec_per_gpu: 9.028506E+03 | global_batch_size: 64 | lm_loss: 1.104636E+01 | lr: 1.463E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 5623578.500 [ip-26-0-154-121:0]:06/21/2023 17:27:41 [INFO|DP=0|PP=0|TP=0]: iteration: 976 / 150000 | consumed_samples: 62464 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779361E+05 | tokens_per_sec_per_gpu: 9.030252E+03 | global_batch_size: 64 | lm_loss: 1.104203E+01 | lr: 1.464E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 5989263.000 [ip-26-0-154-121:0]:06/21/2023 17:27:42 [INFO|DP=0|PP=0|TP=0]: iteration: 977 / 150000 | consumed_samples: 62528 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778075E+05 | tokens_per_sec_per_gpu: 9.028242E+03 | global_batch_size: 64 | lm_loss: 1.103961E+01 | lr: 1.465E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 5128241.500 [ip-26-0-154-121:0]:06/21/2023 17:27:43 [INFO|DP=0|PP=0|TP=0]: iteration: 978 / 150000 | consumed_samples: 62592 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779376E+05 | tokens_per_sec_per_gpu: 9.030276E+03 | global_batch_size: 64 | lm_loss: 1.101519E+01 | lr: 1.467E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4384235.000 [ip-26-0-154-121:0]:06/21/2023 17:27:44 [INFO|DP=0|PP=0|TP=0]: iteration: 979 / 150000 | consumed_samples: 62656 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780384E+05 | tokens_per_sec_per_gpu: 9.031849E+03 | global_batch_size: 64 | lm_loss: 1.104297E+01 | lr: 1.468E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 4085166.750 [ip-26-0-154-121:0]:06/21/2023 17:27:45 [INFO|DP=0|PP=0|TP=0]: iteration: 980 / 150000 | consumed_samples: 62720 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777069E+05 | tokens_per_sec_per_gpu: 9.026670E+03 | global_batch_size: 64 | lm_loss: 1.105111E+01 | lr: 1.470E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 4373401.500 [ip-26-0-154-121:0]:06/21/2023 17:27:46 [INFO|DP=0|PP=0|TP=0]: iteration: 981 / 150000 | consumed_samples: 62784 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.775979E+05 | tokens_per_sec_per_gpu: 9.024967E+03 | global_batch_size: 64 | lm_loss: 1.103645E+01 | lr: 1.471E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 4604693.500 [ip-26-0-154-121:0]:06/21/2023 17:27:47 [INFO|DP=0|PP=0|TP=0]: iteration: 982 / 150000 | consumed_samples: 62848 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.774205E+05 | tokens_per_sec_per_gpu: 9.022195E+03 | global_batch_size: 64 | lm_loss: 1.105394E+01 | lr: 1.473E-04 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 5274607.000 [ip-26-0-154-121:0]:06/21/2023 17:27:48 [INFO|DP=0|PP=0|TP=0]: iteration: 983 / 150000 | consumed_samples: 62912 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775503E+05 | tokens_per_sec_per_gpu: 9.024223E+03 | global_batch_size: 64 | lm_loss: 1.104951E+01 | lr: 1.475E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 5513626.500 [ip-26-0-154-121:0]:06/21/2023 17:27:48 [INFO|DP=0|PP=0|TP=0]: iteration: 984 / 150000 | consumed_samples: 62976 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776548E+05 | tokens_per_sec_per_gpu: 9.025856E+03 | global_batch_size: 64 | lm_loss: 1.102620E+01 | lr: 1.476E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 4337957.500 [ip-26-0-154-121:0]:06/21/2023 17:27:49 [INFO|DP=0|PP=0|TP=0]: iteration: 985 / 150000 | consumed_samples: 63040 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781051E+05 | tokens_per_sec_per_gpu: 9.032892E+03 | global_batch_size: 64 | lm_loss: 1.104579E+01 | lr: 1.477E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 5494241.000 [ip-26-0-154-121:0]:06/21/2023 17:27:50 [INFO|DP=0|PP=0|TP=0]: iteration: 986 / 150000 | consumed_samples: 63104 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776800E+05 | tokens_per_sec_per_gpu: 9.026250E+03 | global_batch_size: 64 | lm_loss: 1.105024E+01 | lr: 1.479E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 4751620.000 [ip-26-0-154-121:0]:06/21/2023 17:27:51 [INFO|DP=0|PP=0|TP=0]: iteration: 987 / 150000 | consumed_samples: 63168 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776454E+05 | tokens_per_sec_per_gpu: 9.025709E+03 | global_batch_size: 64 | lm_loss: 1.104694E+01 | lr: 1.480E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 4247697.000 [ip-26-0-154-121:0]:06/21/2023 17:27:52 [INFO|DP=0|PP=0|TP=0]: iteration: 988 / 150000 | consumed_samples: 63232 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779317E+05 | tokens_per_sec_per_gpu: 9.030183E+03 | global_batch_size: 64 | lm_loss: 1.105974E+01 | lr: 1.482E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4904758.500 [ip-26-0-154-121:0]:06/21/2023 17:27:53 [INFO|DP=0|PP=0|TP=0]: iteration: 989 / 150000 | consumed_samples: 63296 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782115E+05 | tokens_per_sec_per_gpu: 9.034554E+03 | global_batch_size: 64 | lm_loss: 1.105019E+01 | lr: 1.483E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 4474081.500 [ip-26-0-154-121:0]:06/21/2023 17:27:54 [INFO|DP=0|PP=0|TP=0]: iteration: 990 / 150000 | consumed_samples: 63360 | elapsed_time_per_iteration_ms: 908.2 | tokens_per_sec: 5.773034E+05 | tokens_per_sec_per_gpu: 9.020366E+03 | global_batch_size: 64 | lm_loss: 1.104633E+01 | lr: 1.485E-04 | model_tflops_per_gpu: 104.20 | hardware_tflops_per_gpu: 104.20 | grad_norm: 5662805.500 [ip-26-0-154-121:0]:06/21/2023 17:27:55 [INFO|DP=0|PP=0|TP=0]: iteration: 991 / 150000 | consumed_samples: 63424 | elapsed_time_per_iteration_ms: 908.1 | tokens_per_sec: 5.773662E+05 | tokens_per_sec_per_gpu: 9.021347E+03 | global_batch_size: 64 | lm_loss: 1.103846E+01 | lr: 1.486E-04 | model_tflops_per_gpu: 104.21 | hardware_tflops_per_gpu: 104.21 | grad_norm: 4522195.500 [ip-26-0-154-121:0]:06/21/2023 17:27:56 [INFO|DP=0|PP=0|TP=0]: iteration: 992 / 150000 | consumed_samples: 63488 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.776088E+05 | tokens_per_sec_per_gpu: 9.025138E+03 | global_batch_size: 64 | lm_loss: 1.104517E+01 | lr: 1.488E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 4769709.500 [ip-26-0-154-121:0]:06/21/2023 17:27:57 [INFO|DP=0|PP=0|TP=0]: iteration: 993 / 150000 | consumed_samples: 63552 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779379E+05 | tokens_per_sec_per_gpu: 9.030280E+03 | global_batch_size: 64 | lm_loss: 1.103569E+01 | lr: 1.489E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4986027.500 [ip-26-0-154-121:0]:06/21/2023 17:27:58 [INFO|DP=0|PP=0|TP=0]: iteration: 994 / 150000 | consumed_samples: 63616 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775289E+05 | tokens_per_sec_per_gpu: 9.023889E+03 | global_batch_size: 64 | lm_loss: 1.104778E+01 | lr: 1.491E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 4575967.500 [ip-26-0-154-121:0]:06/21/2023 17:27:58 [INFO|DP=0|PP=0|TP=0]: iteration: 995 / 150000 | consumed_samples: 63680 | elapsed_time_per_iteration_ms: 908.2 | tokens_per_sec: 5.772609E+05 | tokens_per_sec_per_gpu: 9.019701E+03 | global_batch_size: 64 | lm_loss: 1.103697E+01 | lr: 1.492E-04 | model_tflops_per_gpu: 104.19 | hardware_tflops_per_gpu: 104.19 | grad_norm: 4787461.000 [ip-26-0-154-121:0]:06/21/2023 17:27:59 [INFO|DP=0|PP=0|TP=0]: iteration: 996 / 150000 | consumed_samples: 63744 | elapsed_time_per_iteration_ms: 908.1 | tokens_per_sec: 5.773538E+05 | tokens_per_sec_per_gpu: 9.021152E+03 | global_batch_size: 64 | lm_loss: 1.104803E+01 | lr: 1.494E-04 | model_tflops_per_gpu: 104.20 | hardware_tflops_per_gpu: 104.20 | grad_norm: 7285177.000 [ip-26-0-154-121:0]:06/21/2023 17:28:00 [INFO|DP=0|PP=0|TP=0]: iteration: 997 / 150000 | consumed_samples: 63808 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774423E+05 | tokens_per_sec_per_gpu: 9.022536E+03 | global_batch_size: 64 | lm_loss: 1.105041E+01 | lr: 1.495E-04 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 5067756.000 [ip-26-0-154-121:0]:06/21/2023 17:28:01 [INFO|DP=0|PP=0|TP=0]: iteration: 998 / 150000 | consumed_samples: 63872 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775064E+05 | tokens_per_sec_per_gpu: 9.023538E+03 | global_batch_size: 64 | lm_loss: 1.105011E+01 | lr: 1.497E-04 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 5356337.000 [ip-26-0-154-121:0]:06/21/2023 17:28:02 [INFO|DP=0|PP=0|TP=0]: iteration: 999 / 150000 | consumed_samples: 63936 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777569E+05 | tokens_per_sec_per_gpu: 9.027452E+03 | global_batch_size: 64 | lm_loss: 1.103823E+01 | lr: 1.498E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 4793176.000 [ip-26-0-154-121:0]:06/21/2023 17:28:03 [INFO|DP=0|PP=0|TP=0]: iteration: 1000 / 150000 | consumed_samples: 64000 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778160E+05 | tokens_per_sec_per_gpu: 9.028375E+03 | global_batch_size: 64 | lm_loss: 1.102240E+01 | lr: 1.500E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 4933161.500 [ip-26-0-154-121:0]:06/21/2023 17:28:04 [INFO|DP=0|PP=0|TP=0]: iteration: 1001 / 150000 | consumed_samples: 64064 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781802E+05 | tokens_per_sec_per_gpu: 9.034065E+03 | global_batch_size: 64 | lm_loss: 1.104436E+01 | lr: 1.501E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 4887308.500 [ip-26-0-154-121:0]:06/21/2023 17:28:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1002 / 150000 | consumed_samples: 64128 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777703E+05 | tokens_per_sec_per_gpu: 9.027661E+03 | global_batch_size: 64 | lm_loss: 1.103922E+01 | lr: 1.503E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 4990066.000 [ip-26-0-154-121:0]:06/21/2023 17:28:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1003 / 150000 | consumed_samples: 64192 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780312E+05 | tokens_per_sec_per_gpu: 9.031738E+03 | global_batch_size: 64 | lm_loss: 1.104222E+01 | lr: 1.504E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 5725748.000 [ip-26-0-154-121:0]:06/21/2023 17:28:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1004 / 150000 | consumed_samples: 64256 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778098E+05 | tokens_per_sec_per_gpu: 9.028278E+03 | global_batch_size: 64 | lm_loss: 1.103696E+01 | lr: 1.506E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 5546227.500 [ip-26-0-154-121:0]:06/21/2023 17:28:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1005 / 150000 | consumed_samples: 64320 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779437E+05 | tokens_per_sec_per_gpu: 9.030371E+03 | global_batch_size: 64 | lm_loss: 1.102443E+01 | lr: 1.508E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 5655783.500 [ip-26-0-154-121:0]:06/21/2023 17:28:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1006 / 150000 | consumed_samples: 64384 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.773914E+05 | tokens_per_sec_per_gpu: 9.021740E+03 | global_batch_size: 64 | lm_loss: 1.103546E+01 | lr: 1.509E-04 | model_tflops_per_gpu: 104.21 | hardware_tflops_per_gpu: 104.21 | grad_norm: 5279175.500 [ip-26-0-154-121:0]:06/21/2023 17:28:09 [INFO|DP=0|PP=0|TP=0]: iteration: 1007 / 150000 | consumed_samples: 64448 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776604E+05 | tokens_per_sec_per_gpu: 9.025944E+03 | global_batch_size: 64 | lm_loss: 1.103410E+01 | lr: 1.510E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 4941258.500 [ip-26-0-154-121:0]:06/21/2023 17:28:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1008 / 150000 | consumed_samples: 64512 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.773977E+05 | tokens_per_sec_per_gpu: 9.021839E+03 | global_batch_size: 64 | lm_loss: 1.103353E+01 | lr: 1.512E-04 | model_tflops_per_gpu: 104.21 | hardware_tflops_per_gpu: 104.21 | grad_norm: 4785456.000 [ip-26-0-154-121:0]:06/21/2023 17:28:11 [INFO|DP=0|PP=0|TP=0]: iteration: 1009 / 150000 | consumed_samples: 64576 | elapsed_time_per_iteration_ms: 908.1 | tokens_per_sec: 5.773295E+05 | tokens_per_sec_per_gpu: 9.020773E+03 | global_batch_size: 64 | lm_loss: 1.103784E+01 | lr: 1.513E-04 | model_tflops_per_gpu: 104.20 | hardware_tflops_per_gpu: 104.20 | grad_norm: 5026787.500 [ip-26-0-154-121:0]:06/21/2023 17:28:12 [INFO|DP=0|PP=0|TP=0]: iteration: 1010 / 150000 | consumed_samples: 64640 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.775864E+05 | tokens_per_sec_per_gpu: 9.024787E+03 | global_batch_size: 64 | lm_loss: 1.104868E+01 | lr: 1.515E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 4859860.000 [ip-26-0-154-121:0]:06/21/2023 17:28:13 [INFO|DP=0|PP=0|TP=0]: iteration: 1011 / 150000 | consumed_samples: 64704 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777862E+05 | tokens_per_sec_per_gpu: 9.027910E+03 | global_batch_size: 64 | lm_loss: 1.104569E+01 | lr: 1.516E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 5182321.000 [ip-26-0-154-121:0]:06/21/2023 17:28:14 [INFO|DP=0|PP=0|TP=0]: iteration: 1012 / 150000 | consumed_samples: 64768 | elapsed_time_per_iteration_ms: 908.3 | tokens_per_sec: 5.772383E+05 | tokens_per_sec_per_gpu: 9.019348E+03 | global_batch_size: 64 | lm_loss: 1.104769E+01 | lr: 1.518E-04 | model_tflops_per_gpu: 104.18 | hardware_tflops_per_gpu: 104.18 | grad_norm: 5849797.000 [ip-26-0-154-121:0]:06/21/2023 17:28:15 [INFO|DP=0|PP=0|TP=0]: iteration: 1013 / 150000 | consumed_samples: 64832 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777249E+05 | tokens_per_sec_per_gpu: 9.026952E+03 | global_batch_size: 64 | lm_loss: 1.104173E+01 | lr: 1.519E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 5310652.000 [ip-26-0-154-121:0]:06/21/2023 17:28:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1014 / 150000 | consumed_samples: 64896 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777293E+05 | tokens_per_sec_per_gpu: 9.027021E+03 | global_batch_size: 64 | lm_loss: 1.103236E+01 | lr: 1.521E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 5301149.000 [ip-26-0-154-121:0]:06/21/2023 17:28:17 [INFO|DP=0|PP=0|TP=0]: iteration: 1015 / 150000 | consumed_samples: 64960 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777088E+05 | tokens_per_sec_per_gpu: 9.026701E+03 | global_batch_size: 64 | lm_loss: 1.103591E+01 | lr: 1.522E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 5081073.000 [ip-26-0-154-121:0]:06/21/2023 17:28:18 [INFO|DP=0|PP=0|TP=0]: iteration: 1016 / 150000 | consumed_samples: 65024 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.776155E+05 | tokens_per_sec_per_gpu: 9.025242E+03 | global_batch_size: 64 | lm_loss: 1.104205E+01 | lr: 1.524E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 4241087.000 [ip-26-0-154-121:0]:06/21/2023 17:28:18 [INFO|DP=0|PP=0|TP=0]: iteration: 1017 / 150000 | consumed_samples: 65088 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780195E+05 | tokens_per_sec_per_gpu: 9.031555E+03 | global_batch_size: 64 | lm_loss: 1.103729E+01 | lr: 1.525E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 4863748.000 [ip-26-0-154-121:0]:06/21/2023 17:28:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1018 / 150000 | consumed_samples: 65152 | elapsed_time_per_iteration_ms: 909.7 | tokens_per_sec: 5.763026E+05 | tokens_per_sec_per_gpu: 9.004728E+03 | global_batch_size: 64 | lm_loss: 1.104121E+01 | lr: 1.527E-04 | model_tflops_per_gpu: 104.01 | hardware_tflops_per_gpu: 104.01 | grad_norm: 4366984.000 [ip-26-0-154-121:0]:06/21/2023 17:28:20 [INFO|DP=0|PP=0|TP=0]: iteration: 1019 / 150000 | consumed_samples: 65216 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783232E+05 | tokens_per_sec_per_gpu: 9.036301E+03 | global_batch_size: 64 | lm_loss: 1.103881E+01 | lr: 1.528E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 5119866.500 [ip-26-0-154-121:0]:06/21/2023 17:28:21 [INFO|DP=0|PP=0|TP=0]: iteration: 1020 / 150000 | consumed_samples: 65280 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783756E+05 | tokens_per_sec_per_gpu: 9.037118E+03 | global_batch_size: 64 | lm_loss: 1.103380E+01 | lr: 1.530E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 6001104.000 [ip-26-0-154-121:0]:06/21/2023 17:28:22 [INFO|DP=0|PP=0|TP=0]: iteration: 1021 / 150000 | consumed_samples: 65344 | elapsed_time_per_iteration_ms: 908.2 | tokens_per_sec: 5.772598E+05 | tokens_per_sec_per_gpu: 9.019684E+03 | global_batch_size: 64 | lm_loss: 1.102079E+01 | lr: 1.531E-04 | model_tflops_per_gpu: 104.19 | hardware_tflops_per_gpu: 104.19 | grad_norm: 5685377.000 [ip-26-0-154-121:0]:06/21/2023 17:28:23 [INFO|DP=0|PP=0|TP=0]: iteration: 1022 / 150000 | consumed_samples: 65408 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777846E+05 | tokens_per_sec_per_gpu: 9.027884E+03 | global_batch_size: 64 | lm_loss: 1.097738E+01 | lr: 1.533E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 7080647.500 [ip-26-0-154-121:0]:06/21/2023 17:28:24 [INFO|DP=0|PP=0|TP=0]: iteration: 1023 / 150000 | consumed_samples: 65472 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.773939E+05 | tokens_per_sec_per_gpu: 9.021780E+03 | global_batch_size: 64 | lm_loss: 1.103360E+01 | lr: 1.534E-04 | model_tflops_per_gpu: 104.21 | hardware_tflops_per_gpu: 104.21 | grad_norm: 5940419.500 [ip-26-0-154-121:0]:06/21/2023 17:28:25 [INFO|DP=0|PP=0|TP=0]: iteration: 1024 / 150000 | consumed_samples: 65536 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779065E+05 | tokens_per_sec_per_gpu: 9.029789E+03 | global_batch_size: 64 | lm_loss: 1.102683E+01 | lr: 1.536E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 6073995.500 [ip-26-0-154-121:0]:06/21/2023 17:28:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1025 / 150000 | consumed_samples: 65600 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782186E+05 | tokens_per_sec_per_gpu: 9.034666E+03 | global_batch_size: 64 | lm_loss: 1.103150E+01 | lr: 1.537E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 6376969.500 [ip-26-0-154-121:0]:06/21/2023 17:28:27 [INFO|DP=0|PP=0|TP=0]: iteration: 1026 / 150000 | consumed_samples: 65664 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780648E+05 | tokens_per_sec_per_gpu: 9.032263E+03 | global_batch_size: 64 | lm_loss: 1.101984E+01 | lr: 1.539E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 5731925.500 [ip-26-0-154-121:0]:06/21/2023 17:28:28 [INFO|DP=0|PP=0|TP=0]: iteration: 1027 / 150000 | consumed_samples: 65728 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781435E+05 | tokens_per_sec_per_gpu: 9.033493E+03 | global_batch_size: 64 | lm_loss: 1.103815E+01 | lr: 1.541E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 7274199.500 [ip-26-0-154-121:0]:06/21/2023 17:28:28 [INFO|DP=0|PP=0|TP=0]: iteration: 1028 / 150000 | consumed_samples: 65792 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.775724E+05 | tokens_per_sec_per_gpu: 9.024569E+03 | global_batch_size: 64 | lm_loss: 1.102594E+01 | lr: 1.542E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 6449014.000 [ip-26-0-154-121:0]:06/21/2023 17:28:29 [INFO|DP=0|PP=0|TP=0]: iteration: 1029 / 150000 | consumed_samples: 65856 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779483E+05 | tokens_per_sec_per_gpu: 9.030442E+03 | global_batch_size: 64 | lm_loss: 1.102833E+01 | lr: 1.543E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 6252677.500 [ip-26-0-154-121:0]:06/21/2023 17:28:30 [INFO|DP=0|PP=0|TP=0]: iteration: 1030 / 150000 | consumed_samples: 65920 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.778954E+05 | tokens_per_sec_per_gpu: 9.029616E+03 | global_batch_size: 64 | lm_loss: 1.103349E+01 | lr: 1.545E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 7239946.000 [ip-26-0-154-121:0]:06/21/2023 17:28:31 [INFO|DP=0|PP=0|TP=0]: iteration: 1031 / 150000 | consumed_samples: 65984 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782942E+05 | tokens_per_sec_per_gpu: 9.035847E+03 | global_batch_size: 64 | lm_loss: 1.102007E+01 | lr: 1.547E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 7685724.000 [ip-26-0-154-121:0]:06/21/2023 17:28:32 [INFO|DP=0|PP=0|TP=0]: iteration: 1032 / 150000 | consumed_samples: 66048 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.782007E+05 | tokens_per_sec_per_gpu: 9.034386E+03 | global_batch_size: 64 | lm_loss: 1.103927E+01 | lr: 1.548E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 13542614.000 [ip-26-0-154-121:0]:06/21/2023 17:28:33 [INFO|DP=0|PP=0|TP=0]: iteration: 1033 / 150000 | consumed_samples: 66112 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783849E+05 | tokens_per_sec_per_gpu: 9.037263E+03 | global_batch_size: 64 | lm_loss: 1.103986E+01 | lr: 1.549E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 14186396.000 [ip-26-0-154-121:0]:06/21/2023 17:28:34 [INFO|DP=0|PP=0|TP=0]: iteration: 1034 / 150000 | consumed_samples: 66176 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782592E+05 | tokens_per_sec_per_gpu: 9.035300E+03 | global_batch_size: 64 | lm_loss: 1.105385E+01 | lr: 1.551E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 15997368.000 [ip-26-0-154-121:0]:06/21/2023 17:28:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1035 / 150000 | consumed_samples: 66240 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785318E+05 | tokens_per_sec_per_gpu: 9.039560E+03 | global_batch_size: 64 | lm_loss: 1.104609E+01 | lr: 1.553E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 16332384.000 [ip-26-0-154-121:0]:06/21/2023 17:28:36 [INFO|DP=0|PP=0|TP=0]: iteration: 1036 / 150000 | consumed_samples: 66304 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785321E+05 | tokens_per_sec_per_gpu: 9.039565E+03 | global_batch_size: 64 | lm_loss: 1.104056E+01 | lr: 1.554E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 11694816.000 [ip-26-0-154-121:0]:06/21/2023 17:28:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1037 / 150000 | consumed_samples: 66368 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785402E+05 | tokens_per_sec_per_gpu: 9.039691E+03 | global_batch_size: 64 | lm_loss: 1.103688E+01 | lr: 1.555E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 12300040.000 [ip-26-0-154-121:0]:06/21/2023 17:28:38 [INFO|DP=0|PP=0|TP=0]: iteration: 1038 / 150000 | consumed_samples: 66432 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787508E+05 | tokens_per_sec_per_gpu: 9.042981E+03 | global_batch_size: 64 | lm_loss: 1.105069E+01 | lr: 1.557E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 8512822.000 [ip-26-0-154-121:0]:06/21/2023 17:28:38 [INFO|DP=0|PP=0|TP=0]: iteration: 1039 / 150000 | consumed_samples: 66496 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788382E+05 | tokens_per_sec_per_gpu: 9.044347E+03 | global_batch_size: 64 | lm_loss: 1.104337E+01 | lr: 1.559E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 6901790.000 [ip-26-0-154-121:0]:06/21/2023 17:28:39 [INFO|DP=0|PP=0|TP=0]: iteration: 1040 / 150000 | consumed_samples: 66560 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786679E+05 | tokens_per_sec_per_gpu: 9.041687E+03 | global_batch_size: 64 | lm_loss: 1.104006E+01 | lr: 1.560E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 5682064.500 [ip-26-0-154-121:0]:06/21/2023 17:28:40 [INFO|DP=0|PP=0|TP=0]: iteration: 1041 / 150000 | consumed_samples: 66624 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781336E+05 | tokens_per_sec_per_gpu: 9.033338E+03 | global_batch_size: 64 | lm_loss: 1.102208E+01 | lr: 1.561E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 4751510.500 [ip-26-0-154-121:0]:06/21/2023 17:28:41 [INFO|DP=0|PP=0|TP=0]: iteration: 1042 / 150000 | consumed_samples: 66688 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778063E+05 | tokens_per_sec_per_gpu: 9.028223E+03 | global_batch_size: 64 | lm_loss: 1.103757E+01 | lr: 1.563E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 6049413.500 [ip-26-0-154-121:0]:06/21/2023 17:28:42 [INFO|DP=0|PP=0|TP=0]: iteration: 1043 / 150000 | consumed_samples: 66752 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780884E+05 | tokens_per_sec_per_gpu: 9.032631E+03 | global_batch_size: 64 | lm_loss: 1.102871E+01 | lr: 1.564E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 6556423.000 [ip-26-0-154-121:0]:06/21/2023 17:28:43 [INFO|DP=0|PP=0|TP=0]: iteration: 1044 / 150000 | consumed_samples: 66816 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782539E+05 | tokens_per_sec_per_gpu: 9.035217E+03 | global_batch_size: 64 | lm_loss: 1.103728E+01 | lr: 1.566E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 8057823.000 [ip-26-0-154-121:0]:06/21/2023 17:28:44 [INFO|DP=0|PP=0|TP=0]: iteration: 1045 / 150000 | consumed_samples: 66880 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781536E+05 | tokens_per_sec_per_gpu: 9.033649E+03 | global_batch_size: 64 | lm_loss: 1.102675E+01 | lr: 1.567E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 7756297.000 [ip-26-0-154-121:0]:06/21/2023 17:28:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1046 / 150000 | consumed_samples: 66944 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781989E+05 | tokens_per_sec_per_gpu: 9.034357E+03 | global_batch_size: 64 | lm_loss: 1.102652E+01 | lr: 1.569E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 16407186.000 [ip-26-0-154-121:0]:06/21/2023 17:28:46 [INFO|DP=0|PP=0|TP=0]: iteration: 1047 / 150000 | consumed_samples: 67008 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781615E+05 | tokens_per_sec_per_gpu: 9.033773E+03 | global_batch_size: 64 | lm_loss: 1.104065E+01 | lr: 1.571E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 11536549.000 [ip-26-0-154-121:0]:06/21/2023 17:28:47 [INFO|DP=0|PP=0|TP=0]: iteration: 1048 / 150000 | consumed_samples: 67072 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784027E+05 | tokens_per_sec_per_gpu: 9.037541E+03 | global_batch_size: 64 | lm_loss: 1.102884E+01 | lr: 1.572E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 17610086.000 [ip-26-0-154-121:0]:06/21/2023 17:28:48 [INFO|DP=0|PP=0|TP=0]: iteration: 1049 / 150000 | consumed_samples: 67136 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782297E+05 | tokens_per_sec_per_gpu: 9.034839E+03 | global_batch_size: 64 | lm_loss: 1.102817E+01 | lr: 1.574E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 14012641.000 [ip-26-0-154-121:0]:06/21/2023 17:28:48 [INFO|DP=0|PP=0|TP=0]: iteration: 1050 / 150000 | consumed_samples: 67200 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782583E+05 | tokens_per_sec_per_gpu: 9.035286E+03 | global_batch_size: 64 | lm_loss: 1.101658E+01 | lr: 1.575E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 14995516.000 [ip-26-0-154-121:0]:06/21/2023 17:28:49 [INFO|DP=0|PP=0|TP=0]: iteration: 1051 / 150000 | consumed_samples: 67264 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780782E+05 | tokens_per_sec_per_gpu: 9.032472E+03 | global_batch_size: 64 | lm_loss: 1.103061E+01 | lr: 1.576E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 12064310.000 [ip-26-0-154-121:0]:06/21/2023 17:28:50 [INFO|DP=0|PP=0|TP=0]: iteration: 1052 / 150000 | consumed_samples: 67328 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782989E+05 | tokens_per_sec_per_gpu: 9.035921E+03 | global_batch_size: 64 | lm_loss: 1.101252E+01 | lr: 1.578E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 10286942.000 [ip-26-0-154-121:0]:06/21/2023 17:28:51 [INFO|DP=0|PP=0|TP=0]: iteration: 1053 / 150000 | consumed_samples: 67392 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784580E+05 | tokens_per_sec_per_gpu: 9.038407E+03 | global_batch_size: 64 | lm_loss: 1.102482E+01 | lr: 1.579E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 12983963.000 [ip-26-0-154-121:0]:06/21/2023 17:28:52 [INFO|DP=0|PP=0|TP=0]: iteration: 1054 / 150000 | consumed_samples: 67456 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784221E+05 | tokens_per_sec_per_gpu: 9.037846E+03 | global_batch_size: 64 | lm_loss: 1.103160E+01 | lr: 1.581E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 9979615.000 [ip-26-0-154-121:0]:06/21/2023 17:28:53 [INFO|DP=0|PP=0|TP=0]: iteration: 1055 / 150000 | consumed_samples: 67520 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779164E+05 | tokens_per_sec_per_gpu: 9.029943E+03 | global_batch_size: 64 | lm_loss: 1.101417E+01 | lr: 1.582E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 11079277.000 [ip-26-0-154-121:0]:06/21/2023 17:28:54 [INFO|DP=0|PP=0|TP=0]: iteration: 1056 / 150000 | consumed_samples: 67584 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784681E+05 | tokens_per_sec_per_gpu: 9.038564E+03 | global_batch_size: 64 | lm_loss: 1.101961E+01 | lr: 1.584E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 9629140.000 [ip-26-0-154-121:0]:06/21/2023 17:28:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1057 / 150000 | consumed_samples: 67648 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779616E+05 | tokens_per_sec_per_gpu: 9.030651E+03 | global_batch_size: 64 | lm_loss: 1.101489E+01 | lr: 1.586E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 6254622.500 [ip-26-0-154-121:0]:06/21/2023 17:28:56 [INFO|DP=0|PP=0|TP=0]: iteration: 1058 / 150000 | consumed_samples: 67712 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784448E+05 | tokens_per_sec_per_gpu: 9.038200E+03 | global_batch_size: 64 | lm_loss: 1.102495E+01 | lr: 1.587E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 9485916.000 [ip-26-0-154-121:0]:06/21/2023 17:28:57 [INFO|DP=0|PP=0|TP=0]: iteration: 1059 / 150000 | consumed_samples: 67776 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782822E+05 | tokens_per_sec_per_gpu: 9.035659E+03 | global_batch_size: 64 | lm_loss: 1.102243E+01 | lr: 1.588E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 7573471.500 [ip-26-0-154-121:0]:06/21/2023 17:28:58 [INFO|DP=0|PP=0|TP=0]: iteration: 1060 / 150000 | consumed_samples: 67840 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784669E+05 | tokens_per_sec_per_gpu: 9.038545E+03 | global_batch_size: 64 | lm_loss: 1.101686E+01 | lr: 1.590E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 8501096.000 [ip-26-0-154-121:0]:06/21/2023 17:28:58 [INFO|DP=0|PP=0|TP=0]: iteration: 1061 / 150000 | consumed_samples: 67904 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786823E+05 | tokens_per_sec_per_gpu: 9.041910E+03 | global_batch_size: 64 | lm_loss: 1.102930E+01 | lr: 1.591E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 6589083.000 [ip-26-0-154-121:0]:06/21/2023 17:28:59 [INFO|DP=0|PP=0|TP=0]: iteration: 1062 / 150000 | consumed_samples: 67968 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784183E+05 | tokens_per_sec_per_gpu: 9.037786E+03 | global_batch_size: 64 | lm_loss: 1.103356E+01 | lr: 1.593E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 6399578.500 [ip-26-0-154-121:0]:06/21/2023 17:29:00 [INFO|DP=0|PP=0|TP=0]: iteration: 1063 / 150000 | consumed_samples: 68032 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782808E+05 | tokens_per_sec_per_gpu: 9.035638E+03 | global_batch_size: 64 | lm_loss: 1.102675E+01 | lr: 1.594E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 7855274.500 [ip-26-0-154-121:0]:06/21/2023 17:29:01 [INFO|DP=0|PP=0|TP=0]: iteration: 1064 / 150000 | consumed_samples: 68096 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784585E+05 | tokens_per_sec_per_gpu: 9.038414E+03 | global_batch_size: 64 | lm_loss: 1.103663E+01 | lr: 1.596E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 8240962.000 [ip-26-0-154-121:0]:06/21/2023 17:29:02 [INFO|DP=0|PP=0|TP=0]: iteration: 1065 / 150000 | consumed_samples: 68160 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783887E+05 | tokens_per_sec_per_gpu: 9.037323E+03 | global_batch_size: 64 | lm_loss: 1.101699E+01 | lr: 1.597E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 7939006.000 [ip-26-0-154-121:0]:06/21/2023 17:29:03 [INFO|DP=0|PP=0|TP=0]: iteration: 1066 / 150000 | consumed_samples: 68224 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780454E+05 | tokens_per_sec_per_gpu: 9.031959E+03 | global_batch_size: 64 | lm_loss: 1.103476E+01 | lr: 1.599E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 8173014.500 [ip-26-0-154-121:0]:06/21/2023 17:29:04 [INFO|DP=0|PP=0|TP=0]: iteration: 1067 / 150000 | consumed_samples: 68288 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790080E+05 | tokens_per_sec_per_gpu: 9.047000E+03 | global_batch_size: 64 | lm_loss: 1.102977E+01 | lr: 1.600E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 9447304.000 [ip-26-0-154-121:0]:06/21/2023 17:29:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1068 / 150000 | consumed_samples: 68352 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784156E+05 | tokens_per_sec_per_gpu: 9.037743E+03 | global_batch_size: 64 | lm_loss: 1.102761E+01 | lr: 1.602E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 8365267.000 [ip-26-0-154-121:0]:06/21/2023 17:29:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1069 / 150000 | consumed_samples: 68416 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784981E+05 | tokens_per_sec_per_gpu: 9.039032E+03 | global_batch_size: 64 | lm_loss: 1.102698E+01 | lr: 1.603E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 8702274.000 [ip-26-0-154-121:0]:06/21/2023 17:29:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1070 / 150000 | consumed_samples: 68480 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782539E+05 | tokens_per_sec_per_gpu: 9.035217E+03 | global_batch_size: 64 | lm_loss: 1.102573E+01 | lr: 1.605E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 7879335.000 [ip-26-0-154-121:0]:06/21/2023 17:29:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1071 / 150000 | consumed_samples: 68544 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781736E+05 | tokens_per_sec_per_gpu: 9.033963E+03 | global_batch_size: 64 | lm_loss: 1.101875E+01 | lr: 1.606E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 9352464.000 [ip-26-0-154-121:0]:06/21/2023 17:29:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1072 / 150000 | consumed_samples: 68608 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783550E+05 | tokens_per_sec_per_gpu: 9.036797E+03 | global_batch_size: 64 | lm_loss: 1.102432E+01 | lr: 1.608E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 7672039.000 [ip-26-0-154-121:0]:06/21/2023 17:29:09 [INFO|DP=0|PP=0|TP=0]: iteration: 1073 / 150000 | consumed_samples: 68672 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781604E+05 | tokens_per_sec_per_gpu: 9.033756E+03 | global_batch_size: 64 | lm_loss: 1.102581E+01 | lr: 1.609E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 7513581.500 [ip-26-0-154-121:0]:06/21/2023 17:29:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1074 / 150000 | consumed_samples: 68736 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784183E+05 | tokens_per_sec_per_gpu: 9.037786E+03 | global_batch_size: 64 | lm_loss: 1.101705E+01 | lr: 1.611E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 9243204.000 [ip-26-0-154-121:0]:06/21/2023 17:29:11 [INFO|DP=0|PP=0|TP=0]: iteration: 1075 / 150000 | consumed_samples: 68800 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785906E+05 | tokens_per_sec_per_gpu: 9.040478E+03 | global_batch_size: 64 | lm_loss: 1.101715E+01 | lr: 1.612E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 7429761.500 [ip-26-0-154-121:0]:06/21/2023 17:29:12 [INFO|DP=0|PP=0|TP=0]: iteration: 1076 / 150000 | consumed_samples: 68864 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782855E+05 | tokens_per_sec_per_gpu: 9.035711E+03 | global_batch_size: 64 | lm_loss: 1.102746E+01 | lr: 1.614E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 9716739.000 [ip-26-0-154-121:0]:06/21/2023 17:29:13 [INFO|DP=0|PP=0|TP=0]: iteration: 1077 / 150000 | consumed_samples: 68928 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787520E+05 | tokens_per_sec_per_gpu: 9.043000E+03 | global_batch_size: 64 | lm_loss: 1.102385E+01 | lr: 1.616E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 9756612.000 [ip-26-0-154-121:0]:06/21/2023 17:29:14 [INFO|DP=0|PP=0|TP=0]: iteration: 1078 / 150000 | consumed_samples: 68992 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784300E+05 | tokens_per_sec_per_gpu: 9.037969E+03 | global_batch_size: 64 | lm_loss: 1.102513E+01 | lr: 1.617E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 12668414.000 [ip-26-0-154-121:0]:06/21/2023 17:29:15 [INFO|DP=0|PP=0|TP=0]: iteration: 1079 / 150000 | consumed_samples: 69056 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786045E+05 | tokens_per_sec_per_gpu: 9.040695E+03 | global_batch_size: 64 | lm_loss: 1.101467E+01 | lr: 1.619E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 9441857.000 [ip-26-0-154-121:0]:06/21/2023 17:29:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1080 / 150000 | consumed_samples: 69120 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788166E+05 | tokens_per_sec_per_gpu: 9.044009E+03 | global_batch_size: 64 | lm_loss: 1.102180E+01 | lr: 1.620E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 8173206.500 [ip-26-0-154-121:0]:06/21/2023 17:29:17 [INFO|DP=0|PP=0|TP=0]: iteration: 1081 / 150000 | consumed_samples: 69184 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785000E+05 | tokens_per_sec_per_gpu: 9.039063E+03 | global_batch_size: 64 | lm_loss: 1.100632E+01 | lr: 1.621E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 6594170.000 [ip-26-0-154-121:0]:06/21/2023 17:29:17 [INFO|DP=0|PP=0|TP=0]: iteration: 1082 / 150000 | consumed_samples: 69248 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783613E+05 | tokens_per_sec_per_gpu: 9.036895E+03 | global_batch_size: 64 | lm_loss: 1.102468E+01 | lr: 1.623E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 8679053.000 [ip-26-0-154-121:0]:06/21/2023 17:29:18 [INFO|DP=0|PP=0|TP=0]: iteration: 1083 / 150000 | consumed_samples: 69312 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783237E+05 | tokens_per_sec_per_gpu: 9.036308E+03 | global_batch_size: 64 | lm_loss: 1.102146E+01 | lr: 1.624E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 7111765.000 [ip-26-0-154-121:0]:06/21/2023 17:29:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1084 / 150000 | consumed_samples: 69376 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784221E+05 | tokens_per_sec_per_gpu: 9.037846E+03 | global_batch_size: 64 | lm_loss: 1.102103E+01 | lr: 1.626E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 10213300.000 [ip-26-0-154-121:0]:06/21/2023 17:29:20 [INFO|DP=0|PP=0|TP=0]: iteration: 1085 / 150000 | consumed_samples: 69440 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786995E+05 | tokens_per_sec_per_gpu: 9.042179E+03 | global_batch_size: 64 | lm_loss: 1.102671E+01 | lr: 1.627E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 9152481.000 [ip-26-0-154-121:0]:06/21/2023 17:29:21 [INFO|DP=0|PP=0|TP=0]: iteration: 1086 / 150000 | consumed_samples: 69504 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785108E+05 | tokens_per_sec_per_gpu: 9.039232E+03 | global_batch_size: 64 | lm_loss: 1.103396E+01 | lr: 1.629E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 8386743.000 [ip-26-0-154-121:0]:06/21/2023 17:29:22 [INFO|DP=0|PP=0|TP=0]: iteration: 1087 / 150000 | consumed_samples: 69568 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788024E+05 | tokens_per_sec_per_gpu: 9.043788E+03 | global_batch_size: 64 | lm_loss: 1.102712E+01 | lr: 1.630E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 7723724.500 [ip-26-0-154-121:0]:06/21/2023 17:29:23 [INFO|DP=0|PP=0|TP=0]: iteration: 1088 / 150000 | consumed_samples: 69632 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785898E+05 | tokens_per_sec_per_gpu: 9.040466E+03 | global_batch_size: 64 | lm_loss: 1.102805E+01 | lr: 1.632E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 8696741.000 [ip-26-0-154-121:0]:06/21/2023 17:29:24 [INFO|DP=0|PP=0|TP=0]: iteration: 1089 / 150000 | consumed_samples: 69696 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785146E+05 | tokens_per_sec_per_gpu: 9.039291E+03 | global_batch_size: 64 | lm_loss: 1.102529E+01 | lr: 1.633E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 10095112.000 [ip-26-0-154-121:0]:06/21/2023 17:29:25 [INFO|DP=0|PP=0|TP=0]: iteration: 1090 / 150000 | consumed_samples: 69760 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784813E+05 | tokens_per_sec_per_gpu: 9.038771E+03 | global_batch_size: 64 | lm_loss: 1.102989E+01 | lr: 1.635E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 9491399.000 [ip-26-0-154-121:0]:06/21/2023 17:29:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1091 / 150000 | consumed_samples: 69824 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783658E+05 | tokens_per_sec_per_gpu: 9.036966E+03 | global_batch_size: 64 | lm_loss: 1.103214E+01 | lr: 1.636E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 12946828.000 [ip-26-0-154-121:0]:06/21/2023 17:29:27 [INFO|DP=0|PP=0|TP=0]: iteration: 1092 / 150000 | consumed_samples: 69888 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787124E+05 | tokens_per_sec_per_gpu: 9.042381E+03 | global_batch_size: 64 | lm_loss: 1.102190E+01 | lr: 1.638E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 15834446.000 [ip-26-0-154-121:0]:06/21/2023 17:29:27 [INFO|DP=0|PP=0|TP=0]: iteration: 1093 / 150000 | consumed_samples: 69952 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784806E+05 | tokens_per_sec_per_gpu: 9.038759E+03 | global_batch_size: 64 | lm_loss: 1.102581E+01 | lr: 1.639E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 10303790.000 [ip-26-0-154-121:0]:06/21/2023 17:29:28 [INFO|DP=0|PP=0|TP=0]: iteration: 1094 / 150000 | consumed_samples: 70016 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788288E+05 | tokens_per_sec_per_gpu: 9.044200E+03 | global_batch_size: 64 | lm_loss: 1.102286E+01 | lr: 1.641E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 13555296.000 [ip-26-0-154-121:0]:06/21/2023 17:29:29 [INFO|DP=0|PP=0|TP=0]: iteration: 1095 / 150000 | consumed_samples: 70080 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774525E+05 | tokens_per_sec_per_gpu: 9.022695E+03 | global_batch_size: 64 | lm_loss: 1.100029E+01 | lr: 1.642E-04 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 12348230.000 [ip-26-0-154-121:0]:06/21/2023 17:29:30 [INFO|DP=0|PP=0|TP=0]: iteration: 1096 / 150000 | consumed_samples: 70144 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779935E+05 | tokens_per_sec_per_gpu: 9.031149E+03 | global_batch_size: 64 | lm_loss: 1.101982E+01 | lr: 1.644E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 12730342.000 [ip-26-0-154-121:0]:06/21/2023 17:29:31 [INFO|DP=0|PP=0|TP=0]: iteration: 1097 / 150000 | consumed_samples: 70208 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786253E+05 | tokens_per_sec_per_gpu: 9.041020E+03 | global_batch_size: 64 | lm_loss: 1.101767E+01 | lr: 1.645E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 13289731.000 [ip-26-0-154-121:0]:06/21/2023 17:29:32 [INFO|DP=0|PP=0|TP=0]: iteration: 1098 / 150000 | consumed_samples: 70272 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784262E+05 | tokens_per_sec_per_gpu: 9.037910E+03 | global_batch_size: 64 | lm_loss: 1.101230E+01 | lr: 1.647E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 9671267.000 [ip-26-0-154-121:0]:06/21/2023 17:29:33 [INFO|DP=0|PP=0|TP=0]: iteration: 1099 / 150000 | consumed_samples: 70336 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786876E+05 | tokens_per_sec_per_gpu: 9.041994E+03 | global_batch_size: 64 | lm_loss: 1.101279E+01 | lr: 1.648E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 9801890.000 [ip-26-0-154-121:0]:06/21/2023 17:29:34 [INFO|DP=0|PP=0|TP=0]: iteration: 1100 / 150000 | consumed_samples: 70400 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788651E+05 | tokens_per_sec_per_gpu: 9.044767E+03 | global_batch_size: 64 | lm_loss: 1.101686E+01 | lr: 1.650E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 9814251.000 [ip-26-0-154-121:0]:06/21/2023 17:29:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1101 / 150000 | consumed_samples: 70464 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786584E+05 | tokens_per_sec_per_gpu: 9.041537E+03 | global_batch_size: 64 | lm_loss: 1.100778E+01 | lr: 1.652E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 8268478.500 [ip-26-0-154-121:0]:06/21/2023 17:29:36 [INFO|DP=0|PP=0|TP=0]: iteration: 1102 / 150000 | consumed_samples: 70528 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788165E+05 | tokens_per_sec_per_gpu: 9.044007E+03 | global_batch_size: 64 | lm_loss: 1.102262E+01 | lr: 1.653E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 11336181.000 [ip-26-0-154-121:0]:06/21/2023 17:29:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1103 / 150000 | consumed_samples: 70592 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785793E+05 | tokens_per_sec_per_gpu: 9.040302E+03 | global_batch_size: 64 | lm_loss: 1.101894E+01 | lr: 1.654E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 12587876.000 [ip-26-0-154-121:0]:06/21/2023 17:29:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1104 / 150000 | consumed_samples: 70656 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787610E+05 | tokens_per_sec_per_gpu: 9.043141E+03 | global_batch_size: 64 | lm_loss: 1.101678E+01 | lr: 1.656E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 24129120.000 [ip-26-0-154-121:0]:06/21/2023 17:29:38 [INFO|DP=0|PP=0|TP=0]: iteration: 1105 / 150000 | consumed_samples: 70720 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787741E+05 | tokens_per_sec_per_gpu: 9.043345E+03 | global_batch_size: 64 | lm_loss: 1.101985E+01 | lr: 1.657E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 16860896.000 [ip-26-0-154-121:0]:06/21/2023 17:29:39 [INFO|DP=0|PP=0|TP=0]: iteration: 1106 / 150000 | consumed_samples: 70784 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783220E+05 | tokens_per_sec_per_gpu: 9.036282E+03 | global_batch_size: 64 | lm_loss: 1.102178E+01 | lr: 1.659E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 25706406.000 [ip-26-0-154-121:0]:06/21/2023 17:29:40 [INFO|DP=0|PP=0|TP=0]: iteration: 1107 / 150000 | consumed_samples: 70848 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787749E+05 | tokens_per_sec_per_gpu: 9.043357E+03 | global_batch_size: 64 | lm_loss: 1.101815E+01 | lr: 1.660E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 22818878.000 [ip-26-0-154-121:0]:06/21/2023 17:29:41 [INFO|DP=0|PP=0|TP=0]: iteration: 1108 / 150000 | consumed_samples: 70912 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787912E+05 | tokens_per_sec_per_gpu: 9.043612E+03 | global_batch_size: 64 | lm_loss: 1.102140E+01 | lr: 1.662E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 19710426.000 [ip-26-0-154-121:0]:06/21/2023 17:29:42 [INFO|DP=0|PP=0|TP=0]: iteration: 1109 / 150000 | consumed_samples: 70976 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786090E+05 | tokens_per_sec_per_gpu: 9.040766E+03 | global_batch_size: 64 | lm_loss: 1.100587E+01 | lr: 1.664E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 18707116.000 [ip-26-0-154-121:0]:06/21/2023 17:29:43 [INFO|DP=0|PP=0|TP=0]: iteration: 1110 / 150000 | consumed_samples: 71040 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787133E+05 | tokens_per_sec_per_gpu: 9.042396E+03 | global_batch_size: 64 | lm_loss: 1.102147E+01 | lr: 1.665E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 17235120.000 [ip-26-0-154-121:0]:06/21/2023 17:29:44 [INFO|DP=0|PP=0|TP=0]: iteration: 1111 / 150000 | consumed_samples: 71104 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786739E+05 | tokens_per_sec_per_gpu: 9.041779E+03 | global_batch_size: 64 | lm_loss: 1.101352E+01 | lr: 1.666E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 14533392.000 [ip-26-0-154-121:0]:06/21/2023 17:29:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1112 / 150000 | consumed_samples: 71168 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783836E+05 | tokens_per_sec_per_gpu: 9.037244E+03 | global_batch_size: 64 | lm_loss: 1.101920E+01 | lr: 1.668E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 13232128.000 [ip-26-0-154-121:0]:06/21/2023 17:29:46 [INFO|DP=0|PP=0|TP=0]: iteration: 1113 / 150000 | consumed_samples: 71232 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787161E+05 | tokens_per_sec_per_gpu: 9.042439E+03 | global_batch_size: 64 | lm_loss: 1.101976E+01 | lr: 1.669E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 15560074.000 [ip-26-0-154-121:0]:06/21/2023 17:29:47 [INFO|DP=0|PP=0|TP=0]: iteration: 1114 / 150000 | consumed_samples: 71296 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791113E+05 | tokens_per_sec_per_gpu: 9.048613E+03 | global_batch_size: 64 | lm_loss: 1.102236E+01 | lr: 1.671E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 15304032.000 [ip-26-0-154-121:0]:06/21/2023 17:29:47 [INFO|DP=0|PP=0|TP=0]: iteration: 1115 / 150000 | consumed_samples: 71360 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787721E+05 | tokens_per_sec_per_gpu: 9.043314E+03 | global_batch_size: 64 | lm_loss: 1.102219E+01 | lr: 1.672E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 11685658.000 [ip-26-0-154-121:0]:06/21/2023 17:29:48 [INFO|DP=0|PP=0|TP=0]: iteration: 1116 / 150000 | consumed_samples: 71424 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785025E+05 | tokens_per_sec_per_gpu: 9.039101E+03 | global_batch_size: 64 | lm_loss: 1.101842E+01 | lr: 1.674E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 11117110.000 [ip-26-0-154-121:0]:06/21/2023 17:29:49 [INFO|DP=0|PP=0|TP=0]: iteration: 1117 / 150000 | consumed_samples: 71488 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783964E+05 | tokens_per_sec_per_gpu: 9.037444E+03 | global_batch_size: 64 | lm_loss: 1.101736E+01 | lr: 1.675E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 16306956.000 [ip-26-0-154-121:0]:06/21/2023 17:29:50 [INFO|DP=0|PP=0|TP=0]: iteration: 1118 / 150000 | consumed_samples: 71552 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786736E+05 | tokens_per_sec_per_gpu: 9.041775E+03 | global_batch_size: 64 | lm_loss: 1.101096E+01 | lr: 1.677E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 10787040.000 [ip-26-0-154-121:0]:06/21/2023 17:29:51 [INFO|DP=0|PP=0|TP=0]: iteration: 1119 / 150000 | consumed_samples: 71616 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785836E+05 | tokens_per_sec_per_gpu: 9.040369E+03 | global_batch_size: 64 | lm_loss: 1.101398E+01 | lr: 1.678E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 13173974.000 [ip-26-0-154-121:0]:06/21/2023 17:29:52 [INFO|DP=0|PP=0|TP=0]: iteration: 1120 / 150000 | consumed_samples: 71680 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790467E+05 | tokens_per_sec_per_gpu: 9.047605E+03 | global_batch_size: 64 | lm_loss: 1.100278E+01 | lr: 1.680E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 13879688.000 [ip-26-0-154-121:0]:06/21/2023 17:29:53 [INFO|DP=0|PP=0|TP=0]: iteration: 1121 / 150000 | consumed_samples: 71744 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785734E+05 | tokens_per_sec_per_gpu: 9.040209E+03 | global_batch_size: 64 | lm_loss: 1.102236E+01 | lr: 1.681E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 13159873.000 [ip-26-0-154-121:0]:06/21/2023 17:29:54 [INFO|DP=0|PP=0|TP=0]: iteration: 1122 / 150000 | consumed_samples: 71808 | elapsed_time_per_iteration_ms: 911.4 | tokens_per_sec: 5.752404E+05 | tokens_per_sec_per_gpu: 8.988131E+03 | global_batch_size: 64 | lm_loss: 1.101947E+01 | lr: 1.683E-04 | model_tflops_per_gpu: 103.82 | hardware_tflops_per_gpu: 103.82 | grad_norm: 20677794.000 [ip-26-0-154-121:0]:06/21/2023 17:29:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1123 / 150000 | consumed_samples: 71872 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785595E+05 | tokens_per_sec_per_gpu: 9.039993E+03 | global_batch_size: 64 | lm_loss: 1.098509E+01 | lr: 1.685E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 15823588.000 [ip-26-0-154-121:0]:06/21/2023 17:29:56 [INFO|DP=0|PP=0|TP=0]: iteration: 1124 / 150000 | consumed_samples: 71936 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783584E+05 | tokens_per_sec_per_gpu: 9.036850E+03 | global_batch_size: 64 | lm_loss: 1.101131E+01 | lr: 1.686E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 26696022.000 [ip-26-0-154-121:0]:06/21/2023 17:29:57 [INFO|DP=0|PP=0|TP=0]: iteration: 1125 / 150000 | consumed_samples: 72000 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784536E+05 | tokens_per_sec_per_gpu: 9.038338E+03 | global_batch_size: 64 | lm_loss: 1.100708E+01 | lr: 1.687E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 22932632.000 [ip-26-0-154-121:0]:06/21/2023 17:29:57 [INFO|DP=0|PP=0|TP=0]: iteration: 1126 / 150000 | consumed_samples: 72064 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785775E+05 | tokens_per_sec_per_gpu: 9.040274E+03 | global_batch_size: 64 | lm_loss: 1.101201E+01 | lr: 1.689E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 19637818.000 [ip-26-0-154-121:0]:06/21/2023 17:29:58 [INFO|DP=0|PP=0|TP=0]: iteration: 1127 / 150000 | consumed_samples: 72128 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781624E+05 | tokens_per_sec_per_gpu: 9.033787E+03 | global_batch_size: 64 | lm_loss: 1.098784E+01 | lr: 1.690E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 18329460.000 [ip-26-0-154-121:0]:06/21/2023 17:29:59 [INFO|DP=0|PP=0|TP=0]: iteration: 1128 / 150000 | consumed_samples: 72192 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782212E+05 | tokens_per_sec_per_gpu: 9.034706E+03 | global_batch_size: 64 | lm_loss: 1.101003E+01 | lr: 1.692E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 18303650.000 [ip-26-0-154-121:0]:06/21/2023 17:30:00 [INFO|DP=0|PP=0|TP=0]: iteration: 1129 / 150000 | consumed_samples: 72256 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783940E+05 | tokens_per_sec_per_gpu: 9.037406E+03 | global_batch_size: 64 | lm_loss: 1.099129E+01 | lr: 1.693E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 14581032.000 [ip-26-0-154-121:0]:06/21/2023 17:30:01 [INFO|DP=0|PP=0|TP=0]: iteration: 1130 / 150000 | consumed_samples: 72320 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778326E+05 | tokens_per_sec_per_gpu: 9.028634E+03 | global_batch_size: 64 | lm_loss: 1.100699E+01 | lr: 1.695E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 20518816.000 [ip-26-0-154-121:0]:06/21/2023 17:30:02 [INFO|DP=0|PP=0|TP=0]: iteration: 1131 / 150000 | consumed_samples: 72384 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781928E+05 | tokens_per_sec_per_gpu: 9.034262E+03 | global_batch_size: 64 | lm_loss: 1.100768E+01 | lr: 1.697E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 16402500.000 [ip-26-0-154-121:0]:06/21/2023 17:30:03 [INFO|DP=0|PP=0|TP=0]: iteration: 1132 / 150000 | consumed_samples: 72448 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780074E+05 | tokens_per_sec_per_gpu: 9.031365E+03 | global_batch_size: 64 | lm_loss: 1.101108E+01 | lr: 1.698E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 20210792.000 [ip-26-0-154-121:0]:06/21/2023 17:30:04 [INFO|DP=0|PP=0|TP=0]: iteration: 1133 / 150000 | consumed_samples: 72512 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779841E+05 | tokens_per_sec_per_gpu: 9.031002E+03 | global_batch_size: 64 | lm_loss: 1.101027E+01 | lr: 1.699E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 19723890.000 [ip-26-0-154-121:0]:06/21/2023 17:30:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1134 / 150000 | consumed_samples: 72576 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782842E+05 | tokens_per_sec_per_gpu: 9.035690E+03 | global_batch_size: 64 | lm_loss: 1.101165E+01 | lr: 1.701E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 18891226.000 [ip-26-0-154-121:0]:06/21/2023 17:30:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1135 / 150000 | consumed_samples: 72640 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785157E+05 | tokens_per_sec_per_gpu: 9.039308E+03 | global_batch_size: 64 | lm_loss: 1.101261E+01 | lr: 1.702E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 21061446.000 [ip-26-0-154-121:0]:06/21/2023 17:30:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1136 / 150000 | consumed_samples: 72704 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787112E+05 | tokens_per_sec_per_gpu: 9.042362E+03 | global_batch_size: 64 | lm_loss: 1.100644E+01 | lr: 1.704E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 16112748.000 [ip-26-0-154-121:0]:06/21/2023 17:30:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1137 / 150000 | consumed_samples: 72768 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785735E+05 | tokens_per_sec_per_gpu: 9.040212E+03 | global_batch_size: 64 | lm_loss: 1.100208E+01 | lr: 1.705E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 14685952.000 [ip-26-0-154-121:0]:06/21/2023 17:30:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1138 / 150000 | consumed_samples: 72832 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789268E+05 | tokens_per_sec_per_gpu: 9.045731E+03 | global_batch_size: 64 | lm_loss: 1.100946E+01 | lr: 1.707E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 12523727.000 [ip-26-0-154-121:0]:06/21/2023 17:30:09 [INFO|DP=0|PP=0|TP=0]: iteration: 1139 / 150000 | consumed_samples: 72896 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784999E+05 | tokens_per_sec_per_gpu: 9.039061E+03 | global_batch_size: 64 | lm_loss: 1.100871E+01 | lr: 1.708E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 12149716.000 [ip-26-0-154-121:0]:06/21/2023 17:30:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1140 / 150000 | consumed_samples: 72960 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784153E+05 | tokens_per_sec_per_gpu: 9.037739E+03 | global_batch_size: 64 | lm_loss: 1.100176E+01 | lr: 1.710E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 13047400.000 [ip-26-0-154-121:0]:06/21/2023 17:30:11 [INFO|DP=0|PP=0|TP=0]: iteration: 1141 / 150000 | consumed_samples: 73024 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785172E+05 | tokens_per_sec_per_gpu: 9.039332E+03 | global_batch_size: 64 | lm_loss: 1.100909E+01 | lr: 1.711E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 13286984.000 [ip-26-0-154-121:0]:06/21/2023 17:30:12 [INFO|DP=0|PP=0|TP=0]: iteration: 1142 / 150000 | consumed_samples: 73088 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784705E+05 | tokens_per_sec_per_gpu: 9.038602E+03 | global_batch_size: 64 | lm_loss: 1.099853E+01 | lr: 1.713E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 12305638.000 [ip-26-0-154-121:0]:06/21/2023 17:30:13 [INFO|DP=0|PP=0|TP=0]: iteration: 1143 / 150000 | consumed_samples: 73152 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782820E+05 | tokens_per_sec_per_gpu: 9.035657E+03 | global_batch_size: 64 | lm_loss: 1.101052E+01 | lr: 1.714E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 13357822.000 [ip-26-0-154-121:0]:06/21/2023 17:30:14 [INFO|DP=0|PP=0|TP=0]: iteration: 1144 / 150000 | consumed_samples: 73216 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783336E+05 | tokens_per_sec_per_gpu: 9.036462E+03 | global_batch_size: 64 | lm_loss: 1.100611E+01 | lr: 1.716E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 14121892.000 [ip-26-0-154-121:0]:06/21/2023 17:30:15 [INFO|DP=0|PP=0|TP=0]: iteration: 1145 / 150000 | consumed_samples: 73280 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783903E+05 | tokens_per_sec_per_gpu: 9.037349E+03 | global_batch_size: 64 | lm_loss: 1.100256E+01 | lr: 1.718E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 15102021.000 [ip-26-0-154-121:0]:06/21/2023 17:30:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1146 / 150000 | consumed_samples: 73344 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780911E+05 | tokens_per_sec_per_gpu: 9.032673E+03 | global_batch_size: 64 | lm_loss: 1.100673E+01 | lr: 1.719E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 14062234.000 [ip-26-0-154-121:0]:06/21/2023 17:30:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1147 / 150000 | consumed_samples: 73408 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782709E+05 | tokens_per_sec_per_gpu: 9.035483E+03 | global_batch_size: 64 | lm_loss: 1.099238E+01 | lr: 1.720E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 13744708.000 [ip-26-0-154-121:0]:06/21/2023 17:30:17 [INFO|DP=0|PP=0|TP=0]: iteration: 1148 / 150000 | consumed_samples: 73472 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783651E+05 | tokens_per_sec_per_gpu: 9.036954E+03 | global_batch_size: 64 | lm_loss: 1.100018E+01 | lr: 1.722E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 17016260.000 [ip-26-0-154-121:0]:06/21/2023 17:30:18 [INFO|DP=0|PP=0|TP=0]: iteration: 1149 / 150000 | consumed_samples: 73536 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785836E+05 | tokens_per_sec_per_gpu: 9.040369E+03 | global_batch_size: 64 | lm_loss: 1.099586E+01 | lr: 1.723E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 18171824.000 [ip-26-0-154-121:0]:06/21/2023 17:30:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1150 / 150000 | consumed_samples: 73600 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783585E+05 | tokens_per_sec_per_gpu: 9.036852E+03 | global_batch_size: 64 | lm_loss: 1.100116E+01 | lr: 1.725E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 14720824.000 [ip-26-0-154-121:0]:06/21/2023 17:30:20 [INFO|DP=0|PP=0|TP=0]: iteration: 1151 / 150000 | consumed_samples: 73664 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783333E+05 | tokens_per_sec_per_gpu: 9.036458E+03 | global_batch_size: 64 | lm_loss: 1.098913E+01 | lr: 1.726E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 19428612.000 [ip-26-0-154-121:0]:06/21/2023 17:30:21 [INFO|DP=0|PP=0|TP=0]: iteration: 1152 / 150000 | consumed_samples: 73728 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784713E+05 | tokens_per_sec_per_gpu: 9.038614E+03 | global_batch_size: 64 | lm_loss: 1.100522E+01 | lr: 1.728E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 16879752.000 [ip-26-0-154-121:0]:06/21/2023 17:30:22 [INFO|DP=0|PP=0|TP=0]: iteration: 1153 / 150000 | consumed_samples: 73792 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782623E+05 | tokens_per_sec_per_gpu: 9.035348E+03 | global_batch_size: 64 | lm_loss: 1.100927E+01 | lr: 1.729E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 17231186.000 [ip-26-0-154-121:0]:06/21/2023 17:30:23 [INFO|DP=0|PP=0|TP=0]: iteration: 1154 / 150000 | consumed_samples: 73856 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785040E+05 | tokens_per_sec_per_gpu: 9.039125E+03 | global_batch_size: 64 | lm_loss: 1.099731E+01 | lr: 1.731E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 20632198.000 [ip-26-0-154-121:0]:06/21/2023 17:30:24 [INFO|DP=0|PP=0|TP=0]: iteration: 1155 / 150000 | consumed_samples: 73920 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787156E+05 | tokens_per_sec_per_gpu: 9.042431E+03 | global_batch_size: 64 | lm_loss: 1.099303E+01 | lr: 1.732E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 21170048.000 [ip-26-0-154-121:0]:06/21/2023 17:30:25 [INFO|DP=0|PP=0|TP=0]: iteration: 1156 / 150000 | consumed_samples: 73984 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787043E+05 | tokens_per_sec_per_gpu: 9.042255E+03 | global_batch_size: 64 | lm_loss: 1.099626E+01 | lr: 1.734E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 24966882.000 [ip-26-0-154-121:0]:06/21/2023 17:30:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1157 / 150000 | consumed_samples: 74048 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786022E+05 | tokens_per_sec_per_gpu: 9.040659E+03 | global_batch_size: 64 | lm_loss: 1.100206E+01 | lr: 1.735E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 20236894.000 [ip-26-0-154-121:0]:06/21/2023 17:30:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1158 / 150000 | consumed_samples: 74112 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783829E+05 | tokens_per_sec_per_gpu: 9.037232E+03 | global_batch_size: 64 | lm_loss: 1.100710E+01 | lr: 1.737E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 19840228.000 [ip-26-0-154-121:0]:06/21/2023 17:30:27 [INFO|DP=0|PP=0|TP=0]: iteration: 1159 / 150000 | consumed_samples: 74176 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783514E+05 | tokens_per_sec_per_gpu: 9.036740E+03 | global_batch_size: 64 | lm_loss: 1.100426E+01 | lr: 1.738E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 17153840.000 [ip-26-0-154-121:0]:06/21/2023 17:30:28 [INFO|DP=0|PP=0|TP=0]: iteration: 1160 / 150000 | consumed_samples: 74240 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784723E+05 | tokens_per_sec_per_gpu: 9.038630E+03 | global_batch_size: 64 | lm_loss: 1.100433E+01 | lr: 1.740E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 17117008.000 [ip-26-0-154-121:0]:06/21/2023 17:30:29 [INFO|DP=0|PP=0|TP=0]: iteration: 1161 / 150000 | consumed_samples: 74304 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785308E+05 | tokens_per_sec_per_gpu: 9.039543E+03 | global_batch_size: 64 | lm_loss: 1.100583E+01 | lr: 1.741E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 15133595.000 [ip-26-0-154-121:0]:06/21/2023 17:30:30 [INFO|DP=0|PP=0|TP=0]: iteration: 1162 / 150000 | consumed_samples: 74368 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781777E+05 | tokens_per_sec_per_gpu: 9.034027E+03 | global_batch_size: 64 | lm_loss: 1.098880E+01 | lr: 1.743E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 16693222.000 [ip-26-0-154-121:0]:06/21/2023 17:30:31 [INFO|DP=0|PP=0|TP=0]: iteration: 1163 / 150000 | consumed_samples: 74432 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783430E+05 | tokens_per_sec_per_gpu: 9.036610E+03 | global_batch_size: 64 | lm_loss: 1.100166E+01 | lr: 1.744E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 25021300.000 [ip-26-0-154-121:0]:06/21/2023 17:30:32 [INFO|DP=0|PP=0|TP=0]: iteration: 1164 / 150000 | consumed_samples: 74496 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786504E+05 | tokens_per_sec_per_gpu: 9.041413E+03 | global_batch_size: 64 | lm_loss: 1.100303E+01 | lr: 1.746E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 26495938.000 [ip-26-0-154-121:0]:06/21/2023 17:30:33 [INFO|DP=0|PP=0|TP=0]: iteration: 1165 / 150000 | consumed_samples: 74560 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786358E+05 | tokens_per_sec_per_gpu: 9.041185E+03 | global_batch_size: 64 | lm_loss: 1.099761E+01 | lr: 1.747E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 24830782.000 [ip-26-0-154-121:0]:06/21/2023 17:30:34 [INFO|DP=0|PP=0|TP=0]: iteration: 1166 / 150000 | consumed_samples: 74624 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778849E+05 | tokens_per_sec_per_gpu: 9.029452E+03 | global_batch_size: 64 | lm_loss: 1.099105E+01 | lr: 1.749E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 33023374.000 [ip-26-0-154-121:0]:06/21/2023 17:30:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1167 / 150000 | consumed_samples: 74688 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791382E+05 | tokens_per_sec_per_gpu: 9.049035E+03 | global_batch_size: 64 | lm_loss: 1.099495E+01 | lr: 1.750E-04 | model_tflops_per_gpu: 104.53 | hardware_tflops_per_gpu: 104.53 | grad_norm: 30683062.000 [ip-26-0-154-121:0]:06/21/2023 17:30:36 [INFO|DP=0|PP=0|TP=0]: iteration: 1168 / 150000 | consumed_samples: 74752 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787636E+05 | tokens_per_sec_per_gpu: 9.043181E+03 | global_batch_size: 64 | lm_loss: 1.099638E+01 | lr: 1.752E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 28461434.000 [ip-26-0-154-121:0]:06/21/2023 17:30:36 [INFO|DP=0|PP=0|TP=0]: iteration: 1169 / 150000 | consumed_samples: 74816 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789376E+05 | tokens_per_sec_per_gpu: 9.045900E+03 | global_batch_size: 64 | lm_loss: 1.099267E+01 | lr: 1.753E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 28171942.000 [ip-26-0-154-121:0]:06/21/2023 17:30:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1170 / 150000 | consumed_samples: 74880 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788097E+05 | tokens_per_sec_per_gpu: 9.043902E+03 | global_batch_size: 64 | lm_loss: 1.100366E+01 | lr: 1.755E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 24146282.000 [ip-26-0-154-121:0]:06/21/2023 17:30:38 [INFO|DP=0|PP=0|TP=0]: iteration: 1171 / 150000 | consumed_samples: 74944 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788504E+05 | tokens_per_sec_per_gpu: 9.044538E+03 | global_batch_size: 64 | lm_loss: 1.098437E+01 | lr: 1.756E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 21191826.000 [ip-26-0-154-121:0]:06/21/2023 17:30:39 [INFO|DP=0|PP=0|TP=0]: iteration: 1172 / 150000 | consumed_samples: 75008 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785337E+05 | tokens_per_sec_per_gpu: 9.039589E+03 | global_batch_size: 64 | lm_loss: 1.099656E+01 | lr: 1.758E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 24270610.000 [ip-26-0-154-121:0]:06/21/2023 17:30:40 [INFO|DP=0|PP=0|TP=0]: iteration: 1173 / 150000 | consumed_samples: 75072 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787438E+05 | tokens_per_sec_per_gpu: 9.042872E+03 | global_batch_size: 64 | lm_loss: 1.100198E+01 | lr: 1.759E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 20608648.000 [ip-26-0-154-121:0]:06/21/2023 17:30:41 [INFO|DP=0|PP=0|TP=0]: iteration: 1174 / 150000 | consumed_samples: 75136 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787427E+05 | tokens_per_sec_per_gpu: 9.042855E+03 | global_batch_size: 64 | lm_loss: 1.100452E+01 | lr: 1.761E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 18295446.000 [ip-26-0-154-121:0]:06/21/2023 17:30:42 [INFO|DP=0|PP=0|TP=0]: iteration: 1175 / 150000 | consumed_samples: 75200 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789277E+05 | tokens_per_sec_per_gpu: 9.045745E+03 | global_batch_size: 64 | lm_loss: 1.100189E+01 | lr: 1.763E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 17234042.000 [ip-26-0-154-121:0]:06/21/2023 17:30:43 [INFO|DP=0|PP=0|TP=0]: iteration: 1176 / 150000 | consumed_samples: 75264 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788134E+05 | tokens_per_sec_per_gpu: 9.043959E+03 | global_batch_size: 64 | lm_loss: 1.100411E+01 | lr: 1.764E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 20553032.000 [ip-26-0-154-121:0]:06/21/2023 17:30:44 [INFO|DP=0|PP=0|TP=0]: iteration: 1177 / 150000 | consumed_samples: 75328 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784121E+05 | tokens_per_sec_per_gpu: 9.037689E+03 | global_batch_size: 64 | lm_loss: 1.096157E+01 | lr: 1.765E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 18112268.000 [ip-26-0-154-121:0]:06/21/2023 17:30:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1178 / 150000 | consumed_samples: 75392 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784701E+05 | tokens_per_sec_per_gpu: 9.038595E+03 | global_batch_size: 64 | lm_loss: 1.100700E+01 | lr: 1.767E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 32612794.000 [ip-26-0-154-121:0]:06/21/2023 17:30:46 [INFO|DP=0|PP=0|TP=0]: iteration: 1179 / 150000 | consumed_samples: 75456 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784714E+05 | tokens_per_sec_per_gpu: 9.038616E+03 | global_batch_size: 64 | lm_loss: 1.099742E+01 | lr: 1.768E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 34427964.000 [ip-26-0-154-121:0]:06/21/2023 17:30:46 [INFO|DP=0|PP=0|TP=0]: iteration: 1180 / 150000 | consumed_samples: 75520 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786661E+05 | tokens_per_sec_per_gpu: 9.041658E+03 | global_batch_size: 64 | lm_loss: 1.099489E+01 | lr: 1.770E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 38831580.000 [ip-26-0-154-121:0]:06/21/2023 17:30:47 [INFO|DP=0|PP=0|TP=0]: iteration: 1181 / 150000 | consumed_samples: 75584 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786970E+05 | tokens_per_sec_per_gpu: 9.042141E+03 | global_batch_size: 64 | lm_loss: 1.099847E+01 | lr: 1.771E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 42197128.000 [ip-26-0-154-121:0]:06/21/2023 17:30:48 [INFO|DP=0|PP=0|TP=0]: iteration: 1182 / 150000 | consumed_samples: 75648 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787141E+05 | tokens_per_sec_per_gpu: 9.042408E+03 | global_batch_size: 64 | lm_loss: 1.100217E+01 | lr: 1.773E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 43185996.000 [ip-26-0-154-121:0]:06/21/2023 17:30:49 [INFO|DP=0|PP=0|TP=0]: iteration: 1183 / 150000 | consumed_samples: 75712 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786791E+05 | tokens_per_sec_per_gpu: 9.041860E+03 | global_batch_size: 64 | lm_loss: 1.100145E+01 | lr: 1.775E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 36465956.000 [ip-26-0-154-121:0]:06/21/2023 17:30:50 [INFO|DP=0|PP=0|TP=0]: iteration: 1184 / 150000 | consumed_samples: 75776 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787479E+05 | tokens_per_sec_per_gpu: 9.042936E+03 | global_batch_size: 64 | lm_loss: 1.100197E+01 | lr: 1.776E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 38897364.000 [ip-26-0-154-121:0]:06/21/2023 17:30:51 [INFO|DP=0|PP=0|TP=0]: iteration: 1185 / 150000 | consumed_samples: 75840 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785416E+05 | tokens_per_sec_per_gpu: 9.039712E+03 | global_batch_size: 64 | lm_loss: 1.101714E+01 | lr: 1.777E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 34530600.000 [ip-26-0-154-121:0]:06/21/2023 17:30:52 [INFO|DP=0|PP=0|TP=0]: iteration: 1186 / 150000 | consumed_samples: 75904 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782866E+05 | tokens_per_sec_per_gpu: 9.035728E+03 | global_batch_size: 64 | lm_loss: 1.099817E+01 | lr: 1.779E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 42466900.000 [ip-26-0-154-121:0]:06/21/2023 17:30:53 [INFO|DP=0|PP=0|TP=0]: iteration: 1187 / 150000 | consumed_samples: 75968 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785259E+05 | tokens_per_sec_per_gpu: 9.039467E+03 | global_batch_size: 64 | lm_loss: 1.097926E+01 | lr: 1.780E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 41510724.000 [ip-26-0-154-121:0]:06/21/2023 17:30:54 [INFO|DP=0|PP=0|TP=0]: iteration: 1188 / 150000 | consumed_samples: 76032 | elapsed_time_per_iteration_ms: 911.1 | tokens_per_sec: 5.754602E+05 | tokens_per_sec_per_gpu: 8.991565E+03 | global_batch_size: 64 | lm_loss: 1.099971E+01 | lr: 1.782E-04 | model_tflops_per_gpu: 103.86 | hardware_tflops_per_gpu: 103.86 | grad_norm: 33651732.000 [ip-26-0-154-121:0]:06/21/2023 17:30:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1189 / 150000 | consumed_samples: 76096 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789248E+05 | tokens_per_sec_per_gpu: 9.045700E+03 | global_batch_size: 64 | lm_loss: 1.099763E+01 | lr: 1.783E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 34262540.000 [ip-26-0-154-121:0]:06/21/2023 17:30:56 [INFO|DP=0|PP=0|TP=0]: iteration: 1190 / 150000 | consumed_samples: 76160 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786450E+05 | tokens_per_sec_per_gpu: 9.041327E+03 | global_batch_size: 64 | lm_loss: 1.100050E+01 | lr: 1.785E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 26416702.000 [ip-26-0-154-121:0]:06/21/2023 17:30:56 [INFO|DP=0|PP=0|TP=0]: iteration: 1191 / 150000 | consumed_samples: 76224 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783739E+05 | tokens_per_sec_per_gpu: 9.037092E+03 | global_batch_size: 64 | lm_loss: 1.099944E+01 | lr: 1.787E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 29520406.000 [ip-26-0-154-121:0]:06/21/2023 17:30:57 [INFO|DP=0|PP=0|TP=0]: iteration: 1192 / 150000 | consumed_samples: 76288 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783026E+05 | tokens_per_sec_per_gpu: 9.035978E+03 | global_batch_size: 64 | lm_loss: 1.099249E+01 | lr: 1.788E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 19039518.000 [ip-26-0-154-121:0]:06/21/2023 17:30:58 [INFO|DP=0|PP=0|TP=0]: iteration: 1193 / 150000 | consumed_samples: 76352 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782227E+05 | tokens_per_sec_per_gpu: 9.034730E+03 | global_batch_size: 64 | lm_loss: 1.098957E+01 | lr: 1.789E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 17679588.000 [ip-26-0-154-121:0]:06/21/2023 17:30:59 [INFO|DP=0|PP=0|TP=0]: iteration: 1194 / 150000 | consumed_samples: 76416 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784691E+05 | tokens_per_sec_per_gpu: 9.038580E+03 | global_batch_size: 64 | lm_loss: 1.099049E+01 | lr: 1.791E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 21467306.000 [ip-26-0-154-121:0]:06/21/2023 17:31:00 [INFO|DP=0|PP=0|TP=0]: iteration: 1195 / 150000 | consumed_samples: 76480 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779054E+05 | tokens_per_sec_per_gpu: 9.029773E+03 | global_batch_size: 64 | lm_loss: 1.097882E+01 | lr: 1.792E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 20616512.000 [ip-26-0-154-121:0]:06/21/2023 17:31:01 [INFO|DP=0|PP=0|TP=0]: iteration: 1196 / 150000 | consumed_samples: 76544 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779288E+05 | tokens_per_sec_per_gpu: 9.030138E+03 | global_batch_size: 64 | lm_loss: 1.099848E+01 | lr: 1.794E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 25576070.000 [ip-26-0-154-121:0]:06/21/2023 17:31:02 [INFO|DP=0|PP=0|TP=0]: iteration: 1197 / 150000 | consumed_samples: 76608 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784981E+05 | tokens_per_sec_per_gpu: 9.039032E+03 | global_batch_size: 64 | lm_loss: 1.099922E+01 | lr: 1.796E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 32569908.000 [ip-26-0-154-121:0]:06/21/2023 17:31:03 [INFO|DP=0|PP=0|TP=0]: iteration: 1198 / 150000 | consumed_samples: 76672 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784644E+05 | tokens_per_sec_per_gpu: 9.038507E+03 | global_batch_size: 64 | lm_loss: 1.099422E+01 | lr: 1.797E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 37412800.000 [ip-26-0-154-121:0]:06/21/2023 17:31:04 [INFO|DP=0|PP=0|TP=0]: iteration: 1199 / 150000 | consumed_samples: 76736 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782518E+05 | tokens_per_sec_per_gpu: 9.035184E+03 | global_batch_size: 64 | lm_loss: 1.099222E+01 | lr: 1.798E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 28973276.000 [ip-26-0-154-121:0]:06/21/2023 17:31:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1200 / 150000 | consumed_samples: 76800 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787698E+05 | tokens_per_sec_per_gpu: 9.043279E+03 | global_batch_size: 64 | lm_loss: 1.099507E+01 | lr: 1.800E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 45407224.000 [ip-26-0-154-121:0]:06/21/2023 17:31:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1201 / 150000 | consumed_samples: 76864 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782825E+05 | tokens_per_sec_per_gpu: 9.035664E+03 | global_batch_size: 64 | lm_loss: 1.099089E+01 | lr: 1.801E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 42374384.000 [ip-26-0-154-121:0]:06/21/2023 17:31:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1202 / 150000 | consumed_samples: 76928 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781381E+05 | tokens_per_sec_per_gpu: 9.033407E+03 | global_batch_size: 64 | lm_loss: 1.098354E+01 | lr: 1.803E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 38114660.000 [ip-26-0-154-121:0]:06/21/2023 17:31:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1203 / 150000 | consumed_samples: 76992 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783447E+05 | tokens_per_sec_per_gpu: 9.036636E+03 | global_batch_size: 64 | lm_loss: 1.099115E+01 | lr: 1.804E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 27268644.000 [ip-26-0-154-121:0]:06/21/2023 17:31:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1204 / 150000 | consumed_samples: 77056 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786462E+05 | tokens_per_sec_per_gpu: 9.041346E+03 | global_batch_size: 64 | lm_loss: 1.098946E+01 | lr: 1.806E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 26891672.000 [ip-26-0-154-121:0]:06/21/2023 17:31:09 [INFO|DP=0|PP=0|TP=0]: iteration: 1205 / 150000 | consumed_samples: 77120 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781703E+05 | tokens_per_sec_per_gpu: 9.033911E+03 | global_batch_size: 64 | lm_loss: 1.099277E+01 | lr: 1.808E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 35564264.000 [ip-26-0-154-121:0]:06/21/2023 17:31:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1206 / 150000 | consumed_samples: 77184 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783537E+05 | tokens_per_sec_per_gpu: 9.036776E+03 | global_batch_size: 64 | lm_loss: 1.099405E+01 | lr: 1.809E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 36294180.000 [ip-26-0-154-121:0]:06/21/2023 17:31:11 [INFO|DP=0|PP=0|TP=0]: iteration: 1207 / 150000 | consumed_samples: 77248 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787152E+05 | tokens_per_sec_per_gpu: 9.042424E+03 | global_batch_size: 64 | lm_loss: 1.099912E+01 | lr: 1.810E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 29244538.000 [ip-26-0-154-121:0]:06/21/2023 17:31:12 [INFO|DP=0|PP=0|TP=0]: iteration: 1208 / 150000 | consumed_samples: 77312 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783876E+05 | tokens_per_sec_per_gpu: 9.037306E+03 | global_batch_size: 64 | lm_loss: 1.099402E+01 | lr: 1.812E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 36081404.000 [ip-26-0-154-121:0]:06/21/2023 17:31:13 [INFO|DP=0|PP=0|TP=0]: iteration: 1209 / 150000 | consumed_samples: 77376 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787056E+05 | tokens_per_sec_per_gpu: 9.042274E+03 | global_batch_size: 64 | lm_loss: 1.099303E+01 | lr: 1.813E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 39141056.000 [ip-26-0-154-121:0]:06/21/2023 17:31:14 [INFO|DP=0|PP=0|TP=0]: iteration: 1210 / 150000 | consumed_samples: 77440 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784554E+05 | tokens_per_sec_per_gpu: 9.038366E+03 | global_batch_size: 64 | lm_loss: 1.100006E+01 | lr: 1.815E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 29517922.000 [ip-26-0-154-121:0]:06/21/2023 17:31:15 [INFO|DP=0|PP=0|TP=0]: iteration: 1211 / 150000 | consumed_samples: 77504 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785789E+05 | tokens_per_sec_per_gpu: 9.040295E+03 | global_batch_size: 64 | lm_loss: 1.097864E+01 | lr: 1.816E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 39534656.000 [ip-26-0-154-121:0]:06/21/2023 17:31:15 [INFO|DP=0|PP=0|TP=0]: iteration: 1212 / 150000 | consumed_samples: 77568 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789829E+05 | tokens_per_sec_per_gpu: 9.046607E+03 | global_batch_size: 64 | lm_loss: 1.098805E+01 | lr: 1.818E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 46443496.000 [ip-26-0-154-121:0]:06/21/2023 17:31:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1213 / 150000 | consumed_samples: 77632 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787295E+05 | tokens_per_sec_per_gpu: 9.042648E+03 | global_batch_size: 64 | lm_loss: 1.099238E+01 | lr: 1.819E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 38154008.000 [ip-26-0-154-121:0]:06/21/2023 17:31:17 [INFO|DP=0|PP=0|TP=0]: iteration: 1214 / 150000 | consumed_samples: 77696 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784212E+05 | tokens_per_sec_per_gpu: 9.037831E+03 | global_batch_size: 64 | lm_loss: 1.098754E+01 | lr: 1.821E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 40287836.000 [ip-26-0-154-121:0]:06/21/2023 17:31:18 [INFO|DP=0|PP=0|TP=0]: iteration: 1215 / 150000 | consumed_samples: 77760 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783228E+05 | tokens_per_sec_per_gpu: 9.036294E+03 | global_batch_size: 64 | lm_loss: 1.099377E+01 | lr: 1.822E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 48474512.000 [ip-26-0-154-121:0]:06/21/2023 17:31:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1216 / 150000 | consumed_samples: 77824 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787573E+05 | tokens_per_sec_per_gpu: 9.043084E+03 | global_batch_size: 64 | lm_loss: 1.098129E+01 | lr: 1.824E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 48888464.000 [ip-26-0-154-121:0]:06/21/2023 17:31:20 [INFO|DP=0|PP=0|TP=0]: iteration: 1217 / 150000 | consumed_samples: 77888 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790388E+05 | tokens_per_sec_per_gpu: 9.047482E+03 | global_batch_size: 64 | lm_loss: 1.098781E+01 | lr: 1.825E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 37341664.000 [ip-26-0-154-121:0]:06/21/2023 17:31:21 [INFO|DP=0|PP=0|TP=0]: iteration: 1218 / 150000 | consumed_samples: 77952 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786669E+05 | tokens_per_sec_per_gpu: 9.041670E+03 | global_batch_size: 64 | lm_loss: 1.098377E+01 | lr: 1.827E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 41603328.000 [ip-26-0-154-121:0]:06/21/2023 17:31:22 [INFO|DP=0|PP=0|TP=0]: iteration: 1219 / 150000 | consumed_samples: 78016 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787692E+05 | tokens_per_sec_per_gpu: 9.043269E+03 | global_batch_size: 64 | lm_loss: 1.098177E+01 | lr: 1.828E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 38033624.000 [ip-26-0-154-121:0]:06/21/2023 17:31:23 [INFO|DP=0|PP=0|TP=0]: iteration: 1220 / 150000 | consumed_samples: 78080 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784112E+05 | tokens_per_sec_per_gpu: 9.037675E+03 | global_batch_size: 64 | lm_loss: 1.098364E+01 | lr: 1.830E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 42662800.000 [ip-26-0-154-121:0]:06/21/2023 17:31:24 [INFO|DP=0|PP=0|TP=0]: iteration: 1221 / 150000 | consumed_samples: 78144 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782332E+05 | tokens_per_sec_per_gpu: 9.034894E+03 | global_batch_size: 64 | lm_loss: 1.098442E+01 | lr: 1.831E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 35962304.000 [ip-26-0-154-121:0]:06/21/2023 17:31:25 [INFO|DP=0|PP=0|TP=0]: iteration: 1222 / 150000 | consumed_samples: 78208 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787057E+05 | tokens_per_sec_per_gpu: 9.042277E+03 | global_batch_size: 64 | lm_loss: 1.096850E+01 | lr: 1.833E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 37532868.000 [ip-26-0-154-121:0]:06/21/2023 17:31:25 [INFO|DP=0|PP=0|TP=0]: iteration: 1223 / 150000 | consumed_samples: 78272 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784644E+05 | tokens_per_sec_per_gpu: 9.038507E+03 | global_batch_size: 64 | lm_loss: 1.097563E+01 | lr: 1.834E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 31774096.000 [ip-26-0-154-121:0]:06/21/2023 17:31:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1224 / 150000 | consumed_samples: 78336 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786148E+05 | tokens_per_sec_per_gpu: 9.040856E+03 | global_batch_size: 64 | lm_loss: 1.098740E+01 | lr: 1.836E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 25081370.000 [ip-26-0-154-121:0]:06/21/2023 17:31:27 [INFO|DP=0|PP=0|TP=0]: iteration: 1225 / 150000 | consumed_samples: 78400 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783874E+05 | tokens_per_sec_per_gpu: 9.037304E+03 | global_batch_size: 64 | lm_loss: 1.098609E+01 | lr: 1.837E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 25974960.000 [ip-26-0-154-121:0]:06/21/2023 17:31:28 [INFO|DP=0|PP=0|TP=0]: iteration: 1226 / 150000 | consumed_samples: 78464 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786143E+05 | tokens_per_sec_per_gpu: 9.040849E+03 | global_batch_size: 64 | lm_loss: 1.098280E+01 | lr: 1.839E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 22603854.000 [ip-26-0-154-121:0]:06/21/2023 17:31:29 [INFO|DP=0|PP=0|TP=0]: iteration: 1227 / 150000 | consumed_samples: 78528 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784431E+05 | tokens_per_sec_per_gpu: 9.038174E+03 | global_batch_size: 64 | lm_loss: 1.098280E+01 | lr: 1.841E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 27046880.000 [ip-26-0-154-121:0]:06/21/2023 17:31:30 [INFO|DP=0|PP=0|TP=0]: iteration: 1228 / 150000 | consumed_samples: 78592 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782962E+05 | tokens_per_sec_per_gpu: 9.035878E+03 | global_batch_size: 64 | lm_loss: 1.098302E+01 | lr: 1.842E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 25637108.000 [ip-26-0-154-121:0]:06/21/2023 17:31:31 [INFO|DP=0|PP=0|TP=0]: iteration: 1229 / 150000 | consumed_samples: 78656 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782881E+05 | tokens_per_sec_per_gpu: 9.035752E+03 | global_batch_size: 64 | lm_loss: 1.097737E+01 | lr: 1.844E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 45668688.000 [ip-26-0-154-121:0]:06/21/2023 17:31:32 [INFO|DP=0|PP=0|TP=0]: iteration: 1230 / 150000 | consumed_samples: 78720 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785224E+05 | tokens_per_sec_per_gpu: 9.039413E+03 | global_batch_size: 64 | lm_loss: 1.099096E+01 | lr: 1.845E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 46507380.000 [ip-26-0-154-121:0]:06/21/2023 17:31:33 [INFO|DP=0|PP=0|TP=0]: iteration: 1231 / 150000 | consumed_samples: 78784 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785413E+05 | tokens_per_sec_per_gpu: 9.039707E+03 | global_batch_size: 64 | lm_loss: 1.098943E+01 | lr: 1.846E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 40807952.000 [ip-26-0-154-121:0]:06/21/2023 17:31:34 [INFO|DP=0|PP=0|TP=0]: iteration: 1232 / 150000 | consumed_samples: 78848 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787697E+05 | tokens_per_sec_per_gpu: 9.043276E+03 | global_batch_size: 64 | lm_loss: 1.099101E+01 | lr: 1.848E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 59486580.000 [ip-26-0-154-121:0]:06/21/2023 17:31:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1233 / 150000 | consumed_samples: 78912 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784291E+05 | tokens_per_sec_per_gpu: 9.037955E+03 | global_batch_size: 64 | lm_loss: 1.099030E+01 | lr: 1.849E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 55289824.000 [ip-26-0-154-121:0]:06/21/2023 17:31:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1234 / 150000 | consumed_samples: 78976 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784827E+05 | tokens_per_sec_per_gpu: 9.038792E+03 | global_batch_size: 64 | lm_loss: 1.098703E+01 | lr: 1.851E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 46129368.000 [ip-26-0-154-121:0]:06/21/2023 17:31:36 [INFO|DP=0|PP=0|TP=0]: iteration: 1235 / 150000 | consumed_samples: 79040 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787733E+05 | tokens_per_sec_per_gpu: 9.043333E+03 | global_batch_size: 64 | lm_loss: 1.099125E+01 | lr: 1.852E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 54919476.000 [ip-26-0-154-121:0]:06/21/2023 17:31:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1236 / 150000 | consumed_samples: 79104 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784456E+05 | tokens_per_sec_per_gpu: 9.038212E+03 | global_batch_size: 64 | lm_loss: 1.098412E+01 | lr: 1.854E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 41219196.000 [ip-26-0-154-121:0]:06/21/2023 17:31:38 [INFO|DP=0|PP=0|TP=0]: iteration: 1237 / 150000 | consumed_samples: 79168 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785113E+05 | tokens_per_sec_per_gpu: 9.039239E+03 | global_batch_size: 64 | lm_loss: 1.098535E+01 | lr: 1.856E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 52943340.000 [ip-26-0-154-121:0]:06/21/2023 17:31:39 [INFO|DP=0|PP=0|TP=0]: iteration: 1238 / 150000 | consumed_samples: 79232 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786753E+05 | tokens_per_sec_per_gpu: 9.041801E+03 | global_batch_size: 64 | lm_loss: 1.098724E+01 | lr: 1.857E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 50035352.000 [ip-26-0-154-121:0]:06/21/2023 17:31:40 [INFO|DP=0|PP=0|TP=0]: iteration: 1239 / 150000 | consumed_samples: 79296 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782677E+05 | tokens_per_sec_per_gpu: 9.035433E+03 | global_batch_size: 64 | lm_loss: 1.097667E+01 | lr: 1.858E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 35472964.000 [ip-26-0-154-121:0]:06/21/2023 17:31:41 [INFO|DP=0|PP=0|TP=0]: iteration: 1240 / 150000 | consumed_samples: 79360 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784691E+05 | tokens_per_sec_per_gpu: 9.038580E+03 | global_batch_size: 64 | lm_loss: 1.097994E+01 | lr: 1.860E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 30995018.000 [ip-26-0-154-121:0]:06/21/2023 17:31:42 [INFO|DP=0|PP=0|TP=0]: iteration: 1241 / 150000 | consumed_samples: 79424 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783680E+05 | tokens_per_sec_per_gpu: 9.036999E+03 | global_batch_size: 64 | lm_loss: 1.098371E+01 | lr: 1.861E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 31765804.000 [ip-26-0-154-121:0]:06/21/2023 17:31:43 [INFO|DP=0|PP=0|TP=0]: iteration: 1242 / 150000 | consumed_samples: 79488 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784170E+05 | tokens_per_sec_per_gpu: 9.037765E+03 | global_batch_size: 64 | lm_loss: 1.097966E+01 | lr: 1.863E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 29634388.000 [ip-26-0-154-121:0]:06/21/2023 17:31:44 [INFO|DP=0|PP=0|TP=0]: iteration: 1243 / 150000 | consumed_samples: 79552 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785792E+05 | tokens_per_sec_per_gpu: 9.040300E+03 | global_batch_size: 64 | lm_loss: 1.098068E+01 | lr: 1.864E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 40849952.000 [ip-26-0-154-121:0]:06/21/2023 17:31:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1244 / 150000 | consumed_samples: 79616 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783772E+05 | tokens_per_sec_per_gpu: 9.037144E+03 | global_batch_size: 64 | lm_loss: 1.096807E+01 | lr: 1.866E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 39564976.000 [ip-26-0-154-121:0]:06/21/2023 17:31:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1245 / 150000 | consumed_samples: 79680 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789245E+05 | tokens_per_sec_per_gpu: 9.045695E+03 | global_batch_size: 64 | lm_loss: 1.097754E+01 | lr: 1.867E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 40068840.000 [ip-26-0-154-121:0]:06/21/2023 17:31:46 [INFO|DP=0|PP=0|TP=0]: iteration: 1246 / 150000 | consumed_samples: 79744 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786285E+05 | tokens_per_sec_per_gpu: 9.041070E+03 | global_batch_size: 64 | lm_loss: 1.097846E+01 | lr: 1.869E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 34292820.000 [ip-26-0-154-121:0]:06/21/2023 17:31:47 [INFO|DP=0|PP=0|TP=0]: iteration: 1247 / 150000 | consumed_samples: 79808 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787883E+05 | tokens_per_sec_per_gpu: 9.043567E+03 | global_batch_size: 64 | lm_loss: 1.097615E+01 | lr: 1.870E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 49812496.000 [ip-26-0-154-121:0]:06/21/2023 17:31:48 [INFO|DP=0|PP=0|TP=0]: iteration: 1248 / 150000 | consumed_samples: 79872 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783459E+05 | tokens_per_sec_per_gpu: 9.036655E+03 | global_batch_size: 64 | lm_loss: 1.098112E+01 | lr: 1.872E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 33712840.000 [ip-26-0-154-121:0]:06/21/2023 17:31:49 [INFO|DP=0|PP=0|TP=0]: iteration: 1249 / 150000 | consumed_samples: 79936 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789018E+05 | tokens_per_sec_per_gpu: 9.045340E+03 | global_batch_size: 64 | lm_loss: 1.097438E+01 | lr: 1.874E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 44061320.000 [ip-26-0-154-121:0]:06/21/2023 17:31:50 [INFO|DP=0|PP=0|TP=0]: iteration: 1250 / 150000 | consumed_samples: 80000 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780382E+05 | tokens_per_sec_per_gpu: 9.031847E+03 | global_batch_size: 64 | lm_loss: 1.098087E+01 | lr: 1.875E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 34131528.000 [ip-26-0-154-121:0]:06/21/2023 17:31:51 [INFO|DP=0|PP=0|TP=0]: iteration: 1251 / 150000 | consumed_samples: 80064 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782860E+05 | tokens_per_sec_per_gpu: 9.035719E+03 | global_batch_size: 64 | lm_loss: 1.097948E+01 | lr: 1.876E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 29539682.000 [ip-26-0-154-121:0]:06/21/2023 17:31:52 [INFO|DP=0|PP=0|TP=0]: iteration: 1252 / 150000 | consumed_samples: 80128 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780636E+05 | tokens_per_sec_per_gpu: 9.032244E+03 | global_batch_size: 64 | lm_loss: 1.079796E+01 | lr: 1.878E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 36133520.000 [ip-26-0-154-121:0]:06/21/2023 17:31:53 [INFO|DP=0|PP=0|TP=0]: iteration: 1253 / 150000 | consumed_samples: 80192 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783260E+05 | tokens_per_sec_per_gpu: 9.036344E+03 | global_batch_size: 64 | lm_loss: 1.098765E+01 | lr: 1.879E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 32743596.000 [ip-26-0-154-121:0]:06/21/2023 17:31:54 [INFO|DP=0|PP=0|TP=0]: iteration: 1254 / 150000 | consumed_samples: 80256 | elapsed_time_per_iteration_ms: 913.8 | tokens_per_sec: 5.737643E+05 | tokens_per_sec_per_gpu: 8.965067E+03 | global_batch_size: 64 | lm_loss: 1.097751E+01 | lr: 1.881E-04 | model_tflops_per_gpu: 103.56 | hardware_tflops_per_gpu: 103.56 | grad_norm: 34493596.000 [ip-26-0-154-121:0]:06/21/2023 17:31:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1255 / 150000 | consumed_samples: 80320 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786197E+05 | tokens_per_sec_per_gpu: 9.040932E+03 | global_batch_size: 64 | lm_loss: 1.098176E+01 | lr: 1.882E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 37981508.000 [ip-26-0-154-121:0]:06/21/2023 17:31:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1256 / 150000 | consumed_samples: 80384 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787286E+05 | tokens_per_sec_per_gpu: 9.042634E+03 | global_batch_size: 64 | lm_loss: 1.098326E+01 | lr: 1.884E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 31958844.000 [ip-26-0-154-121:0]:06/21/2023 17:31:56 [INFO|DP=0|PP=0|TP=0]: iteration: 1257 / 150000 | consumed_samples: 80448 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788027E+05 | tokens_per_sec_per_gpu: 9.043793E+03 | global_batch_size: 64 | lm_loss: 1.097978E+01 | lr: 1.886E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 23331930.000 [ip-26-0-154-121:0]:06/21/2023 17:31:57 [INFO|DP=0|PP=0|TP=0]: iteration: 1258 / 150000 | consumed_samples: 80512 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783473E+05 | tokens_per_sec_per_gpu: 9.036676E+03 | global_batch_size: 64 | lm_loss: 1.097127E+01 | lr: 1.887E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 30724688.000 [ip-26-0-154-121:0]:06/21/2023 17:31:58 [INFO|DP=0|PP=0|TP=0]: iteration: 1259 / 150000 | consumed_samples: 80576 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785615E+05 | tokens_per_sec_per_gpu: 9.040024E+03 | global_batch_size: 64 | lm_loss: 1.098430E+01 | lr: 1.888E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 29776816.000 [ip-26-0-154-121:0]:06/21/2023 17:31:59 [INFO|DP=0|PP=0|TP=0]: iteration: 1260 / 150000 | consumed_samples: 80640 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786765E+05 | tokens_per_sec_per_gpu: 9.041820E+03 | global_batch_size: 64 | lm_loss: 1.097576E+01 | lr: 1.890E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 26017860.000 [ip-26-0-154-121:0]:06/21/2023 17:32:00 [INFO|DP=0|PP=0|TP=0]: iteration: 1261 / 150000 | consumed_samples: 80704 | elapsed_time_per_iteration_ms: 916.8 | tokens_per_sec: 5.718975E+05 | tokens_per_sec_per_gpu: 8.935899E+03 | global_batch_size: 64 | lm_loss: 1.097458E+01 | lr: 1.891E-04 | model_tflops_per_gpu: 103.22 | hardware_tflops_per_gpu: 103.22 | grad_norm: 24897904.000 [ip-26-0-154-121:0]:06/21/2023 17:32:01 [INFO|DP=0|PP=0|TP=0]: iteration: 1262 / 150000 | consumed_samples: 80768 | elapsed_time_per_iteration_ms: 910.8 | tokens_per_sec: 5.756229E+05 | tokens_per_sec_per_gpu: 8.994107E+03 | global_batch_size: 64 | lm_loss: 1.097989E+01 | lr: 1.893E-04 | model_tflops_per_gpu: 103.89 | hardware_tflops_per_gpu: 103.89 | grad_norm: 33811412.000 [ip-26-0-154-121:0]:06/21/2023 17:32:02 [INFO|DP=0|PP=0|TP=0]: iteration: 1263 / 150000 | consumed_samples: 80832 | elapsed_time_per_iteration_ms: 909.4 | tokens_per_sec: 5.765033E+05 | tokens_per_sec_per_gpu: 9.007863E+03 | global_batch_size: 64 | lm_loss: 1.097915E+01 | lr: 1.894E-04 | model_tflops_per_gpu: 104.05 | hardware_tflops_per_gpu: 104.05 | grad_norm: 36144984.000 [ip-26-0-154-121:0]:06/21/2023 17:32:03 [INFO|DP=0|PP=0|TP=0]: iteration: 1264 / 150000 | consumed_samples: 80896 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778506E+05 | tokens_per_sec_per_gpu: 9.028916E+03 | global_batch_size: 64 | lm_loss: 1.098187E+01 | lr: 1.896E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 38328480.000 [ip-26-0-154-121:0]:06/21/2023 17:32:04 [INFO|DP=0|PP=0|TP=0]: iteration: 1265 / 150000 | consumed_samples: 80960 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783055E+05 | tokens_per_sec_per_gpu: 9.036023E+03 | global_batch_size: 64 | lm_loss: 1.096849E+01 | lr: 1.897E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 33545314.000 [ip-26-0-154-121:0]:06/21/2023 17:32:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1266 / 150000 | consumed_samples: 81024 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784227E+05 | tokens_per_sec_per_gpu: 9.037855E+03 | global_batch_size: 64 | lm_loss: 1.096671E+01 | lr: 1.899E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 49131384.000 [ip-26-0-154-121:0]:06/21/2023 17:32:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1267 / 150000 | consumed_samples: 81088 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782107E+05 | tokens_per_sec_per_gpu: 9.034542E+03 | global_batch_size: 64 | lm_loss: 1.097593E+01 | lr: 1.900E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 55787704.000 [ip-26-0-154-121:0]:06/21/2023 17:32:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1268 / 150000 | consumed_samples: 81152 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783423E+05 | tokens_per_sec_per_gpu: 9.036598E+03 | global_batch_size: 64 | lm_loss: 1.097051E+01 | lr: 1.902E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 44202152.000 [ip-26-0-154-121:0]:06/21/2023 17:32:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1269 / 150000 | consumed_samples: 81216 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780833E+05 | tokens_per_sec_per_gpu: 9.032552E+03 | global_batch_size: 64 | lm_loss: 1.093274E+01 | lr: 1.903E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 50001964.000 [ip-26-0-154-121:0]:06/21/2023 17:32:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1270 / 150000 | consumed_samples: 81280 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782423E+05 | tokens_per_sec_per_gpu: 9.035037E+03 | global_batch_size: 64 | lm_loss: 1.096750E+01 | lr: 1.905E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 39252400.000 [ip-26-0-154-121:0]:06/21/2023 17:32:09 [INFO|DP=0|PP=0|TP=0]: iteration: 1271 / 150000 | consumed_samples: 81344 | elapsed_time_per_iteration_ms: 922.6 | tokens_per_sec: 5.682705E+05 | tokens_per_sec_per_gpu: 8.879226E+03 | global_batch_size: 64 | lm_loss: 1.096323E+01 | lr: 1.906E-04 | model_tflops_per_gpu: 102.56 | hardware_tflops_per_gpu: 102.56 | grad_norm: 45425828.000 [ip-26-0-154-121:0]:06/21/2023 17:32:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1272 / 150000 | consumed_samples: 81408 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785393E+05 | tokens_per_sec_per_gpu: 9.039677E+03 | global_batch_size: 64 | lm_loss: 1.096675E+01 | lr: 1.908E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 47264476.000 [ip-26-0-154-121:0]:06/21/2023 17:32:11 [INFO|DP=0|PP=0|TP=0]: iteration: 1273 / 150000 | consumed_samples: 81472 | elapsed_time_per_iteration_ms: 920.4 | tokens_per_sec: 5.696227E+05 | tokens_per_sec_per_gpu: 8.900354E+03 | global_batch_size: 64 | lm_loss: 1.097073E+01 | lr: 1.909E-04 | model_tflops_per_gpu: 102.81 | hardware_tflops_per_gpu: 102.81 | grad_norm: 43985960.000 [ip-26-0-154-121:0]:06/21/2023 17:32:12 [INFO|DP=0|PP=0|TP=0]: iteration: 1274 / 150000 | consumed_samples: 81536 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784247E+05 | tokens_per_sec_per_gpu: 9.037886E+03 | global_batch_size: 64 | lm_loss: 1.096199E+01 | lr: 1.911E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 31343234.000 [ip-26-0-154-121:0]:06/21/2023 17:32:13 [INFO|DP=0|PP=0|TP=0]: iteration: 1275 / 150000 | consumed_samples: 81600 | elapsed_time_per_iteration_ms: 917.3 | tokens_per_sec: 5.715567E+05 | tokens_per_sec_per_gpu: 8.930573E+03 | global_batch_size: 64 | lm_loss: 1.096469E+01 | lr: 1.913E-04 | model_tflops_per_gpu: 103.16 | hardware_tflops_per_gpu: 103.16 | grad_norm: 32974212.000 [ip-26-0-154-121:0]:06/21/2023 17:32:14 [INFO|DP=0|PP=0|TP=0]: iteration: 1276 / 150000 | consumed_samples: 81664 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789501E+05 | tokens_per_sec_per_gpu: 9.046095E+03 | global_batch_size: 64 | lm_loss: 1.096785E+01 | lr: 1.914E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 44287732.000 [ip-26-0-154-121:0]:06/21/2023 17:32:15 [INFO|DP=0|PP=0|TP=0]: iteration: 1277 / 150000 | consumed_samples: 81728 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783893E+05 | tokens_per_sec_per_gpu: 9.037332E+03 | global_batch_size: 64 | lm_loss: 1.096234E+01 | lr: 1.915E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 41316196.000 [ip-26-0-154-121:0]:06/21/2023 17:32:15 [INFO|DP=0|PP=0|TP=0]: iteration: 1278 / 150000 | consumed_samples: 81792 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784440E+05 | tokens_per_sec_per_gpu: 9.038188E+03 | global_batch_size: 64 | lm_loss: 1.096517E+01 | lr: 1.917E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 46333264.000 [ip-26-0-154-121:0]:06/21/2023 17:32:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1279 / 150000 | consumed_samples: 81856 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779831E+05 | tokens_per_sec_per_gpu: 9.030985E+03 | global_batch_size: 64 | lm_loss: 1.096731E+01 | lr: 1.919E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 60337232.000 [ip-26-0-154-121:0]:06/21/2023 17:32:17 [INFO|DP=0|PP=0|TP=0]: iteration: 1280 / 150000 | consumed_samples: 81920 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784299E+05 | tokens_per_sec_per_gpu: 9.037967E+03 | global_batch_size: 64 | lm_loss: 1.096420E+01 | lr: 1.920E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 61734500.000 [ip-26-0-154-121:0]:06/21/2023 17:32:18 [INFO|DP=0|PP=0|TP=0]: iteration: 1281 / 150000 | consumed_samples: 81984 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778524E+05 | tokens_per_sec_per_gpu: 9.028944E+03 | global_batch_size: 64 | lm_loss: 1.097460E+01 | lr: 1.921E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 43322740.000 [ip-26-0-154-121:0]:06/21/2023 17:32:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1282 / 150000 | consumed_samples: 82048 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780915E+05 | tokens_per_sec_per_gpu: 9.032680E+03 | global_batch_size: 64 | lm_loss: 1.096767E+01 | lr: 1.923E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 37254764.000 [ip-26-0-154-121:0]:06/21/2023 17:32:20 [INFO|DP=0|PP=0|TP=0]: iteration: 1283 / 150000 | consumed_samples: 82112 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777222E+05 | tokens_per_sec_per_gpu: 9.026909E+03 | global_batch_size: 64 | lm_loss: 1.097324E+01 | lr: 1.925E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 33786368.000 [ip-26-0-154-121:0]:06/21/2023 17:32:21 [INFO|DP=0|PP=0|TP=0]: iteration: 1284 / 150000 | consumed_samples: 82176 | elapsed_time_per_iteration_ms: 939.7 | tokens_per_sec: 5.579224E+05 | tokens_per_sec_per_gpu: 8.717538E+03 | global_batch_size: 64 | lm_loss: 1.096952E+01 | lr: 1.926E-04 | model_tflops_per_gpu: 100.70 | hardware_tflops_per_gpu: 100.70 | grad_norm: 36419256.000 [ip-26-0-154-121:0]:06/21/2023 17:32:22 [INFO|DP=0|PP=0|TP=0]: iteration: 1285 / 150000 | consumed_samples: 82240 | elapsed_time_per_iteration_ms: 912.5 | tokens_per_sec: 5.745442E+05 | tokens_per_sec_per_gpu: 8.977254E+03 | global_batch_size: 64 | lm_loss: 1.097339E+01 | lr: 1.927E-04 | model_tflops_per_gpu: 103.70 | hardware_tflops_per_gpu: 103.70 | grad_norm: 36801480.000 [ip-26-0-154-121:0]:06/21/2023 17:32:23 [INFO|DP=0|PP=0|TP=0]: iteration: 1286 / 150000 | consumed_samples: 82304 | elapsed_time_per_iteration_ms: 937.4 | tokens_per_sec: 5.593088E+05 | tokens_per_sec_per_gpu: 8.739200E+03 | global_batch_size: 64 | lm_loss: 1.098304E+01 | lr: 1.929E-04 | model_tflops_per_gpu: 100.95 | hardware_tflops_per_gpu: 100.95 | grad_norm: 34854344.000 [ip-26-0-154-121:0]:06/21/2023 17:32:24 [INFO|DP=0|PP=0|TP=0]: iteration: 1287 / 150000 | consumed_samples: 82368 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787810E+05 | tokens_per_sec_per_gpu: 9.043452E+03 | global_batch_size: 64 | lm_loss: 1.096679E+01 | lr: 1.930E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 47147240.000 [ip-26-0-154-121:0]:06/21/2023 17:32:25 [INFO|DP=0|PP=0|TP=0]: iteration: 1288 / 150000 | consumed_samples: 82432 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783643E+05 | tokens_per_sec_per_gpu: 9.036942E+03 | global_batch_size: 64 | lm_loss: 1.098083E+01 | lr: 1.932E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 39532688.000 [ip-26-0-154-121:0]:06/21/2023 17:32:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1289 / 150000 | consumed_samples: 82496 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786553E+05 | tokens_per_sec_per_gpu: 9.041489E+03 | global_batch_size: 64 | lm_loss: 1.096576E+01 | lr: 1.933E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 46981200.000 [ip-26-0-154-121:0]:06/21/2023 17:32:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1290 / 150000 | consumed_samples: 82560 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786826E+05 | tokens_per_sec_per_gpu: 9.041915E+03 | global_batch_size: 64 | lm_loss: 1.095488E+01 | lr: 1.935E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 45014920.000 [ip-26-0-154-121:0]:06/21/2023 17:32:27 [INFO|DP=0|PP=0|TP=0]: iteration: 1291 / 150000 | consumed_samples: 82624 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785122E+05 | tokens_per_sec_per_gpu: 9.039253E+03 | global_batch_size: 64 | lm_loss: 1.097306E+01 | lr: 1.937E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 48299416.000 [ip-26-0-154-121:0]:06/21/2023 17:32:28 [INFO|DP=0|PP=0|TP=0]: iteration: 1292 / 150000 | consumed_samples: 82688 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788827E+05 | tokens_per_sec_per_gpu: 9.045043E+03 | global_batch_size: 64 | lm_loss: 1.096621E+01 | lr: 1.938E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 45266892.000 [ip-26-0-154-121:0]:06/21/2023 17:32:29 [INFO|DP=0|PP=0|TP=0]: iteration: 1293 / 150000 | consumed_samples: 82752 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784553E+05 | tokens_per_sec_per_gpu: 9.038364E+03 | global_batch_size: 64 | lm_loss: 1.096931E+01 | lr: 1.939E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 37491248.000 [ip-26-0-154-121:0]:06/21/2023 17:32:30 [INFO|DP=0|PP=0|TP=0]: iteration: 1294 / 150000 | consumed_samples: 82816 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785230E+05 | tokens_per_sec_per_gpu: 9.039422E+03 | global_batch_size: 64 | lm_loss: 1.096617E+01 | lr: 1.941E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 39671220.000 [ip-26-0-154-121:0]:06/21/2023 17:32:31 [INFO|DP=0|PP=0|TP=0]: iteration: 1295 / 150000 | consumed_samples: 82880 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784478E+05 | tokens_per_sec_per_gpu: 9.038247E+03 | global_batch_size: 64 | lm_loss: 1.097768E+01 | lr: 1.942E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 41115784.000 [ip-26-0-154-121:0]:06/21/2023 17:32:32 [INFO|DP=0|PP=0|TP=0]: iteration: 1296 / 150000 | consumed_samples: 82944 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781794E+05 | tokens_per_sec_per_gpu: 9.034053E+03 | global_batch_size: 64 | lm_loss: 1.096608E+01 | lr: 1.944E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 35199600.000 [ip-26-0-154-121:0]:06/21/2023 17:32:33 [INFO|DP=0|PP=0|TP=0]: iteration: 1297 / 150000 | consumed_samples: 83008 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783073E+05 | tokens_per_sec_per_gpu: 9.036051E+03 | global_batch_size: 64 | lm_loss: 1.086399E+01 | lr: 1.945E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 44930436.000 [ip-26-0-154-121:0]:06/21/2023 17:32:34 [INFO|DP=0|PP=0|TP=0]: iteration: 1298 / 150000 | consumed_samples: 83072 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784521E+05 | tokens_per_sec_per_gpu: 9.038314E+03 | global_batch_size: 64 | lm_loss: 1.097094E+01 | lr: 1.947E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 35522820.000 [ip-26-0-154-121:0]:06/21/2023 17:32:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1299 / 150000 | consumed_samples: 83136 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788161E+05 | tokens_per_sec_per_gpu: 9.044002E+03 | global_batch_size: 64 | lm_loss: 1.096426E+01 | lr: 1.948E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 31195436.000 [ip-26-0-154-121:0]:06/21/2023 17:32:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1300 / 150000 | consumed_samples: 83200 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786354E+05 | tokens_per_sec_per_gpu: 9.041178E+03 | global_batch_size: 64 | lm_loss: 1.096240E+01 | lr: 1.950E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 40137428.000 [ip-26-0-154-121:0]:06/21/2023 17:32:36 [INFO|DP=0|PP=0|TP=0]: iteration: 1301 / 150000 | consumed_samples: 83264 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787765E+05 | tokens_per_sec_per_gpu: 9.043383E+03 | global_batch_size: 64 | lm_loss: 1.095028E+01 | lr: 1.952E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 41170300.000 [ip-26-0-154-121:0]:06/21/2023 17:32:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1302 / 150000 | consumed_samples: 83328 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783505E+05 | tokens_per_sec_per_gpu: 9.036726E+03 | global_batch_size: 64 | lm_loss: 1.095951E+01 | lr: 1.953E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 36275272.000 [ip-26-0-154-121:0]:06/21/2023 17:32:38 [INFO|DP=0|PP=0|TP=0]: iteration: 1303 / 150000 | consumed_samples: 83392 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788075E+05 | tokens_per_sec_per_gpu: 9.043867E+03 | global_batch_size: 64 | lm_loss: 1.096847E+01 | lr: 1.954E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 52768140.000 [ip-26-0-154-121:0]:06/21/2023 17:32:39 [INFO|DP=0|PP=0|TP=0]: iteration: 1304 / 150000 | consumed_samples: 83456 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786504E+05 | tokens_per_sec_per_gpu: 9.041413E+03 | global_batch_size: 64 | lm_loss: 1.091721E+01 | lr: 1.956E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 49819740.000 [ip-26-0-154-121:0]:06/21/2023 17:32:40 [INFO|DP=0|PP=0|TP=0]: iteration: 1305 / 150000 | consumed_samples: 83520 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786885E+05 | tokens_per_sec_per_gpu: 9.042008E+03 | global_batch_size: 64 | lm_loss: 1.097160E+01 | lr: 1.957E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 40103544.000 [ip-26-0-154-121:0]:06/21/2023 17:32:41 [INFO|DP=0|PP=0|TP=0]: iteration: 1306 / 150000 | consumed_samples: 83584 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786137E+05 | tokens_per_sec_per_gpu: 9.040840E+03 | global_batch_size: 64 | lm_loss: 1.096713E+01 | lr: 1.959E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 34827160.000 [ip-26-0-154-121:0]:06/21/2023 17:32:42 [INFO|DP=0|PP=0|TP=0]: iteration: 1307 / 150000 | consumed_samples: 83648 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788151E+05 | tokens_per_sec_per_gpu: 9.043986E+03 | global_batch_size: 64 | lm_loss: 1.096872E+01 | lr: 1.960E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 30917408.000 [ip-26-0-154-121:0]:06/21/2023 17:32:43 [INFO|DP=0|PP=0|TP=0]: iteration: 1308 / 150000 | consumed_samples: 83712 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790515E+05 | tokens_per_sec_per_gpu: 9.047679E+03 | global_batch_size: 64 | lm_loss: 1.096544E+01 | lr: 1.962E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 30977614.000 [ip-26-0-154-121:0]:06/21/2023 17:32:44 [INFO|DP=0|PP=0|TP=0]: iteration: 1309 / 150000 | consumed_samples: 83776 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788139E+05 | tokens_per_sec_per_gpu: 9.043967E+03 | global_batch_size: 64 | lm_loss: 1.096343E+01 | lr: 1.963E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 45212948.000 [ip-26-0-154-121:0]:06/21/2023 17:32:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1310 / 150000 | consumed_samples: 83840 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788399E+05 | tokens_per_sec_per_gpu: 9.044374E+03 | global_batch_size: 64 | lm_loss: 1.096823E+01 | lr: 1.965E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 38931264.000 [ip-26-0-154-121:0]:06/21/2023 17:32:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1311 / 150000 | consumed_samples: 83904 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786582E+05 | tokens_per_sec_per_gpu: 9.041534E+03 | global_batch_size: 64 | lm_loss: 1.096012E+01 | lr: 1.966E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 44301452.000 [ip-26-0-154-121:0]:06/21/2023 17:32:46 [INFO|DP=0|PP=0|TP=0]: iteration: 1312 / 150000 | consumed_samples: 83968 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787796E+05 | tokens_per_sec_per_gpu: 9.043431E+03 | global_batch_size: 64 | lm_loss: 1.096632E+01 | lr: 1.968E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 42181580.000 [ip-26-0-154-121:0]:06/21/2023 17:32:47 [INFO|DP=0|PP=0|TP=0]: iteration: 1313 / 150000 | consumed_samples: 84032 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784949E+05 | tokens_per_sec_per_gpu: 9.038982E+03 | global_batch_size: 64 | lm_loss: 1.095725E+01 | lr: 1.969E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 36541052.000 [ip-26-0-154-121:0]:06/21/2023 17:32:48 [INFO|DP=0|PP=0|TP=0]: iteration: 1314 / 150000 | consumed_samples: 84096 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779890E+05 | tokens_per_sec_per_gpu: 9.031078E+03 | global_batch_size: 64 | lm_loss: 1.096159E+01 | lr: 1.971E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 37594304.000 [ip-26-0-154-121:0]:06/21/2023 17:32:49 [INFO|DP=0|PP=0|TP=0]: iteration: 1315 / 150000 | consumed_samples: 84160 | elapsed_time_per_iteration_ms: 908.1 | tokens_per_sec: 5.773471E+05 | tokens_per_sec_per_gpu: 9.021048E+03 | global_batch_size: 64 | lm_loss: 1.096461E+01 | lr: 1.972E-04 | model_tflops_per_gpu: 104.20 | hardware_tflops_per_gpu: 104.20 | grad_norm: 40149464.000 [ip-26-0-154-121:0]:06/21/2023 17:32:50 [INFO|DP=0|PP=0|TP=0]: iteration: 1316 / 150000 | consumed_samples: 84224 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780599E+05 | tokens_per_sec_per_gpu: 9.032187E+03 | global_batch_size: 64 | lm_loss: 1.096397E+01 | lr: 1.974E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 35210532.000 [ip-26-0-154-121:0]:06/21/2023 17:32:51 [INFO|DP=0|PP=0|TP=0]: iteration: 1317 / 150000 | consumed_samples: 84288 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785876E+05 | tokens_per_sec_per_gpu: 9.040431E+03 | global_batch_size: 64 | lm_loss: 1.096267E+01 | lr: 1.975E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 56394964.000 [ip-26-0-154-121:0]:06/21/2023 17:32:52 [INFO|DP=0|PP=0|TP=0]: iteration: 1318 / 150000 | consumed_samples: 84352 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787156E+05 | tokens_per_sec_per_gpu: 9.042431E+03 | global_batch_size: 64 | lm_loss: 1.096581E+01 | lr: 1.977E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 47349284.000 [ip-26-0-154-121:0]:06/21/2023 17:32:53 [INFO|DP=0|PP=0|TP=0]: iteration: 1319 / 150000 | consumed_samples: 84416 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787098E+05 | tokens_per_sec_per_gpu: 9.042341E+03 | global_batch_size: 64 | lm_loss: 1.096617E+01 | lr: 1.978E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 48437144.000 [ip-26-0-154-121:0]:06/21/2023 17:32:54 [INFO|DP=0|PP=0|TP=0]: iteration: 1320 / 150000 | consumed_samples: 84480 | elapsed_time_per_iteration_ms: 913.2 | tokens_per_sec: 5.740910E+05 | tokens_per_sec_per_gpu: 8.970171E+03 | global_batch_size: 64 | lm_loss: 1.096575E+01 | lr: 1.980E-04 | model_tflops_per_gpu: 103.62 | hardware_tflops_per_gpu: 103.62 | grad_norm: 43796588.000 [ip-26-0-154-121:0]:06/21/2023 17:32:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1321 / 150000 | consumed_samples: 84544 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785612E+05 | tokens_per_sec_per_gpu: 9.040019E+03 | global_batch_size: 64 | lm_loss: 1.095570E+01 | lr: 1.981E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 39799608.000 [ip-26-0-154-121:0]:06/21/2023 17:32:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1322 / 150000 | consumed_samples: 84608 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785944E+05 | tokens_per_sec_per_gpu: 9.040538E+03 | global_batch_size: 64 | lm_loss: 1.095778E+01 | lr: 1.983E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 51765484.000 [ip-26-0-154-121:0]:06/21/2023 17:32:56 [INFO|DP=0|PP=0|TP=0]: iteration: 1323 / 150000 | consumed_samples: 84672 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787021E+05 | tokens_per_sec_per_gpu: 9.042220E+03 | global_batch_size: 64 | lm_loss: 1.096885E+01 | lr: 1.984E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 42552100.000 [ip-26-0-154-121:0]:06/21/2023 17:32:57 [INFO|DP=0|PP=0|TP=0]: iteration: 1324 / 150000 | consumed_samples: 84736 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790571E+05 | tokens_per_sec_per_gpu: 9.047767E+03 | global_batch_size: 64 | lm_loss: 1.095471E+01 | lr: 1.986E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 41388216.000 [ip-26-0-154-121:0]:06/21/2023 17:32:58 [INFO|DP=0|PP=0|TP=0]: iteration: 1325 / 150000 | consumed_samples: 84800 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789452E+05 | tokens_per_sec_per_gpu: 9.046019E+03 | global_batch_size: 64 | lm_loss: 1.095667E+01 | lr: 1.987E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 36713384.000 [ip-26-0-154-121:0]:06/21/2023 17:32:59 [INFO|DP=0|PP=0|TP=0]: iteration: 1326 / 150000 | consumed_samples: 84864 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788161E+05 | tokens_per_sec_per_gpu: 9.044002E+03 | global_batch_size: 64 | lm_loss: 1.095905E+01 | lr: 1.989E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 31350938.000 [ip-26-0-154-121:0]:06/21/2023 17:33:00 [INFO|DP=0|PP=0|TP=0]: iteration: 1327 / 150000 | consumed_samples: 84928 | elapsed_time_per_iteration_ms: 904.7 | tokens_per_sec: 5.795273E+05 | tokens_per_sec_per_gpu: 9.055114E+03 | global_batch_size: 64 | lm_loss: 1.095900E+01 | lr: 1.990E-04 | model_tflops_per_gpu: 104.60 | hardware_tflops_per_gpu: 104.60 | grad_norm: 30047500.000 [ip-26-0-154-121:0]:06/21/2023 17:33:01 [INFO|DP=0|PP=0|TP=0]: iteration: 1328 / 150000 | consumed_samples: 84992 | elapsed_time_per_iteration_ms: 915.3 | tokens_per_sec: 5.727886E+05 | tokens_per_sec_per_gpu: 8.949823E+03 | global_batch_size: 64 | lm_loss: 1.096321E+01 | lr: 1.992E-04 | model_tflops_per_gpu: 103.38 | hardware_tflops_per_gpu: 103.38 | grad_norm: 25890418.000 [ip-26-0-154-121:0]:06/21/2023 17:33:02 [INFO|DP=0|PP=0|TP=0]: iteration: 1329 / 150000 | consumed_samples: 85056 | elapsed_time_per_iteration_ms: 933.3 | tokens_per_sec: 5.617447E+05 | tokens_per_sec_per_gpu: 8.777261E+03 | global_batch_size: 64 | lm_loss: 1.096152E+01 | lr: 1.993E-04 | model_tflops_per_gpu: 101.39 | hardware_tflops_per_gpu: 101.39 | grad_norm: 30748754.000 [ip-26-0-154-121:0]:06/21/2023 17:33:03 [INFO|DP=0|PP=0|TP=0]: iteration: 1330 / 150000 | consumed_samples: 85120 | elapsed_time_per_iteration_ms: 996.8 | tokens_per_sec: 5.259642E+05 | tokens_per_sec_per_gpu: 8.218191E+03 | global_batch_size: 64 | lm_loss: 1.096667E+01 | lr: 1.995E-04 | model_tflops_per_gpu: 94.93 | hardware_tflops_per_gpu: 94.93 | grad_norm: 38000152.000 [ip-26-0-154-121:0]:06/21/2023 17:33:04 [INFO|DP=0|PP=0|TP=0]: iteration: 1331 / 150000 | consumed_samples: 85184 | elapsed_time_per_iteration_ms: 923.6 | tokens_per_sec: 5.676425E+05 | tokens_per_sec_per_gpu: 8.869414E+03 | global_batch_size: 64 | lm_loss: 1.095219E+01 | lr: 1.997E-04 | model_tflops_per_gpu: 102.45 | hardware_tflops_per_gpu: 102.45 | grad_norm: 32963934.000 [ip-26-0-154-121:0]:06/21/2023 17:33:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1332 / 150000 | consumed_samples: 85248 | elapsed_time_per_iteration_ms: 1007.7 | tokens_per_sec: 5.202976E+05 | tokens_per_sec_per_gpu: 8.129650E+03 | global_batch_size: 64 | lm_loss: 1.096747E+01 | lr: 1.998E-04 | model_tflops_per_gpu: 93.91 | hardware_tflops_per_gpu: 93.91 | grad_norm: 67514760.000 [ip-26-0-154-121:0]:06/21/2023 17:33:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1333 / 150000 | consumed_samples: 85312 | elapsed_time_per_iteration_ms: 927.4 | tokens_per_sec: 5.653422E+05 | tokens_per_sec_per_gpu: 8.833471E+03 | global_batch_size: 64 | lm_loss: 1.093923E+01 | lr: 1.999E-04 | model_tflops_per_gpu: 102.04 | hardware_tflops_per_gpu: 102.04 | grad_norm: 62753392.000 [ip-26-0-154-121:0]:06/21/2023 17:33:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1334 / 150000 | consumed_samples: 85376 | elapsed_time_per_iteration_ms: 916.1 | tokens_per_sec: 5.722778E+05 | tokens_per_sec_per_gpu: 8.941841E+03 | global_batch_size: 64 | lm_loss: 1.095691E+01 | lr: 2.001E-04 | model_tflops_per_gpu: 103.29 | hardware_tflops_per_gpu: 103.29 | grad_norm: 56689976.000 [ip-26-0-154-121:0]:06/21/2023 17:33:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1335 / 150000 | consumed_samples: 85440 | elapsed_time_per_iteration_ms: 931.4 | tokens_per_sec: 5.629268E+05 | tokens_per_sec_per_gpu: 8.795730E+03 | global_batch_size: 64 | lm_loss: 1.096109E+01 | lr: 2.002E-04 | model_tflops_per_gpu: 101.60 | hardware_tflops_per_gpu: 101.60 | grad_norm: 70843280.000 [ip-26-0-154-121:0]:06/21/2023 17:33:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1336 / 150000 | consumed_samples: 85504 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785069E+05 | tokens_per_sec_per_gpu: 9.039170E+03 | global_batch_size: 64 | lm_loss: 1.094839E+01 | lr: 2.004E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 50249876.000 [ip-26-0-154-121:0]:06/21/2023 17:33:09 [INFO|DP=0|PP=0|TP=0]: iteration: 1337 / 150000 | consumed_samples: 85568 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785235E+05 | tokens_per_sec_per_gpu: 9.039429E+03 | global_batch_size: 64 | lm_loss: 1.095188E+01 | lr: 2.005E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 55524308.000 [ip-26-0-154-121:0]:06/21/2023 17:33:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1338 / 150000 | consumed_samples: 85632 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786142E+05 | tokens_per_sec_per_gpu: 9.040847E+03 | global_batch_size: 64 | lm_loss: 1.096567E+01 | lr: 2.007E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 54765192.000 [ip-26-0-154-121:0]:06/21/2023 17:33:11 [INFO|DP=0|PP=0|TP=0]: iteration: 1339 / 150000 | consumed_samples: 85696 | elapsed_time_per_iteration_ms: 910.7 | tokens_per_sec: 5.757070E+05 | tokens_per_sec_per_gpu: 8.995421E+03 | global_batch_size: 64 | lm_loss: 1.095410E+01 | lr: 2.008E-04 | model_tflops_per_gpu: 103.91 | hardware_tflops_per_gpu: 103.91 | grad_norm: 38899556.000 [ip-26-0-154-121:0]:06/21/2023 17:33:12 [INFO|DP=0|PP=0|TP=0]: iteration: 1340 / 150000 | consumed_samples: 85760 | elapsed_time_per_iteration_ms: 948.6 | tokens_per_sec: 5.526963E+05 | tokens_per_sec_per_gpu: 8.635880E+03 | global_batch_size: 64 | lm_loss: 1.095883E+01 | lr: 2.010E-04 | model_tflops_per_gpu: 99.75 | hardware_tflops_per_gpu: 99.75 | grad_norm: 37346784.000 [ip-26-0-154-121:0]:06/21/2023 17:33:13 [INFO|DP=0|PP=0|TP=0]: iteration: 1341 / 150000 | consumed_samples: 85824 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786684E+05 | tokens_per_sec_per_gpu: 9.041694E+03 | global_batch_size: 64 | lm_loss: 1.095179E+01 | lr: 2.011E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 34430876.000 [ip-26-0-154-121:0]:06/21/2023 17:33:14 [INFO|DP=0|PP=0|TP=0]: iteration: 1342 / 150000 | consumed_samples: 85888 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784170E+05 | tokens_per_sec_per_gpu: 9.037765E+03 | global_batch_size: 64 | lm_loss: 1.095983E+01 | lr: 2.013E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 37069816.000 [ip-26-0-154-121:0]:06/21/2023 17:33:15 [INFO|DP=0|PP=0|TP=0]: iteration: 1343 / 150000 | consumed_samples: 85952 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785743E+05 | tokens_per_sec_per_gpu: 9.040224E+03 | global_batch_size: 64 | lm_loss: 1.095755E+01 | lr: 2.014E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 38375024.000 [ip-26-0-154-121:0]:06/21/2023 17:33:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1344 / 150000 | consumed_samples: 86016 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781931E+05 | tokens_per_sec_per_gpu: 9.034267E+03 | global_batch_size: 64 | lm_loss: 1.093507E+01 | lr: 2.016E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 41213972.000 [ip-26-0-154-121:0]:06/21/2023 17:33:17 [INFO|DP=0|PP=0|TP=0]: iteration: 1345 / 150000 | consumed_samples: 86080 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788523E+05 | tokens_per_sec_per_gpu: 9.044567E+03 | global_batch_size: 64 | lm_loss: 1.095539E+01 | lr: 2.018E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 58000772.000 [ip-26-0-154-121:0]:06/21/2023 17:33:18 [INFO|DP=0|PP=0|TP=0]: iteration: 1346 / 150000 | consumed_samples: 86144 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783543E+05 | tokens_per_sec_per_gpu: 9.036786E+03 | global_batch_size: 64 | lm_loss: 1.094230E+01 | lr: 2.019E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 53714272.000 [ip-26-0-154-121:0]:06/21/2023 17:33:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1347 / 150000 | consumed_samples: 86208 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779993E+05 | tokens_per_sec_per_gpu: 9.031239E+03 | global_batch_size: 64 | lm_loss: 1.096979E+01 | lr: 2.020E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 66165620.000 [ip-26-0-154-121:0]:06/21/2023 17:33:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1348 / 150000 | consumed_samples: 86272 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780355E+05 | tokens_per_sec_per_gpu: 9.031804E+03 | global_batch_size: 64 | lm_loss: 1.094439E+01 | lr: 2.022E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 65176760.000 [ip-26-0-154-121:0]:06/21/2023 17:33:20 [INFO|DP=0|PP=0|TP=0]: iteration: 1349 / 150000 | consumed_samples: 86336 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784256E+05 | tokens_per_sec_per_gpu: 9.037900E+03 | global_batch_size: 64 | lm_loss: 1.095473E+01 | lr: 2.023E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 56985492.000 [ip-26-0-154-121:0]:06/21/2023 17:33:21 [INFO|DP=0|PP=0|TP=0]: iteration: 1350 / 150000 | consumed_samples: 86400 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786349E+05 | tokens_per_sec_per_gpu: 9.041170E+03 | global_batch_size: 64 | lm_loss: 1.094516E+01 | lr: 2.025E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 45056028.000 [ip-26-0-154-121:0]:06/21/2023 17:33:22 [INFO|DP=0|PP=0|TP=0]: iteration: 1351 / 150000 | consumed_samples: 86464 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789137E+05 | tokens_per_sec_per_gpu: 9.045526E+03 | global_batch_size: 64 | lm_loss: 1.095863E+01 | lr: 2.026E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 44570456.000 [ip-26-0-154-121:0]:06/21/2023 17:33:23 [INFO|DP=0|PP=0|TP=0]: iteration: 1352 / 150000 | consumed_samples: 86528 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781758E+05 | tokens_per_sec_per_gpu: 9.033996E+03 | global_batch_size: 64 | lm_loss: 1.095910E+01 | lr: 2.028E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 40829472.000 [ip-26-0-154-121:0]:06/21/2023 17:33:24 [INFO|DP=0|PP=0|TP=0]: iteration: 1353 / 150000 | consumed_samples: 86592 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786355E+05 | tokens_per_sec_per_gpu: 9.041180E+03 | global_batch_size: 64 | lm_loss: 1.095133E+01 | lr: 2.029E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 45419196.000 [ip-26-0-154-121:0]:06/21/2023 17:33:25 [INFO|DP=0|PP=0|TP=0]: iteration: 1354 / 150000 | consumed_samples: 86656 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788005E+05 | tokens_per_sec_per_gpu: 9.043757E+03 | global_batch_size: 64 | lm_loss: 1.095097E+01 | lr: 2.031E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 40500956.000 [ip-26-0-154-121:0]:06/21/2023 17:33:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1355 / 150000 | consumed_samples: 86720 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787845E+05 | tokens_per_sec_per_gpu: 9.043507E+03 | global_batch_size: 64 | lm_loss: 1.095061E+01 | lr: 2.032E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 39931560.000 [ip-26-0-154-121:0]:06/21/2023 17:33:27 [INFO|DP=0|PP=0|TP=0]: iteration: 1356 / 150000 | consumed_samples: 86784 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786003E+05 | tokens_per_sec_per_gpu: 9.040630E+03 | global_batch_size: 64 | lm_loss: 1.095403E+01 | lr: 2.034E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 44178284.000 [ip-26-0-154-121:0]:06/21/2023 17:33:28 [INFO|DP=0|PP=0|TP=0]: iteration: 1357 / 150000 | consumed_samples: 86848 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783342E+05 | tokens_per_sec_per_gpu: 9.036472E+03 | global_batch_size: 64 | lm_loss: 1.095130E+01 | lr: 2.035E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 38624048.000 [ip-26-0-154-121:0]:06/21/2023 17:33:28 [INFO|DP=0|PP=0|TP=0]: iteration: 1358 / 150000 | consumed_samples: 86912 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786594E+05 | tokens_per_sec_per_gpu: 9.041553E+03 | global_batch_size: 64 | lm_loss: 1.094918E+01 | lr: 2.037E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 51892312.000 [ip-26-0-154-121:0]:06/21/2023 17:33:29 [INFO|DP=0|PP=0|TP=0]: iteration: 1359 / 150000 | consumed_samples: 86976 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787222E+05 | tokens_per_sec_per_gpu: 9.042534E+03 | global_batch_size: 64 | lm_loss: 1.095345E+01 | lr: 2.038E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 45267868.000 [ip-26-0-154-121:0]:06/21/2023 17:33:30 [INFO|DP=0|PP=0|TP=0]: iteration: 1360 / 150000 | consumed_samples: 87040 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784961E+05 | tokens_per_sec_per_gpu: 9.039001E+03 | global_batch_size: 64 | lm_loss: 1.094986E+01 | lr: 2.040E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 49956064.000 [ip-26-0-154-121:0]:06/21/2023 17:33:31 [INFO|DP=0|PP=0|TP=0]: iteration: 1361 / 150000 | consumed_samples: 87104 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787484E+05 | tokens_per_sec_per_gpu: 9.042943E+03 | global_batch_size: 64 | lm_loss: 1.094266E+01 | lr: 2.041E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 50942776.000 [ip-26-0-154-121:0]:06/21/2023 17:33:32 [INFO|DP=0|PP=0|TP=0]: iteration: 1362 / 150000 | consumed_samples: 87168 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786348E+05 | tokens_per_sec_per_gpu: 9.041168E+03 | global_batch_size: 64 | lm_loss: 1.095077E+01 | lr: 2.043E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 43894100.000 [ip-26-0-154-121:0]:06/21/2023 17:33:33 [INFO|DP=0|PP=0|TP=0]: iteration: 1363 / 150000 | consumed_samples: 87232 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789405E+05 | tokens_per_sec_per_gpu: 9.045945E+03 | global_batch_size: 64 | lm_loss: 1.093978E+01 | lr: 2.044E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 46127620.000 [ip-26-0-154-121:0]:06/21/2023 17:33:34 [INFO|DP=0|PP=0|TP=0]: iteration: 1364 / 150000 | consumed_samples: 87296 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785525E+05 | tokens_per_sec_per_gpu: 9.039883E+03 | global_batch_size: 64 | lm_loss: 1.094451E+01 | lr: 2.046E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 49411704.000 [ip-26-0-154-121:0]:06/21/2023 17:33:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1365 / 150000 | consumed_samples: 87360 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788760E+05 | tokens_per_sec_per_gpu: 9.044938E+03 | global_batch_size: 64 | lm_loss: 1.094929E+01 | lr: 2.047E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 39821440.000 [ip-26-0-154-121:0]:06/21/2023 17:33:36 [INFO|DP=0|PP=0|TP=0]: iteration: 1366 / 150000 | consumed_samples: 87424 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788504E+05 | tokens_per_sec_per_gpu: 9.044538E+03 | global_batch_size: 64 | lm_loss: 1.095027E+01 | lr: 2.049E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 59890752.000 [ip-26-0-154-121:0]:06/21/2023 17:33:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1367 / 150000 | consumed_samples: 87488 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788008E+05 | tokens_per_sec_per_gpu: 9.043762E+03 | global_batch_size: 64 | lm_loss: 1.094462E+01 | lr: 2.050E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 56468488.000 [ip-26-0-154-121:0]:06/21/2023 17:33:38 [INFO|DP=0|PP=0|TP=0]: iteration: 1368 / 150000 | consumed_samples: 87552 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781650E+05 | tokens_per_sec_per_gpu: 9.033828E+03 | global_batch_size: 64 | lm_loss: 1.094973E+01 | lr: 2.052E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 43204132.000 [ip-26-0-154-121:0]:06/21/2023 17:33:38 [INFO|DP=0|PP=0|TP=0]: iteration: 1369 / 150000 | consumed_samples: 87616 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784726E+05 | tokens_per_sec_per_gpu: 9.038635E+03 | global_batch_size: 64 | lm_loss: 1.095208E+01 | lr: 2.053E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 49284348.000 [ip-26-0-154-121:0]:06/21/2023 17:33:39 [INFO|DP=0|PP=0|TP=0]: iteration: 1370 / 150000 | consumed_samples: 87680 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787776E+05 | tokens_per_sec_per_gpu: 9.043400E+03 | global_batch_size: 64 | lm_loss: 1.094771E+01 | lr: 2.055E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 57641748.000 [ip-26-0-154-121:0]:06/21/2023 17:33:40 [INFO|DP=0|PP=0|TP=0]: iteration: 1371 / 150000 | consumed_samples: 87744 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790015E+05 | tokens_per_sec_per_gpu: 9.046898E+03 | global_batch_size: 64 | lm_loss: 1.095278E+01 | lr: 2.056E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 55370184.000 [ip-26-0-154-121:0]:06/21/2023 17:33:41 [INFO|DP=0|PP=0|TP=0]: iteration: 1372 / 150000 | consumed_samples: 87808 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790289E+05 | tokens_per_sec_per_gpu: 9.047327E+03 | global_batch_size: 64 | lm_loss: 1.094588E+01 | lr: 2.058E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 74417928.000 [ip-26-0-154-121:0]:06/21/2023 17:33:42 [INFO|DP=0|PP=0|TP=0]: iteration: 1373 / 150000 | consumed_samples: 87872 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786708E+05 | tokens_per_sec_per_gpu: 9.041732E+03 | global_batch_size: 64 | lm_loss: 1.094784E+01 | lr: 2.059E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 62560500.000 [ip-26-0-154-121:0]:06/21/2023 17:33:43 [INFO|DP=0|PP=0|TP=0]: iteration: 1374 / 150000 | consumed_samples: 87936 | elapsed_time_per_iteration_ms: 905.1 | tokens_per_sec: 5.792334E+05 | tokens_per_sec_per_gpu: 9.050522E+03 | global_batch_size: 64 | lm_loss: 1.094481E+01 | lr: 2.061E-04 | model_tflops_per_gpu: 104.54 | hardware_tflops_per_gpu: 104.54 | grad_norm: 76247920.000 [ip-26-0-154-121:0]:06/21/2023 17:33:44 [INFO|DP=0|PP=0|TP=0]: iteration: 1375 / 150000 | consumed_samples: 88000 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788584E+05 | tokens_per_sec_per_gpu: 9.044662E+03 | global_batch_size: 64 | lm_loss: 1.095063E+01 | lr: 2.063E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 80690792.000 [ip-26-0-154-121:0]:06/21/2023 17:33:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1376 / 150000 | consumed_samples: 88064 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788387E+05 | tokens_per_sec_per_gpu: 9.044355E+03 | global_batch_size: 64 | lm_loss: 1.094924E+01 | lr: 2.064E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 61941008.000 [ip-26-0-154-121:0]:06/21/2023 17:33:46 [INFO|DP=0|PP=0|TP=0]: iteration: 1377 / 150000 | consumed_samples: 88128 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783178E+05 | tokens_per_sec_per_gpu: 9.036215E+03 | global_batch_size: 64 | lm_loss: 1.094621E+01 | lr: 2.065E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 64594352.000 [ip-26-0-154-121:0]:06/21/2023 17:33:47 [INFO|DP=0|PP=0|TP=0]: iteration: 1378 / 150000 | consumed_samples: 88192 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785005E+05 | tokens_per_sec_per_gpu: 9.039070E+03 | global_batch_size: 64 | lm_loss: 1.095017E+01 | lr: 2.067E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 51203892.000 [ip-26-0-154-121:0]:06/21/2023 17:33:48 [INFO|DP=0|PP=0|TP=0]: iteration: 1379 / 150000 | consumed_samples: 88256 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787272E+05 | tokens_per_sec_per_gpu: 9.042612E+03 | global_batch_size: 64 | lm_loss: 1.094988E+01 | lr: 2.068E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 58306764.000 [ip-26-0-154-121:0]:06/21/2023 17:33:48 [INFO|DP=0|PP=0|TP=0]: iteration: 1380 / 150000 | consumed_samples: 88320 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786716E+05 | tokens_per_sec_per_gpu: 9.041744E+03 | global_batch_size: 64 | lm_loss: 1.093237E+01 | lr: 2.070E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 52397836.000 [ip-26-0-154-121:0]:06/21/2023 17:33:49 [INFO|DP=0|PP=0|TP=0]: iteration: 1381 / 150000 | consumed_samples: 88384 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789359E+05 | tokens_per_sec_per_gpu: 9.045874E+03 | global_batch_size: 64 | lm_loss: 1.094593E+01 | lr: 2.071E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 44197952.000 [ip-26-0-154-121:0]:06/21/2023 17:33:50 [INFO|DP=0|PP=0|TP=0]: iteration: 1382 / 150000 | consumed_samples: 88448 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783817E+05 | tokens_per_sec_per_gpu: 9.037213E+03 | global_batch_size: 64 | lm_loss: 1.094973E+01 | lr: 2.073E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 41869420.000 [ip-26-0-154-121:0]:06/21/2023 17:33:51 [INFO|DP=0|PP=0|TP=0]: iteration: 1383 / 150000 | consumed_samples: 88512 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787296E+05 | tokens_per_sec_per_gpu: 9.042650E+03 | global_batch_size: 64 | lm_loss: 1.095015E+01 | lr: 2.075E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 43016956.000 [ip-26-0-154-121:0]:06/21/2023 17:33:52 [INFO|DP=0|PP=0|TP=0]: iteration: 1384 / 150000 | consumed_samples: 88576 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787284E+05 | tokens_per_sec_per_gpu: 9.042631E+03 | global_batch_size: 64 | lm_loss: 1.095014E+01 | lr: 2.076E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 52299296.000 [ip-26-0-154-121:0]:06/21/2023 17:33:53 [INFO|DP=0|PP=0|TP=0]: iteration: 1385 / 150000 | consumed_samples: 88640 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788279E+05 | tokens_per_sec_per_gpu: 9.044186E+03 | global_batch_size: 64 | lm_loss: 1.093177E+01 | lr: 2.077E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 48520392.000 [ip-26-0-154-121:0]:06/21/2023 17:33:54 [INFO|DP=0|PP=0|TP=0]: iteration: 1386 / 150000 | consumed_samples: 88704 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788774E+05 | tokens_per_sec_per_gpu: 9.044959E+03 | global_batch_size: 64 | lm_loss: 1.095268E+01 | lr: 2.079E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 65137112.000 [ip-26-0-154-121:0]:06/21/2023 17:33:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1387 / 150000 | consumed_samples: 88768 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785541E+05 | tokens_per_sec_per_gpu: 9.039907E+03 | global_batch_size: 64 | lm_loss: 1.095092E+01 | lr: 2.080E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 51614692.000 [ip-26-0-154-121:0]:06/21/2023 17:33:56 [INFO|DP=0|PP=0|TP=0]: iteration: 1388 / 150000 | consumed_samples: 88832 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777542E+05 | tokens_per_sec_per_gpu: 9.027410E+03 | global_batch_size: 64 | lm_loss: 1.094897E+01 | lr: 2.082E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 38713484.000 [ip-26-0-154-121:0]:06/21/2023 17:33:57 [INFO|DP=0|PP=0|TP=0]: iteration: 1389 / 150000 | consumed_samples: 88896 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787147E+05 | tokens_per_sec_per_gpu: 9.042417E+03 | global_batch_size: 64 | lm_loss: 1.093074E+01 | lr: 2.083E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 46320780.000 [ip-26-0-154-121:0]:06/21/2023 17:33:58 [INFO|DP=0|PP=0|TP=0]: iteration: 1390 / 150000 | consumed_samples: 88960 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783374E+05 | tokens_per_sec_per_gpu: 9.036522E+03 | global_batch_size: 64 | lm_loss: 1.094404E+01 | lr: 2.085E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 49965784.000 [ip-26-0-154-121:0]:06/21/2023 17:33:58 [INFO|DP=0|PP=0|TP=0]: iteration: 1391 / 150000 | consumed_samples: 89024 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787656E+05 | tokens_per_sec_per_gpu: 9.043212E+03 | global_batch_size: 64 | lm_loss: 1.094360E+01 | lr: 2.086E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 61522640.000 [ip-26-0-154-121:0]:06/21/2023 17:33:59 [INFO|DP=0|PP=0|TP=0]: iteration: 1392 / 150000 | consumed_samples: 89088 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783378E+05 | tokens_per_sec_per_gpu: 9.036529E+03 | global_batch_size: 64 | lm_loss: 1.093287E+01 | lr: 2.088E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 59380284.000 [ip-26-0-154-121:0]:06/21/2023 17:34:00 [INFO|DP=0|PP=0|TP=0]: iteration: 1393 / 150000 | consumed_samples: 89152 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790141E+05 | tokens_per_sec_per_gpu: 9.047096E+03 | global_batch_size: 64 | lm_loss: 1.094137E+01 | lr: 2.089E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 44992008.000 [ip-26-0-154-121:0]:06/21/2023 17:34:01 [INFO|DP=0|PP=0|TP=0]: iteration: 1394 / 150000 | consumed_samples: 89216 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779203E+05 | tokens_per_sec_per_gpu: 9.030005E+03 | global_batch_size: 64 | lm_loss: 1.094055E+01 | lr: 2.091E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 42865288.000 [ip-26-0-154-121:0]:06/21/2023 17:34:02 [INFO|DP=0|PP=0|TP=0]: iteration: 1395 / 150000 | consumed_samples: 89280 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783961E+05 | tokens_per_sec_per_gpu: 9.037439E+03 | global_batch_size: 64 | lm_loss: 1.093852E+01 | lr: 2.092E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 45978408.000 [ip-26-0-154-121:0]:06/21/2023 17:34:03 [INFO|DP=0|PP=0|TP=0]: iteration: 1396 / 150000 | consumed_samples: 89344 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790056E+05 | tokens_per_sec_per_gpu: 9.046962E+03 | global_batch_size: 64 | lm_loss: 1.093651E+01 | lr: 2.094E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 35771456.000 [ip-26-0-154-121:0]:06/21/2023 17:34:04 [INFO|DP=0|PP=0|TP=0]: iteration: 1397 / 150000 | consumed_samples: 89408 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781718E+05 | tokens_per_sec_per_gpu: 9.033934E+03 | global_batch_size: 64 | lm_loss: 1.093051E+01 | lr: 2.095E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 38188568.000 [ip-26-0-154-121:0]:06/21/2023 17:34:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1398 / 150000 | consumed_samples: 89472 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789013E+05 | tokens_per_sec_per_gpu: 9.045333E+03 | global_batch_size: 64 | lm_loss: 1.094008E+01 | lr: 2.097E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 47024344.000 [ip-26-0-154-121:0]:06/21/2023 17:34:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1399 / 150000 | consumed_samples: 89536 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788251E+05 | tokens_per_sec_per_gpu: 9.044143E+03 | global_batch_size: 64 | lm_loss: 1.094576E+01 | lr: 2.098E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 58557064.000 [ip-26-0-154-121:0]:06/21/2023 17:34:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1400 / 150000 | consumed_samples: 89600 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787037E+05 | tokens_per_sec_per_gpu: 9.042246E+03 | global_batch_size: 64 | lm_loss: 1.093192E+01 | lr: 2.100E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 45473940.000 [ip-26-0-154-121:0]:06/21/2023 17:34:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1401 / 150000 | consumed_samples: 89664 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787711E+05 | tokens_per_sec_per_gpu: 9.043298E+03 | global_batch_size: 64 | lm_loss: 1.093735E+01 | lr: 2.101E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 53219900.000 [ip-26-0-154-121:0]:06/21/2023 17:34:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1402 / 150000 | consumed_samples: 89728 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786657E+05 | tokens_per_sec_per_gpu: 9.041651E+03 | global_batch_size: 64 | lm_loss: 1.094044E+01 | lr: 2.103E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 49239496.000 [ip-26-0-154-121:0]:06/21/2023 17:34:09 [INFO|DP=0|PP=0|TP=0]: iteration: 1403 / 150000 | consumed_samples: 89792 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781458E+05 | tokens_per_sec_per_gpu: 9.033528E+03 | global_batch_size: 64 | lm_loss: 1.094136E+01 | lr: 2.104E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 54394900.000 [ip-26-0-154-121:0]:06/21/2023 17:34:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1404 / 150000 | consumed_samples: 89856 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782618E+05 | tokens_per_sec_per_gpu: 9.035341E+03 | global_batch_size: 64 | lm_loss: 1.093902E+01 | lr: 2.106E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 50751244.000 [ip-26-0-154-121:0]:06/21/2023 17:34:11 [INFO|DP=0|PP=0|TP=0]: iteration: 1405 / 150000 | consumed_samples: 89920 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785203E+05 | tokens_per_sec_per_gpu: 9.039379E+03 | global_batch_size: 64 | lm_loss: 1.094081E+01 | lr: 2.107E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 57135044.000 [ip-26-0-154-121:0]:06/21/2023 17:34:12 [INFO|DP=0|PP=0|TP=0]: iteration: 1406 / 150000 | consumed_samples: 89984 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785341E+05 | tokens_per_sec_per_gpu: 9.039596E+03 | global_batch_size: 64 | lm_loss: 1.094305E+01 | lr: 2.109E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 51953360.000 [ip-26-0-154-121:0]:06/21/2023 17:34:13 [INFO|DP=0|PP=0|TP=0]: iteration: 1407 / 150000 | consumed_samples: 90048 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781139E+05 | tokens_per_sec_per_gpu: 9.033030E+03 | global_batch_size: 64 | lm_loss: 1.094037E+01 | lr: 2.110E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 64858436.000 [ip-26-0-154-121:0]:06/21/2023 17:34:14 [INFO|DP=0|PP=0|TP=0]: iteration: 1408 / 150000 | consumed_samples: 90112 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782490E+05 | tokens_per_sec_per_gpu: 9.035141E+03 | global_batch_size: 64 | lm_loss: 1.094349E+01 | lr: 2.112E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 64619000.000 [ip-26-0-154-121:0]:06/21/2023 17:34:15 [INFO|DP=0|PP=0|TP=0]: iteration: 1409 / 150000 | consumed_samples: 90176 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781455E+05 | tokens_per_sec_per_gpu: 9.033524E+03 | global_batch_size: 64 | lm_loss: 1.094388E+01 | lr: 2.113E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 60389092.000 [ip-26-0-154-121:0]:06/21/2023 17:34:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1410 / 150000 | consumed_samples: 90240 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781938E+05 | tokens_per_sec_per_gpu: 9.034279E+03 | global_batch_size: 64 | lm_loss: 1.093957E+01 | lr: 2.115E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 63300764.000 [ip-26-0-154-121:0]:06/21/2023 17:34:17 [INFO|DP=0|PP=0|TP=0]: iteration: 1411 / 150000 | consumed_samples: 90304 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783873E+05 | tokens_per_sec_per_gpu: 9.037301E+03 | global_batch_size: 64 | lm_loss: 1.094868E+01 | lr: 2.116E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 72580456.000 [ip-26-0-154-121:0]:06/21/2023 17:34:17 [INFO|DP=0|PP=0|TP=0]: iteration: 1412 / 150000 | consumed_samples: 90368 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784168E+05 | tokens_per_sec_per_gpu: 9.037763E+03 | global_batch_size: 64 | lm_loss: 1.093980E+01 | lr: 2.118E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 54607684.000 [ip-26-0-154-121:0]:06/21/2023 17:34:18 [INFO|DP=0|PP=0|TP=0]: iteration: 1413 / 150000 | consumed_samples: 90432 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778427E+05 | tokens_per_sec_per_gpu: 9.028793E+03 | global_batch_size: 64 | lm_loss: 1.094427E+01 | lr: 2.119E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 77287120.000 [ip-26-0-154-121:0]:06/21/2023 17:34:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1414 / 150000 | consumed_samples: 90496 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781981E+05 | tokens_per_sec_per_gpu: 9.034345E+03 | global_batch_size: 64 | lm_loss: 1.094217E+01 | lr: 2.121E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 73019496.000 [ip-26-0-154-121:0]:06/21/2023 17:34:20 [INFO|DP=0|PP=0|TP=0]: iteration: 1415 / 150000 | consumed_samples: 90560 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781200E+05 | tokens_per_sec_per_gpu: 9.033125E+03 | global_batch_size: 64 | lm_loss: 1.094141E+01 | lr: 2.122E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 61475268.000 [ip-26-0-154-121:0]:06/21/2023 17:34:21 [INFO|DP=0|PP=0|TP=0]: iteration: 1416 / 150000 | consumed_samples: 90624 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779636E+05 | tokens_per_sec_per_gpu: 9.030682E+03 | global_batch_size: 64 | lm_loss: 1.093498E+01 | lr: 2.124E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 75242992.000 [ip-26-0-154-121:0]:06/21/2023 17:34:22 [INFO|DP=0|PP=0|TP=0]: iteration: 1417 / 150000 | consumed_samples: 90688 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785016E+05 | tokens_per_sec_per_gpu: 9.039087E+03 | global_batch_size: 64 | lm_loss: 1.094015E+01 | lr: 2.125E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 66301956.000 [ip-26-0-154-121:0]:06/21/2023 17:34:23 [INFO|DP=0|PP=0|TP=0]: iteration: 1418 / 150000 | consumed_samples: 90752 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.782042E+05 | tokens_per_sec_per_gpu: 9.034440E+03 | global_batch_size: 64 | lm_loss: 1.093337E+01 | lr: 2.127E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 38907420.000 [ip-26-0-154-121:0]:06/21/2023 17:34:24 [INFO|DP=0|PP=0|TP=0]: iteration: 1419 / 150000 | consumed_samples: 90816 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783342E+05 | tokens_per_sec_per_gpu: 9.036472E+03 | global_batch_size: 64 | lm_loss: 1.093858E+01 | lr: 2.128E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 41654380.000 [ip-26-0-154-121:0]:06/21/2023 17:34:25 [INFO|DP=0|PP=0|TP=0]: iteration: 1420 / 150000 | consumed_samples: 90880 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785815E+05 | tokens_per_sec_per_gpu: 9.040335E+03 | global_batch_size: 64 | lm_loss: 1.093736E+01 | lr: 2.130E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 48958184.000 [ip-26-0-154-121:0]:06/21/2023 17:34:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1421 / 150000 | consumed_samples: 90944 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787631E+05 | tokens_per_sec_per_gpu: 9.043174E+03 | global_batch_size: 64 | lm_loss: 1.093350E+01 | lr: 2.131E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 42922360.000 [ip-26-0-154-121:0]:06/21/2023 17:34:27 [INFO|DP=0|PP=0|TP=0]: iteration: 1422 / 150000 | consumed_samples: 91008 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780435E+05 | tokens_per_sec_per_gpu: 9.031930E+03 | global_batch_size: 64 | lm_loss: 1.093952E+01 | lr: 2.133E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 62412828.000 [ip-26-0-154-121:0]:06/21/2023 17:34:27 [INFO|DP=0|PP=0|TP=0]: iteration: 1423 / 150000 | consumed_samples: 91072 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785093E+05 | tokens_per_sec_per_gpu: 9.039208E+03 | global_batch_size: 64 | lm_loss: 1.094184E+01 | lr: 2.134E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 44703608.000 [ip-26-0-154-121:0]:06/21/2023 17:34:28 [INFO|DP=0|PP=0|TP=0]: iteration: 1424 / 150000 | consumed_samples: 91136 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784290E+05 | tokens_per_sec_per_gpu: 9.037953E+03 | global_batch_size: 64 | lm_loss: 1.093727E+01 | lr: 2.136E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 68486680.000 [ip-26-0-154-121:0]:06/21/2023 17:34:29 [INFO|DP=0|PP=0|TP=0]: iteration: 1425 / 150000 | consumed_samples: 91200 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784201E+05 | tokens_per_sec_per_gpu: 9.037815E+03 | global_batch_size: 64 | lm_loss: 1.094267E+01 | lr: 2.137E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 55541444.000 [ip-26-0-154-121:0]:06/21/2023 17:34:30 [INFO|DP=0|PP=0|TP=0]: iteration: 1426 / 150000 | consumed_samples: 91264 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780136E+05 | tokens_per_sec_per_gpu: 9.031462E+03 | global_batch_size: 64 | lm_loss: 1.094185E+01 | lr: 2.139E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 58398588.000 [ip-26-0-154-121:0]:06/21/2023 17:34:31 [INFO|DP=0|PP=0|TP=0]: iteration: 1427 / 150000 | consumed_samples: 91328 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.783999E+05 | tokens_per_sec_per_gpu: 9.037499E+03 | global_batch_size: 64 | lm_loss: 1.093644E+01 | lr: 2.140E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 36362224.000 [ip-26-0-154-121:0]:06/21/2023 17:34:32 [INFO|DP=0|PP=0|TP=0]: iteration: 1428 / 150000 | consumed_samples: 91392 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784422E+05 | tokens_per_sec_per_gpu: 9.038160E+03 | global_batch_size: 64 | lm_loss: 1.093454E+01 | lr: 2.142E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 43068084.000 [ip-26-0-154-121:0]:06/21/2023 17:34:33 [INFO|DP=0|PP=0|TP=0]: iteration: 1429 / 150000 | consumed_samples: 91456 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782665E+05 | tokens_per_sec_per_gpu: 9.035414E+03 | global_batch_size: 64 | lm_loss: 1.092668E+01 | lr: 2.143E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 44945300.000 [ip-26-0-154-121:0]:06/21/2023 17:34:34 [INFO|DP=0|PP=0|TP=0]: iteration: 1430 / 150000 | consumed_samples: 91520 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783578E+05 | tokens_per_sec_per_gpu: 9.036840E+03 | global_batch_size: 64 | lm_loss: 1.092280E+01 | lr: 2.145E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 42736732.000 [ip-26-0-154-121:0]:06/21/2023 17:34:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1431 / 150000 | consumed_samples: 91584 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778535E+05 | tokens_per_sec_per_gpu: 9.028961E+03 | global_batch_size: 64 | lm_loss: 1.093309E+01 | lr: 2.146E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 55388876.000 [ip-26-0-154-121:0]:06/21/2023 17:34:36 [INFO|DP=0|PP=0|TP=0]: iteration: 1432 / 150000 | consumed_samples: 91648 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788134E+05 | tokens_per_sec_per_gpu: 9.043959E+03 | global_batch_size: 64 | lm_loss: 1.093449E+01 | lr: 2.148E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 46945656.000 [ip-26-0-154-121:0]:06/21/2023 17:34:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1433 / 150000 | consumed_samples: 91712 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783667E+05 | tokens_per_sec_per_gpu: 9.036980E+03 | global_batch_size: 64 | lm_loss: 1.093256E+01 | lr: 2.149E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 62137456.000 [ip-26-0-154-121:0]:06/21/2023 17:34:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1434 / 150000 | consumed_samples: 91776 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786410E+05 | tokens_per_sec_per_gpu: 9.041266E+03 | global_batch_size: 64 | lm_loss: 1.092861E+01 | lr: 2.151E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 53826304.000 [ip-26-0-154-121:0]:06/21/2023 17:34:38 [INFO|DP=0|PP=0|TP=0]: iteration: 1435 / 150000 | consumed_samples: 91840 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785367E+05 | tokens_per_sec_per_gpu: 9.039636E+03 | global_batch_size: 64 | lm_loss: 1.093278E+01 | lr: 2.153E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 54433260.000 [ip-26-0-154-121:0]:06/21/2023 17:34:39 [INFO|DP=0|PP=0|TP=0]: iteration: 1436 / 150000 | consumed_samples: 91904 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779856E+05 | tokens_per_sec_per_gpu: 9.031026E+03 | global_batch_size: 64 | lm_loss: 1.093565E+01 | lr: 2.154E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 54560300.000 [ip-26-0-154-121:0]:06/21/2023 17:34:40 [INFO|DP=0|PP=0|TP=0]: iteration: 1437 / 150000 | consumed_samples: 91968 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777149E+05 | tokens_per_sec_per_gpu: 9.026795E+03 | global_batch_size: 64 | lm_loss: 1.093309E+01 | lr: 2.155E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 51123340.000 [ip-26-0-154-121:0]:06/21/2023 17:34:41 [INFO|DP=0|PP=0|TP=0]: iteration: 1438 / 150000 | consumed_samples: 92032 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780803E+05 | tokens_per_sec_per_gpu: 9.032505E+03 | global_batch_size: 64 | lm_loss: 1.093449E+01 | lr: 2.157E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 57615396.000 [ip-26-0-154-121:0]:06/21/2023 17:34:42 [INFO|DP=0|PP=0|TP=0]: iteration: 1439 / 150000 | consumed_samples: 92096 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781171E+05 | tokens_per_sec_per_gpu: 9.033079E+03 | global_batch_size: 64 | lm_loss: 1.092635E+01 | lr: 2.158E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 59012004.000 [ip-26-0-154-121:0]:06/21/2023 17:34:43 [INFO|DP=0|PP=0|TP=0]: iteration: 1440 / 150000 | consumed_samples: 92160 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780627E+05 | tokens_per_sec_per_gpu: 9.032229E+03 | global_batch_size: 64 | lm_loss: 1.092999E+01 | lr: 2.160E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 66815940.000 [ip-26-0-154-121:0]:06/21/2023 17:34:44 [INFO|DP=0|PP=0|TP=0]: iteration: 1441 / 150000 | consumed_samples: 92224 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781879E+05 | tokens_per_sec_per_gpu: 9.034186E+03 | global_batch_size: 64 | lm_loss: 1.093561E+01 | lr: 2.161E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 63109732.000 [ip-26-0-154-121:0]:06/21/2023 17:34:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1442 / 150000 | consumed_samples: 92288 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779855E+05 | tokens_per_sec_per_gpu: 9.031023E+03 | global_batch_size: 64 | lm_loss: 1.093187E+01 | lr: 2.163E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 42230860.000 [ip-26-0-154-121:0]:06/21/2023 17:34:46 [INFO|DP=0|PP=0|TP=0]: iteration: 1443 / 150000 | consumed_samples: 92352 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779588E+05 | tokens_per_sec_per_gpu: 9.030606E+03 | global_batch_size: 64 | lm_loss: 1.093091E+01 | lr: 2.165E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 43705164.000 [ip-26-0-154-121:0]:06/21/2023 17:34:47 [INFO|DP=0|PP=0|TP=0]: iteration: 1444 / 150000 | consumed_samples: 92416 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785697E+05 | tokens_per_sec_per_gpu: 9.040152E+03 | global_batch_size: 64 | lm_loss: 1.093089E+01 | lr: 2.166E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 54440288.000 [ip-26-0-154-121:0]:06/21/2023 17:34:47 [INFO|DP=0|PP=0|TP=0]: iteration: 1445 / 150000 | consumed_samples: 92480 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781036E+05 | tokens_per_sec_per_gpu: 9.032868E+03 | global_batch_size: 64 | lm_loss: 1.093118E+01 | lr: 2.167E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 73281416.000 [ip-26-0-154-121:0]:06/21/2023 17:34:48 [INFO|DP=0|PP=0|TP=0]: iteration: 1446 / 150000 | consumed_samples: 92544 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778616E+05 | tokens_per_sec_per_gpu: 9.029087E+03 | global_batch_size: 64 | lm_loss: 1.093161E+01 | lr: 2.169E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 52773852.000 [ip-26-0-154-121:0]:06/21/2023 17:34:49 [INFO|DP=0|PP=0|TP=0]: iteration: 1447 / 150000 | consumed_samples: 92608 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786309E+05 | tokens_per_sec_per_gpu: 9.041109E+03 | global_batch_size: 64 | lm_loss: 1.092613E+01 | lr: 2.170E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 59675360.000 [ip-26-0-154-121:0]:06/21/2023 17:34:50 [INFO|DP=0|PP=0|TP=0]: iteration: 1448 / 150000 | consumed_samples: 92672 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783713E+05 | tokens_per_sec_per_gpu: 9.037052E+03 | global_batch_size: 64 | lm_loss: 1.092740E+01 | lr: 2.172E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 47661368.000 [ip-26-0-154-121:0]:06/21/2023 17:34:51 [INFO|DP=0|PP=0|TP=0]: iteration: 1449 / 150000 | consumed_samples: 92736 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781957E+05 | tokens_per_sec_per_gpu: 9.034307E+03 | global_batch_size: 64 | lm_loss: 1.089302E+01 | lr: 2.173E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 58187144.000 [ip-26-0-154-121:0]:06/21/2023 17:34:52 [INFO|DP=0|PP=0|TP=0]: iteration: 1450 / 150000 | consumed_samples: 92800 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780385E+05 | tokens_per_sec_per_gpu: 9.031852E+03 | global_batch_size: 64 | lm_loss: 1.092923E+01 | lr: 2.175E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 59246020.000 [ip-26-0-154-121:0]:06/21/2023 17:34:53 [INFO|DP=0|PP=0|TP=0]: iteration: 1451 / 150000 | consumed_samples: 92864 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787976E+05 | tokens_per_sec_per_gpu: 9.043712E+03 | global_batch_size: 64 | lm_loss: 1.092465E+01 | lr: 2.176E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 36989412.000 [ip-26-0-154-121:0]:06/21/2023 17:34:54 [INFO|DP=0|PP=0|TP=0]: iteration: 1452 / 150000 | consumed_samples: 92928 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781163E+05 | tokens_per_sec_per_gpu: 9.033068E+03 | global_batch_size: 64 | lm_loss: 1.088274E+01 | lr: 2.178E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 55889212.000 [ip-26-0-154-121:0]:06/21/2023 17:34:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1453 / 150000 | consumed_samples: 92992 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784407E+05 | tokens_per_sec_per_gpu: 9.038136E+03 | global_batch_size: 64 | lm_loss: 1.092639E+01 | lr: 2.179E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 53234412.000 [ip-26-0-154-121:0]:06/21/2023 17:34:56 [INFO|DP=0|PP=0|TP=0]: iteration: 1454 / 150000 | consumed_samples: 93056 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785883E+05 | tokens_per_sec_per_gpu: 9.040442E+03 | global_batch_size: 64 | lm_loss: 1.092930E+01 | lr: 2.181E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 37701656.000 [ip-26-0-154-121:0]:06/21/2023 17:34:57 [INFO|DP=0|PP=0|TP=0]: iteration: 1455 / 150000 | consumed_samples: 93120 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785016E+05 | tokens_per_sec_per_gpu: 9.039087E+03 | global_batch_size: 64 | lm_loss: 1.092190E+01 | lr: 2.182E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 53605840.000 [ip-26-0-154-121:0]:06/21/2023 17:34:57 [INFO|DP=0|PP=0|TP=0]: iteration: 1456 / 150000 | consumed_samples: 93184 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784766E+05 | tokens_per_sec_per_gpu: 9.038697E+03 | global_batch_size: 64 | lm_loss: 1.092857E+01 | lr: 2.184E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 49641372.000 [ip-26-0-154-121:0]:06/21/2023 17:34:58 [INFO|DP=0|PP=0|TP=0]: iteration: 1457 / 150000 | consumed_samples: 93248 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779963E+05 | tokens_per_sec_per_gpu: 9.031192E+03 | global_batch_size: 64 | lm_loss: 1.093134E+01 | lr: 2.185E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 31489358.000 [ip-26-0-154-121:0]:06/21/2023 17:34:59 [INFO|DP=0|PP=0|TP=0]: iteration: 1458 / 150000 | consumed_samples: 93312 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784296E+05 | tokens_per_sec_per_gpu: 9.037962E+03 | global_batch_size: 64 | lm_loss: 1.092432E+01 | lr: 2.187E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 41543440.000 [ip-26-0-154-121:0]:06/21/2023 17:35:00 [INFO|DP=0|PP=0|TP=0]: iteration: 1459 / 150000 | consumed_samples: 93376 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782541E+05 | tokens_per_sec_per_gpu: 9.035220E+03 | global_batch_size: 64 | lm_loss: 1.091434E+01 | lr: 2.188E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 35735404.000 [ip-26-0-154-121:0]:06/21/2023 17:35:01 [INFO|DP=0|PP=0|TP=0]: iteration: 1460 / 150000 | consumed_samples: 93440 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786580E+05 | tokens_per_sec_per_gpu: 9.041532E+03 | global_batch_size: 64 | lm_loss: 1.092972E+01 | lr: 2.190E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 37668268.000 [ip-26-0-154-121:0]:06/21/2023 17:35:02 [INFO|DP=0|PP=0|TP=0]: iteration: 1461 / 150000 | consumed_samples: 93504 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782893E+05 | tokens_per_sec_per_gpu: 9.035771E+03 | global_batch_size: 64 | lm_loss: 1.093108E+01 | lr: 2.191E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 38684104.000 [ip-26-0-154-121:0]:06/21/2023 17:35:03 [INFO|DP=0|PP=0|TP=0]: iteration: 1462 / 150000 | consumed_samples: 93568 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780043E+05 | tokens_per_sec_per_gpu: 9.031318E+03 | global_batch_size: 64 | lm_loss: 1.091394E+01 | lr: 2.193E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 45037272.000 [ip-26-0-154-121:0]:06/21/2023 17:35:04 [INFO|DP=0|PP=0|TP=0]: iteration: 1463 / 150000 | consumed_samples: 93632 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786265E+05 | tokens_per_sec_per_gpu: 9.041040E+03 | global_batch_size: 64 | lm_loss: 1.092925E+01 | lr: 2.194E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 45169920.000 [ip-26-0-154-121:0]:06/21/2023 17:35:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1464 / 150000 | consumed_samples: 93696 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784987E+05 | tokens_per_sec_per_gpu: 9.039042E+03 | global_batch_size: 64 | lm_loss: 1.092441E+01 | lr: 2.196E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 44562780.000 [ip-26-0-154-121:0]:06/21/2023 17:35:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1465 / 150000 | consumed_samples: 93760 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786026E+05 | tokens_per_sec_per_gpu: 9.040666E+03 | global_batch_size: 64 | lm_loss: 1.092182E+01 | lr: 2.197E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 34776640.000 [ip-26-0-154-121:0]:06/21/2023 17:35:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1466 / 150000 | consumed_samples: 93824 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783654E+05 | tokens_per_sec_per_gpu: 9.036959E+03 | global_batch_size: 64 | lm_loss: 1.092706E+01 | lr: 2.199E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 39109744.000 [ip-26-0-154-121:0]:06/21/2023 17:35:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1467 / 150000 | consumed_samples: 93888 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781621E+05 | tokens_per_sec_per_gpu: 9.033782E+03 | global_batch_size: 64 | lm_loss: 1.092464E+01 | lr: 2.200E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 43791864.000 [ip-26-0-154-121:0]:06/21/2023 17:35:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1468 / 150000 | consumed_samples: 93952 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781090E+05 | tokens_per_sec_per_gpu: 9.032954E+03 | global_batch_size: 64 | lm_loss: 1.093269E+01 | lr: 2.202E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 38310520.000 [ip-26-0-154-121:0]:06/21/2023 17:35:09 [INFO|DP=0|PP=0|TP=0]: iteration: 1469 / 150000 | consumed_samples: 94016 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779378E+05 | tokens_per_sec_per_gpu: 9.030278E+03 | global_batch_size: 64 | lm_loss: 1.092410E+01 | lr: 2.203E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 69564704.000 [ip-26-0-154-121:0]:06/21/2023 17:35:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1470 / 150000 | consumed_samples: 94080 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785273E+05 | tokens_per_sec_per_gpu: 9.039489E+03 | global_batch_size: 64 | lm_loss: 1.092668E+01 | lr: 2.205E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 47961108.000 [ip-26-0-154-121:0]:06/21/2023 17:35:11 [INFO|DP=0|PP=0|TP=0]: iteration: 1471 / 150000 | consumed_samples: 94144 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780674E+05 | tokens_per_sec_per_gpu: 9.032303E+03 | global_batch_size: 64 | lm_loss: 1.092832E+01 | lr: 2.206E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 41353608.000 [ip-26-0-154-121:0]:06/21/2023 17:35:12 [INFO|DP=0|PP=0|TP=0]: iteration: 1472 / 150000 | consumed_samples: 94208 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779360E+05 | tokens_per_sec_per_gpu: 9.030250E+03 | global_batch_size: 64 | lm_loss: 1.092905E+01 | lr: 2.208E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 53015968.000 [ip-26-0-154-121:0]:06/21/2023 17:35:13 [INFO|DP=0|PP=0|TP=0]: iteration: 1473 / 150000 | consumed_samples: 94272 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783616E+05 | tokens_per_sec_per_gpu: 9.036900E+03 | global_batch_size: 64 | lm_loss: 1.093416E+01 | lr: 2.209E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 40459264.000 [ip-26-0-154-121:0]:06/21/2023 17:35:14 [INFO|DP=0|PP=0|TP=0]: iteration: 1474 / 150000 | consumed_samples: 94336 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784728E+05 | tokens_per_sec_per_gpu: 9.038637E+03 | global_batch_size: 64 | lm_loss: 1.092796E+01 | lr: 2.211E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 43450740.000 [ip-26-0-154-121:0]:06/21/2023 17:35:15 [INFO|DP=0|PP=0|TP=0]: iteration: 1475 / 150000 | consumed_samples: 94400 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782343E+05 | tokens_per_sec_per_gpu: 9.034911E+03 | global_batch_size: 64 | lm_loss: 1.092518E+01 | lr: 2.212E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 43742972.000 [ip-26-0-154-121:0]:06/21/2023 17:35:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1476 / 150000 | consumed_samples: 94464 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786215E+05 | tokens_per_sec_per_gpu: 9.040961E+03 | global_batch_size: 64 | lm_loss: 1.092747E+01 | lr: 2.214E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 30581124.000 [ip-26-0-154-121:0]:06/21/2023 17:35:17 [INFO|DP=0|PP=0|TP=0]: iteration: 1477 / 150000 | consumed_samples: 94528 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780253E+05 | tokens_per_sec_per_gpu: 9.031645E+03 | global_batch_size: 64 | lm_loss: 1.092744E+01 | lr: 2.215E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 30705968.000 [ip-26-0-154-121:0]:06/21/2023 17:35:17 [INFO|DP=0|PP=0|TP=0]: iteration: 1478 / 150000 | consumed_samples: 94592 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781143E+05 | tokens_per_sec_per_gpu: 9.033037E+03 | global_batch_size: 64 | lm_loss: 1.092551E+01 | lr: 2.217E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 36155700.000 [ip-26-0-154-121:0]:06/21/2023 17:35:18 [INFO|DP=0|PP=0|TP=0]: iteration: 1479 / 150000 | consumed_samples: 94656 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785451E+05 | tokens_per_sec_per_gpu: 9.039767E+03 | global_batch_size: 64 | lm_loss: 1.092496E+01 | lr: 2.218E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 21900090.000 [ip-26-0-154-121:0]:06/21/2023 17:35:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1480 / 150000 | consumed_samples: 94720 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783713E+05 | tokens_per_sec_per_gpu: 9.037052E+03 | global_batch_size: 64 | lm_loss: 1.092166E+01 | lr: 2.220E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 15003390.000 [ip-26-0-154-121:0]:06/21/2023 17:35:20 [INFO|DP=0|PP=0|TP=0]: iteration: 1481 / 150000 | consumed_samples: 94784 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787107E+05 | tokens_per_sec_per_gpu: 9.042355E+03 | global_batch_size: 64 | lm_loss: 1.089580E+01 | lr: 2.222E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 12248091.000 [ip-26-0-154-121:0]:06/21/2023 17:35:21 [INFO|DP=0|PP=0|TP=0]: iteration: 1482 / 150000 | consumed_samples: 94848 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785539E+05 | tokens_per_sec_per_gpu: 9.039905E+03 | global_batch_size: 64 | lm_loss: 1.091908E+01 | lr: 2.223E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 12748831.000 [ip-26-0-154-121:0]:06/21/2023 17:35:22 [INFO|DP=0|PP=0|TP=0]: iteration: 1483 / 150000 | consumed_samples: 94912 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783409E+05 | tokens_per_sec_per_gpu: 9.036576E+03 | global_batch_size: 64 | lm_loss: 1.092474E+01 | lr: 2.224E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 14932078.000 [ip-26-0-154-121:0]:06/21/2023 17:35:23 [INFO|DP=0|PP=0|TP=0]: iteration: 1484 / 150000 | consumed_samples: 94976 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788279E+05 | tokens_per_sec_per_gpu: 9.044186E+03 | global_batch_size: 64 | lm_loss: 1.092373E+01 | lr: 2.226E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 18478590.000 [ip-26-0-154-121:0]:06/21/2023 17:35:24 [INFO|DP=0|PP=0|TP=0]: iteration: 1485 / 150000 | consumed_samples: 95040 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787698E+05 | tokens_per_sec_per_gpu: 9.043279E+03 | global_batch_size: 64 | lm_loss: 1.092089E+01 | lr: 2.227E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 21779914.000 [ip-26-0-154-121:0]:06/21/2023 17:35:25 [INFO|DP=0|PP=0|TP=0]: iteration: 1486 / 150000 | consumed_samples: 95104 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785317E+05 | tokens_per_sec_per_gpu: 9.039558E+03 | global_batch_size: 64 | lm_loss: 1.092200E+01 | lr: 2.229E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 21070148.000 [ip-26-0-154-121:0]:06/21/2023 17:35:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1487 / 150000 | consumed_samples: 95168 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786571E+05 | tokens_per_sec_per_gpu: 9.041518E+03 | global_batch_size: 64 | lm_loss: 1.092483E+01 | lr: 2.230E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 25801040.000 [ip-26-0-154-121:0]:06/21/2023 17:35:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1488 / 150000 | consumed_samples: 95232 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782547E+05 | tokens_per_sec_per_gpu: 9.035229E+03 | global_batch_size: 64 | lm_loss: 1.092440E+01 | lr: 2.232E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 29662766.000 [ip-26-0-154-121:0]:06/21/2023 17:35:27 [INFO|DP=0|PP=0|TP=0]: iteration: 1489 / 150000 | consumed_samples: 95296 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790930E+05 | tokens_per_sec_per_gpu: 9.048327E+03 | global_batch_size: 64 | lm_loss: 1.092065E+01 | lr: 2.234E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 35460656.000 [ip-26-0-154-121:0]:06/21/2023 17:35:28 [INFO|DP=0|PP=0|TP=0]: iteration: 1490 / 150000 | consumed_samples: 95360 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780444E+05 | tokens_per_sec_per_gpu: 9.031944E+03 | global_batch_size: 64 | lm_loss: 1.091850E+01 | lr: 2.235E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 35563500.000 [ip-26-0-154-121:0]:06/21/2023 17:35:29 [INFO|DP=0|PP=0|TP=0]: iteration: 1491 / 150000 | consumed_samples: 95424 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786448E+05 | tokens_per_sec_per_gpu: 9.041325E+03 | global_batch_size: 64 | lm_loss: 1.091906E+01 | lr: 2.236E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 33057034.000 [ip-26-0-154-121:0]:06/21/2023 17:35:30 [INFO|DP=0|PP=0|TP=0]: iteration: 1492 / 150000 | consumed_samples: 95488 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781534E+05 | tokens_per_sec_per_gpu: 9.033647E+03 | global_batch_size: 64 | lm_loss: 1.091847E+01 | lr: 2.238E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 41060756.000 [ip-26-0-154-121:0]:06/21/2023 17:35:31 [INFO|DP=0|PP=0|TP=0]: iteration: 1493 / 150000 | consumed_samples: 95552 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784500E+05 | tokens_per_sec_per_gpu: 9.038281E+03 | global_batch_size: 64 | lm_loss: 1.091729E+01 | lr: 2.239E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 40487836.000 [ip-26-0-154-121:0]:06/21/2023 17:35:32 [INFO|DP=0|PP=0|TP=0]: iteration: 1494 / 150000 | consumed_samples: 95616 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785337E+05 | tokens_per_sec_per_gpu: 9.039589E+03 | global_batch_size: 64 | lm_loss: 1.091866E+01 | lr: 2.241E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 30539378.000 [ip-26-0-154-121:0]:06/21/2023 17:35:33 [INFO|DP=0|PP=0|TP=0]: iteration: 1495 / 150000 | consumed_samples: 95680 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788495E+05 | tokens_per_sec_per_gpu: 9.044524E+03 | global_batch_size: 64 | lm_loss: 1.091431E+01 | lr: 2.242E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 24331898.000 [ip-26-0-154-121:0]:06/21/2023 17:35:34 [INFO|DP=0|PP=0|TP=0]: iteration: 1496 / 150000 | consumed_samples: 95744 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785909E+05 | tokens_per_sec_per_gpu: 9.040483E+03 | global_batch_size: 64 | lm_loss: 1.091532E+01 | lr: 2.244E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 19399098.000 [ip-26-0-154-121:0]:06/21/2023 17:35:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1497 / 150000 | consumed_samples: 95808 | elapsed_time_per_iteration_ms: 920.7 | tokens_per_sec: 5.694573E+05 | tokens_per_sec_per_gpu: 8.897771E+03 | global_batch_size: 64 | lm_loss: 1.091480E+01 | lr: 2.246E-04 | model_tflops_per_gpu: 102.78 | hardware_tflops_per_gpu: 102.78 | grad_norm: 21234602.000 [ip-26-0-154-121:0]:06/21/2023 17:35:36 [INFO|DP=0|PP=0|TP=0]: iteration: 1498 / 150000 | consumed_samples: 95872 | elapsed_time_per_iteration_ms: 903.4 | tokens_per_sec: 5.803419E+05 | tokens_per_sec_per_gpu: 9.067842E+03 | global_batch_size: 64 | lm_loss: 1.092180E+01 | lr: 2.247E-04 | model_tflops_per_gpu: 104.74 | hardware_tflops_per_gpu: 104.74 | grad_norm: 24739146.000 [ip-26-0-154-121:0]:06/21/2023 17:35:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1499 / 150000 | consumed_samples: 95936 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784442E+05 | tokens_per_sec_per_gpu: 9.038190E+03 | global_batch_size: 64 | lm_loss: 1.092526E+01 | lr: 2.248E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 25419384.000 [ip-26-0-154-121:0]:06/21/2023 17:35:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1500 / 150000 | consumed_samples: 96000 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781566E+05 | tokens_per_sec_per_gpu: 9.033697E+03 | global_batch_size: 64 | lm_loss: 1.091209E+01 | lr: 2.250E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 28142994.000 [ip-26-0-154-121:0]:06/21/2023 17:35:38 [INFO|DP=0|PP=0|TP=0]: iteration: 1501 / 150000 | consumed_samples: 96064 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784253E+05 | tokens_per_sec_per_gpu: 9.037896E+03 | global_batch_size: 64 | lm_loss: 1.091677E+01 | lr: 2.251E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 25766428.000 [ip-26-0-154-121:0]:06/21/2023 17:35:39 [INFO|DP=0|PP=0|TP=0]: iteration: 1502 / 150000 | consumed_samples: 96128 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784597E+05 | tokens_per_sec_per_gpu: 9.038433E+03 | global_batch_size: 64 | lm_loss: 1.092122E+01 | lr: 2.253E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 38840876.000 [ip-26-0-154-121:0]:06/21/2023 17:35:40 [INFO|DP=0|PP=0|TP=0]: iteration: 1503 / 150000 | consumed_samples: 96192 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783620E+05 | tokens_per_sec_per_gpu: 9.036907E+03 | global_batch_size: 64 | lm_loss: 1.092488E+01 | lr: 2.254E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 47494616.000 [ip-26-0-154-121:0]:06/21/2023 17:35:41 [INFO|DP=0|PP=0|TP=0]: iteration: 1504 / 150000 | consumed_samples: 96256 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784515E+05 | tokens_per_sec_per_gpu: 9.038305E+03 | global_batch_size: 64 | lm_loss: 1.091531E+01 | lr: 2.256E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 29836958.000 [ip-26-0-154-121:0]:06/21/2023 17:35:42 [INFO|DP=0|PP=0|TP=0]: iteration: 1505 / 150000 | consumed_samples: 96320 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785478E+05 | tokens_per_sec_per_gpu: 9.039810E+03 | global_batch_size: 64 | lm_loss: 1.092225E+01 | lr: 2.257E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 34454060.000 [ip-26-0-154-121:0]:06/21/2023 17:35:43 [INFO|DP=0|PP=0|TP=0]: iteration: 1506 / 150000 | consumed_samples: 96384 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779651E+05 | tokens_per_sec_per_gpu: 9.030705E+03 | global_batch_size: 64 | lm_loss: 1.092306E+01 | lr: 2.259E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 26792066.000 [ip-26-0-154-121:0]:06/21/2023 17:35:44 [INFO|DP=0|PP=0|TP=0]: iteration: 1507 / 150000 | consumed_samples: 96448 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781881E+05 | tokens_per_sec_per_gpu: 9.034189E+03 | global_batch_size: 64 | lm_loss: 1.091884E+01 | lr: 2.260E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 27353874.000 [ip-26-0-154-121:0]:06/21/2023 17:35:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1508 / 150000 | consumed_samples: 96512 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781920E+05 | tokens_per_sec_per_gpu: 9.034250E+03 | global_batch_size: 64 | lm_loss: 1.092447E+01 | lr: 2.262E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 35002240.000 [ip-26-0-154-121:0]:06/21/2023 17:35:46 [INFO|DP=0|PP=0|TP=0]: iteration: 1509 / 150000 | consumed_samples: 96576 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784702E+05 | tokens_per_sec_per_gpu: 9.038597E+03 | global_batch_size: 64 | lm_loss: 1.092068E+01 | lr: 2.263E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 39742884.000 [ip-26-0-154-121:0]:06/21/2023 17:35:46 [INFO|DP=0|PP=0|TP=0]: iteration: 1510 / 150000 | consumed_samples: 96640 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785073E+05 | tokens_per_sec_per_gpu: 9.039177E+03 | global_batch_size: 64 | lm_loss: 1.091833E+01 | lr: 2.265E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 37607636.000 [ip-26-0-154-121:0]:06/21/2023 17:35:47 [INFO|DP=0|PP=0|TP=0]: iteration: 1511 / 150000 | consumed_samples: 96704 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784599E+05 | tokens_per_sec_per_gpu: 9.038435E+03 | global_batch_size: 64 | lm_loss: 1.091141E+01 | lr: 2.266E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 30674992.000 [ip-26-0-154-121:0]:06/21/2023 17:35:48 [INFO|DP=0|PP=0|TP=0]: iteration: 1512 / 150000 | consumed_samples: 96768 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782685E+05 | tokens_per_sec_per_gpu: 9.035445E+03 | global_batch_size: 64 | lm_loss: 1.091351E+01 | lr: 2.268E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 25216042.000 [ip-26-0-154-121:0]:06/21/2023 17:35:49 [INFO|DP=0|PP=0|TP=0]: iteration: 1513 / 150000 | consumed_samples: 96832 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784705E+05 | tokens_per_sec_per_gpu: 9.038602E+03 | global_batch_size: 64 | lm_loss: 1.091698E+01 | lr: 2.269E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 24410558.000 [ip-26-0-154-121:0]:06/21/2023 17:35:50 [INFO|DP=0|PP=0|TP=0]: iteration: 1514 / 150000 | consumed_samples: 96896 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785740E+05 | tokens_per_sec_per_gpu: 9.040219E+03 | global_batch_size: 64 | lm_loss: 1.090853E+01 | lr: 2.271E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 32942808.000 [ip-26-0-154-121:0]:06/21/2023 17:35:51 [INFO|DP=0|PP=0|TP=0]: iteration: 1515 / 150000 | consumed_samples: 96960 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784343E+05 | tokens_per_sec_per_gpu: 9.038036E+03 | global_batch_size: 64 | lm_loss: 1.091679E+01 | lr: 2.272E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 42715796.000 [ip-26-0-154-121:0]:06/21/2023 17:35:52 [INFO|DP=0|PP=0|TP=0]: iteration: 1516 / 150000 | consumed_samples: 97024 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787054E+05 | tokens_per_sec_per_gpu: 9.042272E+03 | global_batch_size: 64 | lm_loss: 1.091597E+01 | lr: 2.274E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 41326480.000 [ip-26-0-154-121:0]:06/21/2023 17:35:53 [INFO|DP=0|PP=0|TP=0]: iteration: 1517 / 150000 | consumed_samples: 97088 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786823E+05 | tokens_per_sec_per_gpu: 9.041910E+03 | global_batch_size: 64 | lm_loss: 1.091726E+01 | lr: 2.275E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 36504272.000 [ip-26-0-154-121:0]:06/21/2023 17:35:54 [INFO|DP=0|PP=0|TP=0]: iteration: 1518 / 150000 | consumed_samples: 97152 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786718E+05 | tokens_per_sec_per_gpu: 9.041746E+03 | global_batch_size: 64 | lm_loss: 1.091291E+01 | lr: 2.277E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 22263910.000 [ip-26-0-154-121:0]:06/21/2023 17:35:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1519 / 150000 | consumed_samples: 97216 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784582E+05 | tokens_per_sec_per_gpu: 9.038409E+03 | global_batch_size: 64 | lm_loss: 1.091578E+01 | lr: 2.278E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 15284308.000 [ip-26-0-154-121:0]:06/21/2023 17:35:56 [INFO|DP=0|PP=0|TP=0]: iteration: 1520 / 150000 | consumed_samples: 97280 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781432E+05 | tokens_per_sec_per_gpu: 9.033488E+03 | global_batch_size: 64 | lm_loss: 1.091825E+01 | lr: 2.280E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 13563038.000 [ip-26-0-154-121:0]:06/21/2023 17:35:56 [INFO|DP=0|PP=0|TP=0]: iteration: 1521 / 150000 | consumed_samples: 97344 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783506E+05 | tokens_per_sec_per_gpu: 9.036729E+03 | global_batch_size: 64 | lm_loss: 1.091852E+01 | lr: 2.281E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 14535442.000 [ip-26-0-154-121:0]:06/21/2023 17:35:57 [INFO|DP=0|PP=0|TP=0]: iteration: 1522 / 150000 | consumed_samples: 97408 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787569E+05 | tokens_per_sec_per_gpu: 9.043076E+03 | global_batch_size: 64 | lm_loss: 1.091194E+01 | lr: 2.283E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 14166832.000 [ip-26-0-154-121:0]:06/21/2023 17:35:58 [INFO|DP=0|PP=0|TP=0]: iteration: 1523 / 150000 | consumed_samples: 97472 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785608E+05 | tokens_per_sec_per_gpu: 9.040012E+03 | global_batch_size: 64 | lm_loss: 1.091392E+01 | lr: 2.284E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 12088129.000 [ip-26-0-154-121:0]:06/21/2023 17:35:59 [INFO|DP=0|PP=0|TP=0]: iteration: 1524 / 150000 | consumed_samples: 97536 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788827E+05 | tokens_per_sec_per_gpu: 9.045043E+03 | global_batch_size: 64 | lm_loss: 1.091603E+01 | lr: 2.286E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 11495624.000 [ip-26-0-154-121:0]:06/21/2023 17:36:00 [INFO|DP=0|PP=0|TP=0]: iteration: 1525 / 150000 | consumed_samples: 97600 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786935E+05 | tokens_per_sec_per_gpu: 9.042086E+03 | global_batch_size: 64 | lm_loss: 1.091619E+01 | lr: 2.287E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 15434300.000 [ip-26-0-154-121:0]:06/21/2023 17:36:01 [INFO|DP=0|PP=0|TP=0]: iteration: 1526 / 150000 | consumed_samples: 97664 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790266E+05 | tokens_per_sec_per_gpu: 9.047291E+03 | global_batch_size: 64 | lm_loss: 1.091598E+01 | lr: 2.289E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 20208806.000 [ip-26-0-154-121:0]:06/21/2023 17:36:02 [INFO|DP=0|PP=0|TP=0]: iteration: 1527 / 150000 | consumed_samples: 97728 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788582E+05 | tokens_per_sec_per_gpu: 9.044659E+03 | global_batch_size: 64 | lm_loss: 1.091158E+01 | lr: 2.291E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 17022524.000 [ip-26-0-154-121:0]:06/21/2023 17:36:03 [INFO|DP=0|PP=0|TP=0]: iteration: 1528 / 150000 | consumed_samples: 97792 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783526E+05 | tokens_per_sec_per_gpu: 9.036759E+03 | global_batch_size: 64 | lm_loss: 1.091413E+01 | lr: 2.292E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 13918397.000 [ip-26-0-154-121:0]:06/21/2023 17:36:04 [INFO|DP=0|PP=0|TP=0]: iteration: 1529 / 150000 | consumed_samples: 97856 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785959E+05 | tokens_per_sec_per_gpu: 9.040561E+03 | global_batch_size: 64 | lm_loss: 1.090778E+01 | lr: 2.293E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 11107902.000 [ip-26-0-154-121:0]:06/21/2023 17:36:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1530 / 150000 | consumed_samples: 97920 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787057E+05 | tokens_per_sec_per_gpu: 9.042277E+03 | global_batch_size: 64 | lm_loss: 1.091088E+01 | lr: 2.295E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 11555778.000 [ip-26-0-154-121:0]:06/21/2023 17:36:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1531 / 150000 | consumed_samples: 97984 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789265E+05 | tokens_per_sec_per_gpu: 9.045726E+03 | global_batch_size: 64 | lm_loss: 1.091820E+01 | lr: 2.296E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 19156566.000 [ip-26-0-154-121:0]:06/21/2023 17:36:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1532 / 150000 | consumed_samples: 98048 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786051E+05 | tokens_per_sec_per_gpu: 9.040704E+03 | global_batch_size: 64 | lm_loss: 1.091157E+01 | lr: 2.298E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 26646276.000 [ip-26-0-154-121:0]:06/21/2023 17:36:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1533 / 150000 | consumed_samples: 98112 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784494E+05 | tokens_per_sec_per_gpu: 9.038271E+03 | global_batch_size: 64 | lm_loss: 1.091362E+01 | lr: 2.299E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 28070368.000 [ip-26-0-154-121:0]:06/21/2023 17:36:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1534 / 150000 | consumed_samples: 98176 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786108E+05 | tokens_per_sec_per_gpu: 9.040794E+03 | global_batch_size: 64 | lm_loss: 1.091481E+01 | lr: 2.301E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 23726638.000 [ip-26-0-154-121:0]:06/21/2023 17:36:09 [INFO|DP=0|PP=0|TP=0]: iteration: 1535 / 150000 | consumed_samples: 98240 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788838E+05 | tokens_per_sec_per_gpu: 9.045059E+03 | global_batch_size: 64 | lm_loss: 1.090962E+01 | lr: 2.303E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 20242580.000 [ip-26-0-154-121:0]:06/21/2023 17:36:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1536 / 150000 | consumed_samples: 98304 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783342E+05 | tokens_per_sec_per_gpu: 9.036472E+03 | global_batch_size: 64 | lm_loss: 1.091563E+01 | lr: 2.304E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 14031614.000 [ip-26-0-154-121:0]:06/21/2023 17:36:11 [INFO|DP=0|PP=0|TP=0]: iteration: 1537 / 150000 | consumed_samples: 98368 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789536E+05 | tokens_per_sec_per_gpu: 9.046150E+03 | global_batch_size: 64 | lm_loss: 1.091438E+01 | lr: 2.305E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 19665180.000 [ip-26-0-154-121:0]:06/21/2023 17:36:12 [INFO|DP=0|PP=0|TP=0]: iteration: 1538 / 150000 | consumed_samples: 98432 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788326E+05 | tokens_per_sec_per_gpu: 9.044259E+03 | global_batch_size: 64 | lm_loss: 1.091665E+01 | lr: 2.307E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 14093881.000 [ip-26-0-154-121:0]:06/21/2023 17:36:13 [INFO|DP=0|PP=0|TP=0]: iteration: 1539 / 150000 | consumed_samples: 98496 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785043E+05 | tokens_per_sec_per_gpu: 9.039130E+03 | global_batch_size: 64 | lm_loss: 1.090711E+01 | lr: 2.308E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 9667586.000 [ip-26-0-154-121:0]:06/21/2023 17:36:14 [INFO|DP=0|PP=0|TP=0]: iteration: 1540 / 150000 | consumed_samples: 98560 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783588E+05 | tokens_per_sec_per_gpu: 9.036857E+03 | global_batch_size: 64 | lm_loss: 1.090960E+01 | lr: 2.310E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 8677465.000 [ip-26-0-154-121:0]:06/21/2023 17:36:15 [INFO|DP=0|PP=0|TP=0]: iteration: 1541 / 150000 | consumed_samples: 98624 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784020E+05 | tokens_per_sec_per_gpu: 9.037532E+03 | global_batch_size: 64 | lm_loss: 1.091089E+01 | lr: 2.311E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 9773774.000 [ip-26-0-154-121:0]:06/21/2023 17:36:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1542 / 150000 | consumed_samples: 98688 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788705E+05 | tokens_per_sec_per_gpu: 9.044852E+03 | global_batch_size: 64 | lm_loss: 1.090852E+01 | lr: 2.313E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 11300216.000 [ip-26-0-154-121:0]:06/21/2023 17:36:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1543 / 150000 | consumed_samples: 98752 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786848E+05 | tokens_per_sec_per_gpu: 9.041951E+03 | global_batch_size: 64 | lm_loss: 1.089970E+01 | lr: 2.315E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 15930662.000 [ip-26-0-154-121:0]:06/21/2023 17:36:17 [INFO|DP=0|PP=0|TP=0]: iteration: 1544 / 150000 | consumed_samples: 98816 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785363E+05 | tokens_per_sec_per_gpu: 9.039629E+03 | global_batch_size: 64 | lm_loss: 1.091022E+01 | lr: 2.316E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 12725818.000 [ip-26-0-154-121:0]:06/21/2023 17:36:18 [INFO|DP=0|PP=0|TP=0]: iteration: 1545 / 150000 | consumed_samples: 98880 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785933E+05 | tokens_per_sec_per_gpu: 9.040521E+03 | global_batch_size: 64 | lm_loss: 1.090902E+01 | lr: 2.317E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 10460520.000 [ip-26-0-154-121:0]:06/21/2023 17:36:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1546 / 150000 | consumed_samples: 98944 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789396E+05 | tokens_per_sec_per_gpu: 9.045931E+03 | global_batch_size: 64 | lm_loss: 1.090957E+01 | lr: 2.319E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 16938594.000 [ip-26-0-154-121:0]:06/21/2023 17:36:20 [INFO|DP=0|PP=0|TP=0]: iteration: 1547 / 150000 | consumed_samples: 99008 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784806E+05 | tokens_per_sec_per_gpu: 9.038759E+03 | global_batch_size: 64 | lm_loss: 1.090829E+01 | lr: 2.320E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 30064474.000 [ip-26-0-154-121:0]:06/21/2023 17:36:21 [INFO|DP=0|PP=0|TP=0]: iteration: 1548 / 150000 | consumed_samples: 99072 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786565E+05 | tokens_per_sec_per_gpu: 9.041508E+03 | global_batch_size: 64 | lm_loss: 1.090600E+01 | lr: 2.322E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 33899980.000 [ip-26-0-154-121:0]:06/21/2023 17:36:22 [INFO|DP=0|PP=0|TP=0]: iteration: 1549 / 150000 | consumed_samples: 99136 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784087E+05 | tokens_per_sec_per_gpu: 9.037637E+03 | global_batch_size: 64 | lm_loss: 1.090626E+01 | lr: 2.323E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 31805006.000 [ip-26-0-154-121:0]:06/21/2023 17:36:23 [INFO|DP=0|PP=0|TP=0]: iteration: 1550 / 150000 | consumed_samples: 99200 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788035E+05 | tokens_per_sec_per_gpu: 9.043805E+03 | global_batch_size: 64 | lm_loss: 1.090488E+01 | lr: 2.325E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 23212658.000 [ip-26-0-154-121:0]:06/21/2023 17:36:24 [INFO|DP=0|PP=0|TP=0]: iteration: 1551 / 150000 | consumed_samples: 99264 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791552E+05 | tokens_per_sec_per_gpu: 9.049300E+03 | global_batch_size: 64 | lm_loss: 1.090873E+01 | lr: 2.326E-04 | model_tflops_per_gpu: 104.53 | hardware_tflops_per_gpu: 104.53 | grad_norm: 11926750.000 [ip-26-0-154-121:0]:06/21/2023 17:36:25 [INFO|DP=0|PP=0|TP=0]: iteration: 1552 / 150000 | consumed_samples: 99328 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786909E+05 | tokens_per_sec_per_gpu: 9.042046E+03 | global_batch_size: 64 | lm_loss: 1.090379E+01 | lr: 2.328E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 15930308.000 [ip-26-0-154-121:0]:06/21/2023 17:36:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1553 / 150000 | consumed_samples: 99392 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786673E+05 | tokens_per_sec_per_gpu: 9.041677E+03 | global_batch_size: 64 | lm_loss: 1.090888E+01 | lr: 2.329E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 18771548.000 [ip-26-0-154-121:0]:06/21/2023 17:36:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1554 / 150000 | consumed_samples: 99456 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787287E+05 | tokens_per_sec_per_gpu: 9.042636E+03 | global_batch_size: 64 | lm_loss: 1.091145E+01 | lr: 2.331E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 22055770.000 [ip-26-0-154-121:0]:06/21/2023 17:36:27 [INFO|DP=0|PP=0|TP=0]: iteration: 1555 / 150000 | consumed_samples: 99520 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789266E+05 | tokens_per_sec_per_gpu: 9.045728E+03 | global_batch_size: 64 | lm_loss: 1.090717E+01 | lr: 2.332E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 21581634.000 [ip-26-0-154-121:0]:06/21/2023 17:36:28 [INFO|DP=0|PP=0|TP=0]: iteration: 1556 / 150000 | consumed_samples: 99584 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787086E+05 | tokens_per_sec_per_gpu: 9.042322E+03 | global_batch_size: 64 | lm_loss: 1.090556E+01 | lr: 2.334E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 23528346.000 [ip-26-0-154-121:0]:06/21/2023 17:36:29 [INFO|DP=0|PP=0|TP=0]: iteration: 1557 / 150000 | consumed_samples: 99648 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787752E+05 | tokens_per_sec_per_gpu: 9.043362E+03 | global_batch_size: 64 | lm_loss: 1.090376E+01 | lr: 2.335E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 23931958.000 [ip-26-0-154-121:0]:06/21/2023 17:36:30 [INFO|DP=0|PP=0|TP=0]: iteration: 1558 / 150000 | consumed_samples: 99712 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791151E+05 | tokens_per_sec_per_gpu: 9.048673E+03 | global_batch_size: 64 | lm_loss: 1.090965E+01 | lr: 2.337E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 19623844.000 [ip-26-0-154-121:0]:06/21/2023 17:36:31 [INFO|DP=0|PP=0|TP=0]: iteration: 1559 / 150000 | consumed_samples: 99776 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782218E+05 | tokens_per_sec_per_gpu: 9.034716E+03 | global_batch_size: 64 | lm_loss: 1.090827E+01 | lr: 2.338E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 14965788.000 [ip-26-0-154-121:0]:06/21/2023 17:36:32 [INFO|DP=0|PP=0|TP=0]: iteration: 1560 / 150000 | consumed_samples: 99840 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779788E+05 | tokens_per_sec_per_gpu: 9.030919E+03 | global_batch_size: 64 | lm_loss: 1.090446E+01 | lr: 2.340E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 12912515.000 [ip-26-0-154-121:0]:06/21/2023 17:36:33 [INFO|DP=0|PP=0|TP=0]: iteration: 1561 / 150000 | consumed_samples: 99904 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782533E+05 | tokens_per_sec_per_gpu: 9.035208E+03 | global_batch_size: 64 | lm_loss: 1.090899E+01 | lr: 2.341E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 17496836.000 [ip-26-0-154-121:0]:06/21/2023 17:36:34 [INFO|DP=0|PP=0|TP=0]: iteration: 1562 / 150000 | consumed_samples: 99968 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786096E+05 | tokens_per_sec_per_gpu: 9.040775E+03 | global_batch_size: 64 | lm_loss: 1.090989E+01 | lr: 2.343E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 16987568.000 [ip-26-0-154-121:0]:06/21/2023 17:36:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1563 / 150000 | consumed_samples: 100032 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788009E+05 | tokens_per_sec_per_gpu: 9.043764E+03 | global_batch_size: 64 | lm_loss: 1.090155E+01 | lr: 2.344E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 10889407.000 [ip-26-0-154-121:0]:06/21/2023 17:36:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1564 / 150000 | consumed_samples: 100096 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788859E+05 | tokens_per_sec_per_gpu: 9.045093E+03 | global_batch_size: 64 | lm_loss: 1.090550E+01 | lr: 2.346E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 8236634.500 [ip-26-0-154-121:0]:06/21/2023 17:36:36 [INFO|DP=0|PP=0|TP=0]: iteration: 1565 / 150000 | consumed_samples: 100160 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785309E+05 | tokens_per_sec_per_gpu: 9.039546E+03 | global_batch_size: 64 | lm_loss: 1.090537E+01 | lr: 2.347E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 7835511.500 [ip-26-0-154-121:0]:06/21/2023 17:36:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1566 / 150000 | consumed_samples: 100224 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786527E+05 | tokens_per_sec_per_gpu: 9.041449E+03 | global_batch_size: 64 | lm_loss: 1.090262E+01 | lr: 2.349E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 9703609.000 [ip-26-0-154-121:0]:06/21/2023 17:36:38 [INFO|DP=0|PP=0|TP=0]: iteration: 1567 / 150000 | consumed_samples: 100288 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789510E+05 | tokens_per_sec_per_gpu: 9.046110E+03 | global_batch_size: 64 | lm_loss: 1.090671E+01 | lr: 2.350E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 18194230.000 [ip-26-0-154-121:0]:06/21/2023 17:36:39 [INFO|DP=0|PP=0|TP=0]: iteration: 1568 / 150000 | consumed_samples: 100352 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787660E+05 | tokens_per_sec_per_gpu: 9.043219E+03 | global_batch_size: 64 | lm_loss: 1.090833E+01 | lr: 2.352E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 22271426.000 [ip-26-0-154-121:0]:06/21/2023 17:36:40 [INFO|DP=0|PP=0|TP=0]: iteration: 1569 / 150000 | consumed_samples: 100416 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787447E+05 | tokens_per_sec_per_gpu: 9.042886E+03 | global_batch_size: 64 | lm_loss: 1.090682E+01 | lr: 2.353E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 27009902.000 [ip-26-0-154-121:0]:06/21/2023 17:36:41 [INFO|DP=0|PP=0|TP=0]: iteration: 1570 / 150000 | consumed_samples: 100480 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786358E+05 | tokens_per_sec_per_gpu: 9.041185E+03 | global_batch_size: 64 | lm_loss: 1.090625E+01 | lr: 2.355E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 15987931.000 [ip-26-0-154-121:0]:06/21/2023 17:36:42 [INFO|DP=0|PP=0|TP=0]: iteration: 1571 / 150000 | consumed_samples: 100544 | elapsed_time_per_iteration_ms: 913.1 | tokens_per_sec: 5.741816E+05 | tokens_per_sec_per_gpu: 8.971588E+03 | global_batch_size: 64 | lm_loss: 1.090513E+01 | lr: 2.356E-04 | model_tflops_per_gpu: 103.63 | hardware_tflops_per_gpu: 103.63 | grad_norm: 7716547.500 [ip-26-0-154-121:0]:06/21/2023 17:36:43 [INFO|DP=0|PP=0|TP=0]: iteration: 1572 / 150000 | consumed_samples: 100608 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784847E+05 | tokens_per_sec_per_gpu: 9.038823E+03 | global_batch_size: 64 | lm_loss: 1.088262E+01 | lr: 2.358E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 5088322.000 [ip-26-0-154-121:0]:06/21/2023 17:36:44 [INFO|DP=0|PP=0|TP=0]: iteration: 1573 / 150000 | consumed_samples: 100672 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786850E+05 | tokens_per_sec_per_gpu: 9.041953E+03 | global_batch_size: 64 | lm_loss: 1.090445E+01 | lr: 2.359E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 17231228.000 [ip-26-0-154-121:0]:06/21/2023 17:36:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1574 / 150000 | consumed_samples: 100736 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790417E+05 | tokens_per_sec_per_gpu: 9.047527E+03 | global_batch_size: 64 | lm_loss: 1.090140E+01 | lr: 2.361E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 13370843.000 [ip-26-0-154-121:0]:06/21/2023 17:36:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1575 / 150000 | consumed_samples: 100800 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788064E+05 | tokens_per_sec_per_gpu: 9.043850E+03 | global_batch_size: 64 | lm_loss: 1.090455E+01 | lr: 2.362E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 15762053.000 [ip-26-0-154-121:0]:06/21/2023 17:36:46 [INFO|DP=0|PP=0|TP=0]: iteration: 1576 / 150000 | consumed_samples: 100864 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791084E+05 | tokens_per_sec_per_gpu: 9.048568E+03 | global_batch_size: 64 | lm_loss: 1.089106E+01 | lr: 2.364E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 19569684.000 [ip-26-0-154-121:0]:06/21/2023 17:36:47 [INFO|DP=0|PP=0|TP=0]: iteration: 1577 / 150000 | consumed_samples: 100928 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789612E+05 | tokens_per_sec_per_gpu: 9.046269E+03 | global_batch_size: 64 | lm_loss: 1.089259E+01 | lr: 2.365E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 12948010.000 [ip-26-0-154-121:0]:06/21/2023 17:36:48 [INFO|DP=0|PP=0|TP=0]: iteration: 1578 / 150000 | consumed_samples: 100992 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788939E+05 | tokens_per_sec_per_gpu: 9.045217E+03 | global_batch_size: 64 | lm_loss: 1.090348E+01 | lr: 2.367E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 7753491.000 [ip-26-0-154-121:0]:06/21/2023 17:36:49 [INFO|DP=0|PP=0|TP=0]: iteration: 1579 / 150000 | consumed_samples: 101056 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785472E+05 | tokens_per_sec_per_gpu: 9.039800E+03 | global_batch_size: 64 | lm_loss: 1.090196E+01 | lr: 2.368E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 14477706.000 [ip-26-0-154-121:0]:06/21/2023 17:36:50 [INFO|DP=0|PP=0|TP=0]: iteration: 1580 / 150000 | consumed_samples: 101120 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786969E+05 | tokens_per_sec_per_gpu: 9.042139E+03 | global_batch_size: 64 | lm_loss: 1.090428E+01 | lr: 2.370E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 11524293.000 [ip-26-0-154-121:0]:06/21/2023 17:36:51 [INFO|DP=0|PP=0|TP=0]: iteration: 1581 / 150000 | consumed_samples: 101184 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788142E+05 | tokens_per_sec_per_gpu: 9.043971E+03 | global_batch_size: 64 | lm_loss: 1.090707E+01 | lr: 2.371E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 10986000.000 [ip-26-0-154-121:0]:06/21/2023 17:36:52 [INFO|DP=0|PP=0|TP=0]: iteration: 1582 / 150000 | consumed_samples: 101248 | elapsed_time_per_iteration_ms: 905.2 | tokens_per_sec: 5.791758E+05 | tokens_per_sec_per_gpu: 9.049621E+03 | global_batch_size: 64 | lm_loss: 1.088571E+01 | lr: 2.373E-04 | model_tflops_per_gpu: 104.53 | hardware_tflops_per_gpu: 104.53 | grad_norm: 13682488.000 [ip-26-0-154-121:0]:06/21/2023 17:36:53 [INFO|DP=0|PP=0|TP=0]: iteration: 1583 / 150000 | consumed_samples: 101312 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791564E+05 | tokens_per_sec_per_gpu: 9.049319E+03 | global_batch_size: 64 | lm_loss: 1.090576E+01 | lr: 2.374E-04 | model_tflops_per_gpu: 104.53 | hardware_tflops_per_gpu: 104.53 | grad_norm: 12014976.000 [ip-26-0-154-121:0]:06/21/2023 17:36:54 [INFO|DP=0|PP=0|TP=0]: iteration: 1584 / 150000 | consumed_samples: 101376 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786367E+05 | tokens_per_sec_per_gpu: 9.041199E+03 | global_batch_size: 64 | lm_loss: 1.089946E+01 | lr: 2.376E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 6725098.000 [ip-26-0-154-121:0]:06/21/2023 17:36:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1585 / 150000 | consumed_samples: 101440 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784495E+05 | tokens_per_sec_per_gpu: 9.038274E+03 | global_batch_size: 64 | lm_loss: 1.090471E+01 | lr: 2.377E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 7779178.500 [ip-26-0-154-121:0]:06/21/2023 17:36:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1586 / 150000 | consumed_samples: 101504 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783210E+05 | tokens_per_sec_per_gpu: 9.036265E+03 | global_batch_size: 64 | lm_loss: 1.090476E+01 | lr: 2.379E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 16513371.000 [ip-26-0-154-121:0]:06/21/2023 17:36:56 [INFO|DP=0|PP=0|TP=0]: iteration: 1587 / 150000 | consumed_samples: 101568 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785122E+05 | tokens_per_sec_per_gpu: 9.039253E+03 | global_batch_size: 64 | lm_loss: 1.090285E+01 | lr: 2.380E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 18873768.000 [ip-26-0-154-121:0]:06/21/2023 17:36:57 [INFO|DP=0|PP=0|TP=0]: iteration: 1588 / 150000 | consumed_samples: 101632 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784246E+05 | tokens_per_sec_per_gpu: 9.037884E+03 | global_batch_size: 64 | lm_loss: 1.089922E+01 | lr: 2.382E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 19293046.000 [ip-26-0-154-121:0]:06/21/2023 17:36:58 [INFO|DP=0|PP=0|TP=0]: iteration: 1589 / 150000 | consumed_samples: 101696 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787499E+05 | tokens_per_sec_per_gpu: 9.042967E+03 | global_batch_size: 64 | lm_loss: 1.089895E+01 | lr: 2.383E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 13223402.000 [ip-26-0-154-121:0]:06/21/2023 17:36:59 [INFO|DP=0|PP=0|TP=0]: iteration: 1590 / 150000 | consumed_samples: 101760 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787939E+05 | tokens_per_sec_per_gpu: 9.043655E+03 | global_batch_size: 64 | lm_loss: 1.089949E+01 | lr: 2.385E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 10488350.000 [ip-26-0-154-121:0]:06/21/2023 17:37:00 [INFO|DP=0|PP=0|TP=0]: iteration: 1591 / 150000 | consumed_samples: 101824 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784232E+05 | tokens_per_sec_per_gpu: 9.037862E+03 | global_batch_size: 64 | lm_loss: 1.089959E+01 | lr: 2.386E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 10421553.000 [ip-26-0-154-121:0]:06/21/2023 17:37:01 [INFO|DP=0|PP=0|TP=0]: iteration: 1592 / 150000 | consumed_samples: 101888 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786649E+05 | tokens_per_sec_per_gpu: 9.041639E+03 | global_batch_size: 64 | lm_loss: 1.090126E+01 | lr: 2.388E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 10559219.000 [ip-26-0-154-121:0]:06/21/2023 17:37:02 [INFO|DP=0|PP=0|TP=0]: iteration: 1593 / 150000 | consumed_samples: 101952 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789772E+05 | tokens_per_sec_per_gpu: 9.046519E+03 | global_batch_size: 64 | lm_loss: 1.090249E+01 | lr: 2.389E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 7255940.000 [ip-26-0-154-121:0]:06/21/2023 17:37:03 [INFO|DP=0|PP=0|TP=0]: iteration: 1594 / 150000 | consumed_samples: 102016 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785273E+05 | tokens_per_sec_per_gpu: 9.039489E+03 | global_batch_size: 64 | lm_loss: 1.089865E+01 | lr: 2.391E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 5118138.000 [ip-26-0-154-121:0]:06/21/2023 17:37:04 [INFO|DP=0|PP=0|TP=0]: iteration: 1595 / 150000 | consumed_samples: 102080 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784933E+05 | tokens_per_sec_per_gpu: 9.038958E+03 | global_batch_size: 64 | lm_loss: 1.089769E+01 | lr: 2.392E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 5010165.000 [ip-26-0-154-121:0]:06/21/2023 17:37:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1596 / 150000 | consumed_samples: 102144 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786789E+05 | tokens_per_sec_per_gpu: 9.041858E+03 | global_batch_size: 64 | lm_loss: 1.090129E+01 | lr: 2.394E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 6787028.000 [ip-26-0-154-121:0]:06/21/2023 17:37:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1597 / 150000 | consumed_samples: 102208 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783847E+05 | tokens_per_sec_per_gpu: 9.037261E+03 | global_batch_size: 64 | lm_loss: 1.090172E+01 | lr: 2.395E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 7279272.500 [ip-26-0-154-121:0]:06/21/2023 17:37:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1598 / 150000 | consumed_samples: 102272 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785767E+05 | tokens_per_sec_per_gpu: 9.040262E+03 | global_batch_size: 64 | lm_loss: 1.090223E+01 | lr: 2.397E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 6132172.500 [ip-26-0-154-121:0]:06/21/2023 17:37:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1599 / 150000 | consumed_samples: 102336 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784903E+05 | tokens_per_sec_per_gpu: 9.038911E+03 | global_batch_size: 64 | lm_loss: 1.089772E+01 | lr: 2.398E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 6894207.000 [ip-26-0-154-121:0]:06/21/2023 17:37:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1600 / 150000 | consumed_samples: 102400 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786195E+05 | tokens_per_sec_per_gpu: 9.040930E+03 | global_batch_size: 64 | lm_loss: 1.089978E+01 | lr: 2.400E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 10524164.000 [ip-26-0-154-121:0]:06/21/2023 17:37:09 [INFO|DP=0|PP=0|TP=0]: iteration: 1601 / 150000 | consumed_samples: 102464 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787738E+05 | tokens_per_sec_per_gpu: 9.043341E+03 | global_batch_size: 64 | lm_loss: 1.090019E+01 | lr: 2.401E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 17971676.000 [ip-26-0-154-121:0]:06/21/2023 17:37:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1602 / 150000 | consumed_samples: 102528 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784328E+05 | tokens_per_sec_per_gpu: 9.038012E+03 | global_batch_size: 64 | lm_loss: 1.090081E+01 | lr: 2.403E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 20648024.000 [ip-26-0-154-121:0]:06/21/2023 17:37:11 [INFO|DP=0|PP=0|TP=0]: iteration: 1603 / 150000 | consumed_samples: 102592 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784112E+05 | tokens_per_sec_per_gpu: 9.037675E+03 | global_batch_size: 64 | lm_loss: 1.089812E+01 | lr: 2.404E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 16983444.000 [ip-26-0-154-121:0]:06/21/2023 17:37:12 [INFO|DP=0|PP=0|TP=0]: iteration: 1604 / 150000 | consumed_samples: 102656 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786495E+05 | tokens_per_sec_per_gpu: 9.041399E+03 | global_batch_size: 64 | lm_loss: 1.088559E+01 | lr: 2.406E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 12689844.000 [ip-26-0-154-121:0]:06/21/2023 17:37:13 [INFO|DP=0|PP=0|TP=0]: iteration: 1605 / 150000 | consumed_samples: 102720 | elapsed_time_per_iteration_ms: 905.1 | tokens_per_sec: 5.792481E+05 | tokens_per_sec_per_gpu: 9.050751E+03 | global_batch_size: 64 | lm_loss: 1.089754E+01 | lr: 2.407E-04 | model_tflops_per_gpu: 104.55 | hardware_tflops_per_gpu: 104.55 | grad_norm: 13955145.000 [ip-26-0-154-121:0]:06/21/2023 17:37:14 [INFO|DP=0|PP=0|TP=0]: iteration: 1606 / 150000 | consumed_samples: 102784 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788687E+05 | tokens_per_sec_per_gpu: 9.044824E+03 | global_batch_size: 64 | lm_loss: 1.089355E+01 | lr: 2.409E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 24489918.000 [ip-26-0-154-121:0]:06/21/2023 17:37:15 [INFO|DP=0|PP=0|TP=0]: iteration: 1607 / 150000 | consumed_samples: 102848 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781121E+05 | tokens_per_sec_per_gpu: 9.033001E+03 | global_batch_size: 64 | lm_loss: 1.089701E+01 | lr: 2.410E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 20087966.000 [ip-26-0-154-121:0]:06/21/2023 17:37:15 [INFO|DP=0|PP=0|TP=0]: iteration: 1608 / 150000 | consumed_samples: 102912 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789448E+05 | tokens_per_sec_per_gpu: 9.046012E+03 | global_batch_size: 64 | lm_loss: 1.089662E+01 | lr: 2.412E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 13801587.000 [ip-26-0-154-121:0]:06/21/2023 17:37:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1609 / 150000 | consumed_samples: 102976 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787378E+05 | tokens_per_sec_per_gpu: 9.042779E+03 | global_batch_size: 64 | lm_loss: 1.089627E+01 | lr: 2.413E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 8844353.000 [ip-26-0-154-121:0]:06/21/2023 17:37:17 [INFO|DP=0|PP=0|TP=0]: iteration: 1610 / 150000 | consumed_samples: 103040 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786820E+05 | tokens_per_sec_per_gpu: 9.041906E+03 | global_batch_size: 64 | lm_loss: 1.089367E+01 | lr: 2.415E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 6058264.500 [ip-26-0-154-121:0]:06/21/2023 17:37:18 [INFO|DP=0|PP=0|TP=0]: iteration: 1611 / 150000 | consumed_samples: 103104 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787220E+05 | tokens_per_sec_per_gpu: 9.042531E+03 | global_batch_size: 64 | lm_loss: 1.089575E+01 | lr: 2.416E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 20268976.000 [ip-26-0-154-121:0]:06/21/2023 17:37:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1612 / 150000 | consumed_samples: 103168 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785119E+05 | tokens_per_sec_per_gpu: 9.039249E+03 | global_batch_size: 64 | lm_loss: 1.089391E+01 | lr: 2.418E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 20319348.000 [ip-26-0-154-121:0]:06/21/2023 17:37:20 [INFO|DP=0|PP=0|TP=0]: iteration: 1613 / 150000 | consumed_samples: 103232 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786108E+05 | tokens_per_sec_per_gpu: 9.040794E+03 | global_batch_size: 64 | lm_loss: 1.089443E+01 | lr: 2.419E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 12750398.000 [ip-26-0-154-121:0]:06/21/2023 17:37:21 [INFO|DP=0|PP=0|TP=0]: iteration: 1614 / 150000 | consumed_samples: 103296 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787036E+05 | tokens_per_sec_per_gpu: 9.042243E+03 | global_batch_size: 64 | lm_loss: 1.089542E+01 | lr: 2.421E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 14760143.000 [ip-26-0-154-121:0]:06/21/2023 17:37:22 [INFO|DP=0|PP=0|TP=0]: iteration: 1615 / 150000 | consumed_samples: 103360 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785982E+05 | tokens_per_sec_per_gpu: 9.040597E+03 | global_batch_size: 64 | lm_loss: 1.089240E+01 | lr: 2.422E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 10668055.000 [ip-26-0-154-121:0]:06/21/2023 17:37:23 [INFO|DP=0|PP=0|TP=0]: iteration: 1616 / 150000 | consumed_samples: 103424 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785947E+05 | tokens_per_sec_per_gpu: 9.040542E+03 | global_batch_size: 64 | lm_loss: 1.089152E+01 | lr: 2.424E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 6365078.500 [ip-26-0-154-121:0]:06/21/2023 17:37:24 [INFO|DP=0|PP=0|TP=0]: iteration: 1617 / 150000 | consumed_samples: 103488 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789637E+05 | tokens_per_sec_per_gpu: 9.046307E+03 | global_batch_size: 64 | lm_loss: 1.089744E+01 | lr: 2.425E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 6248250.500 [ip-26-0-154-121:0]:06/21/2023 17:37:24 [INFO|DP=0|PP=0|TP=0]: iteration: 1618 / 150000 | consumed_samples: 103552 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786682E+05 | tokens_per_sec_per_gpu: 9.041691E+03 | global_batch_size: 64 | lm_loss: 1.088677E+01 | lr: 2.427E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 25240684.000 [ip-26-0-154-121:0]:06/21/2023 17:37:25 [INFO|DP=0|PP=0|TP=0]: iteration: 1619 / 150000 | consumed_samples: 103616 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786419E+05 | tokens_per_sec_per_gpu: 9.041280E+03 | global_batch_size: 64 | lm_loss: 1.089878E+01 | lr: 2.428E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 13139960.000 [ip-26-0-154-121:0]:06/21/2023 17:37:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1620 / 150000 | consumed_samples: 103680 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784005E+05 | tokens_per_sec_per_gpu: 9.037508E+03 | global_batch_size: 64 | lm_loss: 1.089373E+01 | lr: 2.430E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 13416765.000 [ip-26-0-154-121:0]:06/21/2023 17:37:27 [INFO|DP=0|PP=0|TP=0]: iteration: 1621 / 150000 | consumed_samples: 103744 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787209E+05 | tokens_per_sec_per_gpu: 9.042515E+03 | global_batch_size: 64 | lm_loss: 1.089545E+01 | lr: 2.431E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 25682776.000 [ip-26-0-154-121:0]:06/21/2023 17:37:28 [INFO|DP=0|PP=0|TP=0]: iteration: 1622 / 150000 | consumed_samples: 103808 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787776E+05 | tokens_per_sec_per_gpu: 9.043400E+03 | global_batch_size: 64 | lm_loss: 1.089299E+01 | lr: 2.433E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 19250106.000 [ip-26-0-154-121:0]:06/21/2023 17:37:29 [INFO|DP=0|PP=0|TP=0]: iteration: 1623 / 150000 | consumed_samples: 103872 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787484E+05 | tokens_per_sec_per_gpu: 9.042943E+03 | global_batch_size: 64 | lm_loss: 1.089520E+01 | lr: 2.434E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 10310394.000 [ip-26-0-154-121:0]:06/21/2023 17:37:30 [INFO|DP=0|PP=0|TP=0]: iteration: 1624 / 150000 | consumed_samples: 103936 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785399E+05 | tokens_per_sec_per_gpu: 9.039686E+03 | global_batch_size: 64 | lm_loss: 1.089329E+01 | lr: 2.436E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 12065155.000 [ip-26-0-154-121:0]:06/21/2023 17:37:31 [INFO|DP=0|PP=0|TP=0]: iteration: 1625 / 150000 | consumed_samples: 104000 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786445E+05 | tokens_per_sec_per_gpu: 9.041320E+03 | global_batch_size: 64 | lm_loss: 1.089280E+01 | lr: 2.437E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 26624656.000 [ip-26-0-154-121:0]:06/21/2023 17:37:32 [INFO|DP=0|PP=0|TP=0]: iteration: 1626 / 150000 | consumed_samples: 104064 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786000E+05 | tokens_per_sec_per_gpu: 9.040626E+03 | global_batch_size: 64 | lm_loss: 1.089692E+01 | lr: 2.439E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 55209224.000 [ip-26-0-154-121:0]:06/21/2023 17:37:33 [INFO|DP=0|PP=0|TP=0]: iteration: 1627 / 150000 | consumed_samples: 104128 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785938E+05 | tokens_per_sec_per_gpu: 9.040528E+03 | global_batch_size: 64 | lm_loss: 1.089309E+01 | lr: 2.441E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 48532456.000 [ip-26-0-154-121:0]:06/21/2023 17:37:34 [INFO|DP=0|PP=0|TP=0]: iteration: 1628 / 150000 | consumed_samples: 104192 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788961E+05 | tokens_per_sec_per_gpu: 9.045252E+03 | global_batch_size: 64 | lm_loss: 1.089466E+01 | lr: 2.442E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 47780180.000 [ip-26-0-154-121:0]:06/21/2023 17:37:34 [INFO|DP=0|PP=0|TP=0]: iteration: 1629 / 150000 | consumed_samples: 104256 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787497E+05 | tokens_per_sec_per_gpu: 9.042965E+03 | global_batch_size: 64 | lm_loss: 1.089269E+01 | lr: 2.444E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 36540836.000 [ip-26-0-154-121:0]:06/21/2023 17:37:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1630 / 150000 | consumed_samples: 104320 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784988E+05 | tokens_per_sec_per_gpu: 9.039044E+03 | global_batch_size: 64 | lm_loss: 1.089873E+01 | lr: 2.445E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 16843536.000 [ip-26-0-154-121:0]:06/21/2023 17:37:36 [INFO|DP=0|PP=0|TP=0]: iteration: 1631 / 150000 | consumed_samples: 104384 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787773E+05 | tokens_per_sec_per_gpu: 9.043395E+03 | global_batch_size: 64 | lm_loss: 1.088871E+01 | lr: 2.446E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 17107260.000 [ip-26-0-154-121:0]:06/21/2023 17:37:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1632 / 150000 | consumed_samples: 104448 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784081E+05 | tokens_per_sec_per_gpu: 9.037627E+03 | global_batch_size: 64 | lm_loss: 1.089601E+01 | lr: 2.448E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 15666192.000 [ip-26-0-154-121:0]:06/21/2023 17:37:38 [INFO|DP=0|PP=0|TP=0]: iteration: 1633 / 150000 | consumed_samples: 104512 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786316E+05 | tokens_per_sec_per_gpu: 9.041118E+03 | global_batch_size: 64 | lm_loss: 1.089486E+01 | lr: 2.449E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 10560766.000 [ip-26-0-154-121:0]:06/21/2023 17:37:39 [INFO|DP=0|PP=0|TP=0]: iteration: 1634 / 150000 | consumed_samples: 104576 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787561E+05 | tokens_per_sec_per_gpu: 9.043064E+03 | global_batch_size: 64 | lm_loss: 1.089230E+01 | lr: 2.451E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 7765064.500 [ip-26-0-154-121:0]:06/21/2023 17:37:40 [INFO|DP=0|PP=0|TP=0]: iteration: 1635 / 150000 | consumed_samples: 104640 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788742E+05 | tokens_per_sec_per_gpu: 9.044909E+03 | global_batch_size: 64 | lm_loss: 1.089133E+01 | lr: 2.452E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 8068127.000 [ip-26-0-154-121:0]:06/21/2023 17:37:41 [INFO|DP=0|PP=0|TP=0]: iteration: 1636 / 150000 | consumed_samples: 104704 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786130E+05 | tokens_per_sec_per_gpu: 9.040828E+03 | global_batch_size: 64 | lm_loss: 1.089119E+01 | lr: 2.454E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 13702528.000 [ip-26-0-154-121:0]:06/21/2023 17:37:42 [INFO|DP=0|PP=0|TP=0]: iteration: 1637 / 150000 | consumed_samples: 104768 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787380E+05 | tokens_per_sec_per_gpu: 9.042781E+03 | global_batch_size: 64 | lm_loss: 1.089323E+01 | lr: 2.455E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 35375800.000 [ip-26-0-154-121:0]:06/21/2023 17:37:43 [INFO|DP=0|PP=0|TP=0]: iteration: 1638 / 150000 | consumed_samples: 104832 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789480E+05 | tokens_per_sec_per_gpu: 9.046062E+03 | global_batch_size: 64 | lm_loss: 1.088779E+01 | lr: 2.457E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 29368342.000 [ip-26-0-154-121:0]:06/21/2023 17:37:44 [INFO|DP=0|PP=0|TP=0]: iteration: 1639 / 150000 | consumed_samples: 104896 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786647E+05 | tokens_per_sec_per_gpu: 9.041637E+03 | global_batch_size: 64 | lm_loss: 1.089197E+01 | lr: 2.458E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 21714668.000 [ip-26-0-154-121:0]:06/21/2023 17:37:44 [INFO|DP=0|PP=0|TP=0]: iteration: 1640 / 150000 | consumed_samples: 104960 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783158E+05 | tokens_per_sec_per_gpu: 9.036184E+03 | global_batch_size: 64 | lm_loss: 1.088781E+01 | lr: 2.460E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 18379702.000 [ip-26-0-154-121:0]:06/21/2023 17:37:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1641 / 150000 | consumed_samples: 105024 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783733E+05 | tokens_per_sec_per_gpu: 9.037083E+03 | global_batch_size: 64 | lm_loss: 1.088993E+01 | lr: 2.461E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 14377177.000 [ip-26-0-154-121:0]:06/21/2023 17:37:46 [INFO|DP=0|PP=0|TP=0]: iteration: 1642 / 150000 | consumed_samples: 105088 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783470E+05 | tokens_per_sec_per_gpu: 9.036671E+03 | global_batch_size: 64 | lm_loss: 1.089033E+01 | lr: 2.463E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 8902554.000 [ip-26-0-154-121:0]:06/21/2023 17:37:47 [INFO|DP=0|PP=0|TP=0]: iteration: 1643 / 150000 | consumed_samples: 105152 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790678E+05 | tokens_per_sec_per_gpu: 9.047934E+03 | global_batch_size: 64 | lm_loss: 1.089278E+01 | lr: 2.465E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 9123818.000 [ip-26-0-154-121:0]:06/21/2023 17:37:48 [INFO|DP=0|PP=0|TP=0]: iteration: 1644 / 150000 | consumed_samples: 105216 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783082E+05 | tokens_per_sec_per_gpu: 9.036065E+03 | global_batch_size: 64 | lm_loss: 1.089667E+01 | lr: 2.466E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 9002205.000 [ip-26-0-154-121:0]:06/21/2023 17:37:49 [INFO|DP=0|PP=0|TP=0]: iteration: 1645 / 150000 | consumed_samples: 105280 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786716E+05 | tokens_per_sec_per_gpu: 9.041744E+03 | global_batch_size: 64 | lm_loss: 1.089693E+01 | lr: 2.467E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 10255650.000 [ip-26-0-154-121:0]:06/21/2023 17:37:50 [INFO|DP=0|PP=0|TP=0]: iteration: 1646 / 150000 | consumed_samples: 105344 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786180E+05 | tokens_per_sec_per_gpu: 9.040906E+03 | global_batch_size: 64 | lm_loss: 1.089559E+01 | lr: 2.469E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 9601890.000 [ip-26-0-154-121:0]:06/21/2023 17:37:51 [INFO|DP=0|PP=0|TP=0]: iteration: 1647 / 150000 | consumed_samples: 105408 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787514E+05 | tokens_per_sec_per_gpu: 9.042991E+03 | global_batch_size: 64 | lm_loss: 1.089505E+01 | lr: 2.471E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 6348198.000 [ip-26-0-154-121:0]:06/21/2023 17:37:52 [INFO|DP=0|PP=0|TP=0]: iteration: 1648 / 150000 | consumed_samples: 105472 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786462E+05 | tokens_per_sec_per_gpu: 9.041346E+03 | global_batch_size: 64 | lm_loss: 1.089590E+01 | lr: 2.472E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 5046511.000 [ip-26-0-154-121:0]:06/21/2023 17:37:53 [INFO|DP=0|PP=0|TP=0]: iteration: 1649 / 150000 | consumed_samples: 105536 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789307E+05 | tokens_per_sec_per_gpu: 9.045793E+03 | global_batch_size: 64 | lm_loss: 1.089359E+01 | lr: 2.473E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 5963944.500 [ip-26-0-154-121:0]:06/21/2023 17:37:54 [INFO|DP=0|PP=0|TP=0]: iteration: 1650 / 150000 | consumed_samples: 105600 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787668E+05 | tokens_per_sec_per_gpu: 9.043231E+03 | global_batch_size: 64 | lm_loss: 1.089323E+01 | lr: 2.475E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 9266887.000 [ip-26-0-154-121:0]:06/21/2023 17:37:54 [INFO|DP=0|PP=0|TP=0]: iteration: 1651 / 150000 | consumed_samples: 105664 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790797E+05 | tokens_per_sec_per_gpu: 9.048120E+03 | global_batch_size: 64 | lm_loss: 1.088886E+01 | lr: 2.476E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 14524373.000 [ip-26-0-154-121:0]:06/21/2023 17:37:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1652 / 150000 | consumed_samples: 105728 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785000E+05 | tokens_per_sec_per_gpu: 9.039063E+03 | global_batch_size: 64 | lm_loss: 1.088866E+01 | lr: 2.478E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 11894656.000 [ip-26-0-154-121:0]:06/21/2023 17:37:56 [INFO|DP=0|PP=0|TP=0]: iteration: 1653 / 150000 | consumed_samples: 105792 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787098E+05 | tokens_per_sec_per_gpu: 9.042341E+03 | global_batch_size: 64 | lm_loss: 1.089014E+01 | lr: 2.479E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 12544814.000 [ip-26-0-154-121:0]:06/21/2023 17:37:57 [INFO|DP=0|PP=0|TP=0]: iteration: 1654 / 150000 | consumed_samples: 105856 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788823E+05 | tokens_per_sec_per_gpu: 9.045036E+03 | global_batch_size: 64 | lm_loss: 1.088092E+01 | lr: 2.481E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 6216581.000 [ip-26-0-154-121:0]:06/21/2023 17:37:58 [INFO|DP=0|PP=0|TP=0]: iteration: 1655 / 150000 | consumed_samples: 105920 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788288E+05 | tokens_per_sec_per_gpu: 9.044200E+03 | global_batch_size: 64 | lm_loss: 1.088573E+01 | lr: 2.482E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 4458320.500 [ip-26-0-154-121:0]:06/21/2023 17:37:59 [INFO|DP=0|PP=0|TP=0]: iteration: 1656 / 150000 | consumed_samples: 105984 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790716E+05 | tokens_per_sec_per_gpu: 9.047994E+03 | global_batch_size: 64 | lm_loss: 1.089052E+01 | lr: 2.484E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 4002134.250 [ip-26-0-154-121:0]:06/21/2023 17:38:00 [INFO|DP=0|PP=0|TP=0]: iteration: 1657 / 150000 | consumed_samples: 106048 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784203E+05 | tokens_per_sec_per_gpu: 9.037817E+03 | global_batch_size: 64 | lm_loss: 1.089017E+01 | lr: 2.485E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 4596756.000 [ip-26-0-154-121:0]:06/21/2023 17:38:01 [INFO|DP=0|PP=0|TP=0]: iteration: 1658 / 150000 | consumed_samples: 106112 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785597E+05 | tokens_per_sec_per_gpu: 9.039995E+03 | global_batch_size: 64 | lm_loss: 1.089059E+01 | lr: 2.487E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 5604700.500 [ip-26-0-154-121:0]:06/21/2023 17:38:02 [INFO|DP=0|PP=0|TP=0]: iteration: 1659 / 150000 | consumed_samples: 106176 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789894E+05 | tokens_per_sec_per_gpu: 9.046710E+03 | global_batch_size: 64 | lm_loss: 1.089167E+01 | lr: 2.489E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 6509605.500 [ip-26-0-154-121:0]:06/21/2023 17:38:03 [INFO|DP=0|PP=0|TP=0]: iteration: 1660 / 150000 | consumed_samples: 106240 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786084E+05 | tokens_per_sec_per_gpu: 9.040756E+03 | global_batch_size: 64 | lm_loss: 1.088727E+01 | lr: 2.490E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 6882452.500 [ip-26-0-154-121:0]:06/21/2023 17:38:04 [INFO|DP=0|PP=0|TP=0]: iteration: 1661 / 150000 | consumed_samples: 106304 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788747E+05 | tokens_per_sec_per_gpu: 9.044917E+03 | global_batch_size: 64 | lm_loss: 1.088713E+01 | lr: 2.491E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 6758871.500 [ip-26-0-154-121:0]:06/21/2023 17:38:04 [INFO|DP=0|PP=0|TP=0]: iteration: 1662 / 150000 | consumed_samples: 106368 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789272E+05 | tokens_per_sec_per_gpu: 9.045738E+03 | global_batch_size: 64 | lm_loss: 1.088480E+01 | lr: 2.493E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 5192770.500 [ip-26-0-154-121:0]:06/21/2023 17:38:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1663 / 150000 | consumed_samples: 106432 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789716E+05 | tokens_per_sec_per_gpu: 9.046431E+03 | global_batch_size: 64 | lm_loss: 1.089007E+01 | lr: 2.494E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 4556567.000 [ip-26-0-154-121:0]:06/21/2023 17:38:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1664 / 150000 | consumed_samples: 106496 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787758E+05 | tokens_per_sec_per_gpu: 9.043371E+03 | global_batch_size: 64 | lm_loss: 1.088996E+01 | lr: 2.496E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 5180072.500 [ip-26-0-154-121:0]:06/21/2023 17:38:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1665 / 150000 | consumed_samples: 106560 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788221E+05 | tokens_per_sec_per_gpu: 9.044095E+03 | global_batch_size: 64 | lm_loss: 1.089135E+01 | lr: 2.497E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 8485690.000 [ip-26-0-154-121:0]:06/21/2023 17:38:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1666 / 150000 | consumed_samples: 106624 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785462E+05 | tokens_per_sec_per_gpu: 9.039784E+03 | global_batch_size: 64 | lm_loss: 1.088989E+01 | lr: 2.499E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 24114932.000 [ip-26-0-154-121:0]:06/21/2023 17:38:09 [INFO|DP=0|PP=0|TP=0]: iteration: 1667 / 150000 | consumed_samples: 106688 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786775E+05 | tokens_per_sec_per_gpu: 9.041837E+03 | global_batch_size: 64 | lm_loss: 1.088965E+01 | lr: 2.500E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 92155384.000 [ip-26-0-154-121:0]:06/21/2023 17:38:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1668 / 150000 | consumed_samples: 106752 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784670E+05 | tokens_per_sec_per_gpu: 9.038547E+03 | global_batch_size: 64 | lm_loss: 1.088881E+01 | lr: 2.502E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 73678192.000 [ip-26-0-154-121:0]:06/21/2023 17:38:11 [INFO|DP=0|PP=0|TP=0]: iteration: 1669 / 150000 | consumed_samples: 106816 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790922E+05 | tokens_per_sec_per_gpu: 9.048315E+03 | global_batch_size: 64 | lm_loss: 1.088023E+01 | lr: 2.503E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 38855776.000 [ip-26-0-154-121:0]:06/21/2023 17:38:12 [INFO|DP=0|PP=0|TP=0]: iteration: 1670 / 150000 | consumed_samples: 106880 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785145E+05 | tokens_per_sec_per_gpu: 9.039289E+03 | global_batch_size: 64 | lm_loss: 1.088786E+01 | lr: 2.505E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 10756773.000 [ip-26-0-154-121:0]:06/21/2023 17:38:13 [INFO|DP=0|PP=0|TP=0]: iteration: 1671 / 150000 | consumed_samples: 106944 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787257E+05 | tokens_per_sec_per_gpu: 9.042588E+03 | global_batch_size: 64 | lm_loss: 1.088541E+01 | lr: 2.506E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 5265943.000 [ip-26-0-154-121:0]:06/21/2023 17:38:13 [INFO|DP=0|PP=0|TP=0]: iteration: 1672 / 150000 | consumed_samples: 107008 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791332E+05 | tokens_per_sec_per_gpu: 9.048956E+03 | global_batch_size: 64 | lm_loss: 1.088696E+01 | lr: 2.508E-04 | model_tflops_per_gpu: 104.53 | hardware_tflops_per_gpu: 104.53 | grad_norm: 6654865.500 [ip-26-0-154-121:0]:06/21/2023 17:38:14 [INFO|DP=0|PP=0|TP=0]: iteration: 1673 / 150000 | consumed_samples: 107072 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.783984E+05 | tokens_per_sec_per_gpu: 9.037475E+03 | global_batch_size: 64 | lm_loss: 1.088261E+01 | lr: 2.509E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 23098422.000 [ip-26-0-154-121:0]:06/21/2023 17:38:15 [INFO|DP=0|PP=0|TP=0]: iteration: 1674 / 150000 | consumed_samples: 107136 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783879E+05 | tokens_per_sec_per_gpu: 9.037311E+03 | global_batch_size: 64 | lm_loss: 1.088639E+01 | lr: 2.511E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 27644656.000 [ip-26-0-154-121:0]:06/21/2023 17:38:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1675 / 150000 | consumed_samples: 107200 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786475E+05 | tokens_per_sec_per_gpu: 9.041368E+03 | global_batch_size: 64 | lm_loss: 1.088934E+01 | lr: 2.512E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 15719027.000 [ip-26-0-154-121:0]:06/21/2023 17:38:17 [INFO|DP=0|PP=0|TP=0]: iteration: 1676 / 150000 | consumed_samples: 107264 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785911E+05 | tokens_per_sec_per_gpu: 9.040485E+03 | global_batch_size: 64 | lm_loss: 1.088675E+01 | lr: 2.514E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 6855839.000 [ip-26-0-154-121:0]:06/21/2023 17:38:18 [INFO|DP=0|PP=0|TP=0]: iteration: 1677 / 150000 | consumed_samples: 107328 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787657E+05 | tokens_per_sec_per_gpu: 9.043214E+03 | global_batch_size: 64 | lm_loss: 1.088693E+01 | lr: 2.516E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 4052412.000 [ip-26-0-154-121:0]:06/21/2023 17:38:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1678 / 150000 | consumed_samples: 107392 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785650E+05 | tokens_per_sec_per_gpu: 9.040079E+03 | global_batch_size: 64 | lm_loss: 1.088812E+01 | lr: 2.517E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 5136005.000 [ip-26-0-154-121:0]:06/21/2023 17:38:20 [INFO|DP=0|PP=0|TP=0]: iteration: 1679 / 150000 | consumed_samples: 107456 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786987E+05 | tokens_per_sec_per_gpu: 9.042167E+03 | global_batch_size: 64 | lm_loss: 1.088573E+01 | lr: 2.518E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 14376292.000 [ip-26-0-154-121:0]:06/21/2023 17:38:21 [INFO|DP=0|PP=0|TP=0]: iteration: 1680 / 150000 | consumed_samples: 107520 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787113E+05 | tokens_per_sec_per_gpu: 9.042365E+03 | global_batch_size: 64 | lm_loss: 1.088543E+01 | lr: 2.520E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 18095276.000 [ip-26-0-154-121:0]:06/21/2023 17:38:22 [INFO|DP=0|PP=0|TP=0]: iteration: 1681 / 150000 | consumed_samples: 107584 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786375E+05 | tokens_per_sec_per_gpu: 9.041211E+03 | global_batch_size: 64 | lm_loss: 1.088390E+01 | lr: 2.521E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 16265260.000 [ip-26-0-154-121:0]:06/21/2023 17:38:23 [INFO|DP=0|PP=0|TP=0]: iteration: 1682 / 150000 | consumed_samples: 107648 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787234E+05 | tokens_per_sec_per_gpu: 9.042553E+03 | global_batch_size: 64 | lm_loss: 1.088234E+01 | lr: 2.523E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 10426789.000 [ip-26-0-154-121:0]:06/21/2023 17:38:23 [INFO|DP=0|PP=0|TP=0]: iteration: 1683 / 150000 | consumed_samples: 107712 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780438E+05 | tokens_per_sec_per_gpu: 9.031935E+03 | global_batch_size: 64 | lm_loss: 1.088001E+01 | lr: 2.524E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 5408555.500 [ip-26-0-154-121:0]:06/21/2023 17:38:24 [INFO|DP=0|PP=0|TP=0]: iteration: 1684 / 150000 | consumed_samples: 107776 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782194E+05 | tokens_per_sec_per_gpu: 9.034678E+03 | global_batch_size: 64 | lm_loss: 1.084603E+01 | lr: 2.526E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3216675.750 [ip-26-0-154-121:0]:06/21/2023 17:38:25 [INFO|DP=0|PP=0|TP=0]: iteration: 1685 / 150000 | consumed_samples: 107840 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779275E+05 | tokens_per_sec_per_gpu: 9.030117E+03 | global_batch_size: 64 | lm_loss: 1.087627E+01 | lr: 2.527E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 14417558.000 [ip-26-0-154-121:0]:06/21/2023 17:38:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1686 / 150000 | consumed_samples: 107904 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779191E+05 | tokens_per_sec_per_gpu: 9.029986E+03 | global_batch_size: 64 | lm_loss: 1.088691E+01 | lr: 2.529E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 8167399.500 [ip-26-0-154-121:0]:06/21/2023 17:38:27 [INFO|DP=0|PP=0|TP=0]: iteration: 1687 / 150000 | consumed_samples: 107968 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786722E+05 | tokens_per_sec_per_gpu: 9.041753E+03 | global_batch_size: 64 | lm_loss: 1.088746E+01 | lr: 2.530E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 4870129.000 [ip-26-0-154-121:0]:06/21/2023 17:38:28 [INFO|DP=0|PP=0|TP=0]: iteration: 1688 / 150000 | consumed_samples: 108032 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786195E+05 | tokens_per_sec_per_gpu: 9.040930E+03 | global_batch_size: 64 | lm_loss: 1.088298E+01 | lr: 2.532E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 6560454.500 [ip-26-0-154-121:0]:06/21/2023 17:38:29 [INFO|DP=0|PP=0|TP=0]: iteration: 1689 / 150000 | consumed_samples: 108096 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788452E+05 | tokens_per_sec_per_gpu: 9.044457E+03 | global_batch_size: 64 | lm_loss: 1.087409E+01 | lr: 2.533E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 9286108.000 [ip-26-0-154-121:0]:06/21/2023 17:38:30 [INFO|DP=0|PP=0|TP=0]: iteration: 1690 / 150000 | consumed_samples: 108160 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787583E+05 | tokens_per_sec_per_gpu: 9.043098E+03 | global_batch_size: 64 | lm_loss: 1.088403E+01 | lr: 2.535E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 13915264.000 [ip-26-0-154-121:0]:06/21/2023 17:38:31 [INFO|DP=0|PP=0|TP=0]: iteration: 1691 / 150000 | consumed_samples: 108224 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786807E+05 | tokens_per_sec_per_gpu: 9.041887E+03 | global_batch_size: 64 | lm_loss: 1.088371E+01 | lr: 2.537E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 12391581.000 [ip-26-0-154-121:0]:06/21/2023 17:38:32 [INFO|DP=0|PP=0|TP=0]: iteration: 1692 / 150000 | consumed_samples: 108288 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785586E+05 | tokens_per_sec_per_gpu: 9.039979E+03 | global_batch_size: 64 | lm_loss: 1.088214E+01 | lr: 2.538E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 8390006.000 [ip-26-0-154-121:0]:06/21/2023 17:38:33 [INFO|DP=0|PP=0|TP=0]: iteration: 1693 / 150000 | consumed_samples: 108352 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790190E+05 | tokens_per_sec_per_gpu: 9.047172E+03 | global_batch_size: 64 | lm_loss: 1.088255E+01 | lr: 2.539E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 7326476.000 [ip-26-0-154-121:0]:06/21/2023 17:38:33 [INFO|DP=0|PP=0|TP=0]: iteration: 1694 / 150000 | consumed_samples: 108416 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785417E+05 | tokens_per_sec_per_gpu: 9.039715E+03 | global_batch_size: 64 | lm_loss: 1.088252E+01 | lr: 2.541E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 7623247.000 [ip-26-0-154-121:0]:06/21/2023 17:38:34 [INFO|DP=0|PP=0|TP=0]: iteration: 1695 / 150000 | consumed_samples: 108480 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784200E+05 | tokens_per_sec_per_gpu: 9.037812E+03 | global_batch_size: 64 | lm_loss: 1.088221E+01 | lr: 2.542E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 9362101.000 [ip-26-0-154-121:0]:06/21/2023 17:38:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1696 / 150000 | consumed_samples: 108544 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782857E+05 | tokens_per_sec_per_gpu: 9.035714E+03 | global_batch_size: 64 | lm_loss: 1.088139E+01 | lr: 2.544E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 10657185.000 [ip-26-0-154-121:0]:06/21/2023 17:38:36 [INFO|DP=0|PP=0|TP=0]: iteration: 1697 / 150000 | consumed_samples: 108608 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780995E+05 | tokens_per_sec_per_gpu: 9.032804E+03 | global_batch_size: 64 | lm_loss: 1.087997E+01 | lr: 2.545E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 13411475.000 [ip-26-0-154-121:0]:06/21/2023 17:38:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1698 / 150000 | consumed_samples: 108672 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787688E+05 | tokens_per_sec_per_gpu: 9.043262E+03 | global_batch_size: 64 | lm_loss: 1.087854E+01 | lr: 2.547E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 13900609.000 [ip-26-0-154-121:0]:06/21/2023 17:38:38 [INFO|DP=0|PP=0|TP=0]: iteration: 1699 / 150000 | consumed_samples: 108736 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787648E+05 | tokens_per_sec_per_gpu: 9.043200E+03 | global_batch_size: 64 | lm_loss: 1.088179E+01 | lr: 2.548E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 11287273.000 [ip-26-0-154-121:0]:06/21/2023 17:38:39 [INFO|DP=0|PP=0|TP=0]: iteration: 1700 / 150000 | consumed_samples: 108800 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786853E+05 | tokens_per_sec_per_gpu: 9.041958E+03 | global_batch_size: 64 | lm_loss: 1.088308E+01 | lr: 2.550E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 7752381.500 [ip-26-0-154-121:0]:06/21/2023 17:38:40 [INFO|DP=0|PP=0|TP=0]: iteration: 1701 / 150000 | consumed_samples: 108864 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.782002E+05 | tokens_per_sec_per_gpu: 9.034379E+03 | global_batch_size: 64 | lm_loss: 1.088094E+01 | lr: 2.551E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 4192068.250 [ip-26-0-154-121:0]:06/21/2023 17:38:41 [INFO|DP=0|PP=0|TP=0]: iteration: 1702 / 150000 | consumed_samples: 108928 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781633E+05 | tokens_per_sec_per_gpu: 9.033801E+03 | global_batch_size: 64 | lm_loss: 1.087935E+01 | lr: 2.553E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3128326.750 [ip-26-0-154-121:0]:06/21/2023 17:38:42 [INFO|DP=0|PP=0|TP=0]: iteration: 1703 / 150000 | consumed_samples: 108992 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787938E+05 | tokens_per_sec_per_gpu: 9.043652E+03 | global_batch_size: 64 | lm_loss: 1.088141E+01 | lr: 2.554E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 3702495.250 [ip-26-0-154-121:0]:06/21/2023 17:38:43 [INFO|DP=0|PP=0|TP=0]: iteration: 1704 / 150000 | consumed_samples: 109056 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788273E+05 | tokens_per_sec_per_gpu: 9.044176E+03 | global_batch_size: 64 | lm_loss: 1.087854E+01 | lr: 2.556E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 4048018.000 [ip-26-0-154-121:0]:06/21/2023 17:38:43 [INFO|DP=0|PP=0|TP=0]: iteration: 1705 / 150000 | consumed_samples: 109120 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785970E+05 | tokens_per_sec_per_gpu: 9.040578E+03 | global_batch_size: 64 | lm_loss: 1.088053E+01 | lr: 2.557E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 5029414.000 [ip-26-0-154-121:0]:06/21/2023 17:38:44 [INFO|DP=0|PP=0|TP=0]: iteration: 1706 / 150000 | consumed_samples: 109184 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786629E+05 | tokens_per_sec_per_gpu: 9.041608E+03 | global_batch_size: 64 | lm_loss: 1.087840E+01 | lr: 2.559E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 13345270.000 [ip-26-0-154-121:0]:06/21/2023 17:38:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1707 / 150000 | consumed_samples: 109248 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780335E+05 | tokens_per_sec_per_gpu: 9.031773E+03 | global_batch_size: 64 | lm_loss: 1.087659E+01 | lr: 2.561E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 42798216.000 [ip-26-0-154-121:0]:06/21/2023 17:38:46 [INFO|DP=0|PP=0|TP=0]: iteration: 1708 / 150000 | consumed_samples: 109312 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786095E+05 | tokens_per_sec_per_gpu: 9.040773E+03 | global_batch_size: 64 | lm_loss: 1.087891E+01 | lr: 2.562E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 39894724.000 [ip-26-0-154-121:0]:06/21/2023 17:38:47 [INFO|DP=0|PP=0|TP=0]: iteration: 1709 / 150000 | consumed_samples: 109376 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781668E+05 | tokens_per_sec_per_gpu: 9.033856E+03 | global_batch_size: 64 | lm_loss: 1.087761E+01 | lr: 2.563E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 25316898.000 [ip-26-0-154-121:0]:06/21/2023 17:38:48 [INFO|DP=0|PP=0|TP=0]: iteration: 1710 / 150000 | consumed_samples: 109440 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784533E+05 | tokens_per_sec_per_gpu: 9.038333E+03 | global_batch_size: 64 | lm_loss: 1.087644E+01 | lr: 2.565E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 7140079.000 [ip-26-0-154-121:0]:06/21/2023 17:38:49 [INFO|DP=0|PP=0|TP=0]: iteration: 1711 / 150000 | consumed_samples: 109504 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787433E+05 | tokens_per_sec_per_gpu: 9.042865E+03 | global_batch_size: 64 | lm_loss: 1.087534E+01 | lr: 2.566E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 4343528.500 [ip-26-0-154-121:0]:06/21/2023 17:38:50 [INFO|DP=0|PP=0|TP=0]: iteration: 1712 / 150000 | consumed_samples: 109568 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786941E+05 | tokens_per_sec_per_gpu: 9.042096E+03 | global_batch_size: 64 | lm_loss: 1.087995E+01 | lr: 2.568E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 5676927.000 [ip-26-0-154-121:0]:06/21/2023 17:38:51 [INFO|DP=0|PP=0|TP=0]: iteration: 1713 / 150000 | consumed_samples: 109632 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788344E+05 | tokens_per_sec_per_gpu: 9.044288E+03 | global_batch_size: 64 | lm_loss: 1.087811E+01 | lr: 2.569E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 32064372.000 [ip-26-0-154-121:0]:06/21/2023 17:38:52 [INFO|DP=0|PP=0|TP=0]: iteration: 1714 / 150000 | consumed_samples: 109696 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787356E+05 | tokens_per_sec_per_gpu: 9.042743E+03 | global_batch_size: 64 | lm_loss: 1.087721E+01 | lr: 2.571E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 25955388.000 [ip-26-0-154-121:0]:06/21/2023 17:38:52 [INFO|DP=0|PP=0|TP=0]: iteration: 1715 / 150000 | consumed_samples: 109760 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785270E+05 | tokens_per_sec_per_gpu: 9.039484E+03 | global_batch_size: 64 | lm_loss: 1.087959E+01 | lr: 2.572E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 21175002.000 [ip-26-0-154-121:0]:06/21/2023 17:38:53 [INFO|DP=0|PP=0|TP=0]: iteration: 1716 / 150000 | consumed_samples: 109824 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785746E+05 | tokens_per_sec_per_gpu: 9.040228E+03 | global_batch_size: 64 | lm_loss: 1.087283E+01 | lr: 2.574E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 17482518.000 [ip-26-0-154-121:0]:06/21/2023 17:38:54 [INFO|DP=0|PP=0|TP=0]: iteration: 1717 / 150000 | consumed_samples: 109888 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788081E+05 | tokens_per_sec_per_gpu: 9.043876E+03 | global_batch_size: 64 | lm_loss: 1.087975E+01 | lr: 2.575E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 8493806.000 [ip-26-0-154-121:0]:06/21/2023 17:38:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1718 / 150000 | consumed_samples: 109952 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787235E+05 | tokens_per_sec_per_gpu: 9.042555E+03 | global_batch_size: 64 | lm_loss: 1.087862E+01 | lr: 2.577E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 4200373.000 [ip-26-0-154-121:0]:06/21/2023 17:38:56 [INFO|DP=0|PP=0|TP=0]: iteration: 1719 / 150000 | consumed_samples: 110016 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790469E+05 | tokens_per_sec_per_gpu: 9.047608E+03 | global_batch_size: 64 | lm_loss: 1.087952E+01 | lr: 2.578E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 3016260.750 [ip-26-0-154-121:0]:06/21/2023 17:38:57 [INFO|DP=0|PP=0|TP=0]: iteration: 1720 / 150000 | consumed_samples: 110080 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785904E+05 | tokens_per_sec_per_gpu: 9.040476E+03 | global_batch_size: 64 | lm_loss: 1.086963E+01 | lr: 2.580E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 29948996.000 [ip-26-0-154-121:0]:06/21/2023 17:38:58 [INFO|DP=0|PP=0|TP=0]: iteration: 1721 / 150000 | consumed_samples: 110144 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789393E+05 | tokens_per_sec_per_gpu: 9.045926E+03 | global_batch_size: 64 | lm_loss: 1.087812E+01 | lr: 2.582E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 3691682.000 [ip-26-0-154-121:0]:06/21/2023 17:38:59 [INFO|DP=0|PP=0|TP=0]: iteration: 1722 / 150000 | consumed_samples: 110208 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787014E+05 | tokens_per_sec_per_gpu: 9.042210E+03 | global_batch_size: 64 | lm_loss: 1.087584E+01 | lr: 2.583E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3245183.250 [ip-26-0-154-121:0]:06/21/2023 17:39:00 [INFO|DP=0|PP=0|TP=0]: iteration: 1723 / 150000 | consumed_samples: 110272 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785742E+05 | tokens_per_sec_per_gpu: 9.040221E+03 | global_batch_size: 64 | lm_loss: 1.086131E+01 | lr: 2.584E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 12737729.000 [ip-26-0-154-121:0]:06/21/2023 17:39:01 [INFO|DP=0|PP=0|TP=0]: iteration: 1724 / 150000 | consumed_samples: 110336 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783655E+05 | tokens_per_sec_per_gpu: 9.036961E+03 | global_batch_size: 64 | lm_loss: 1.087511E+01 | lr: 2.586E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 11904591.000 [ip-26-0-154-121:0]:06/21/2023 17:39:02 [INFO|DP=0|PP=0|TP=0]: iteration: 1725 / 150000 | consumed_samples: 110400 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782544E+05 | tokens_per_sec_per_gpu: 9.035224E+03 | global_batch_size: 64 | lm_loss: 1.087250E+01 | lr: 2.587E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3917080.000 [ip-26-0-154-121:0]:06/21/2023 17:39:02 [INFO|DP=0|PP=0|TP=0]: iteration: 1726 / 150000 | consumed_samples: 110464 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784233E+05 | tokens_per_sec_per_gpu: 9.037865E+03 | global_batch_size: 64 | lm_loss: 1.087581E+01 | lr: 2.589E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3374615.250 [ip-26-0-154-121:0]:06/21/2023 17:39:03 [INFO|DP=0|PP=0|TP=0]: iteration: 1727 / 150000 | consumed_samples: 110528 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789332E+05 | tokens_per_sec_per_gpu: 9.045831E+03 | global_batch_size: 64 | lm_loss: 1.087655E+01 | lr: 2.590E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 3585006.750 [ip-26-0-154-121:0]:06/21/2023 17:39:04 [INFO|DP=0|PP=0|TP=0]: iteration: 1728 / 150000 | consumed_samples: 110592 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786670E+05 | tokens_per_sec_per_gpu: 9.041672E+03 | global_batch_size: 64 | lm_loss: 1.087512E+01 | lr: 2.592E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 5354932.500 [ip-26-0-154-121:0]:06/21/2023 17:39:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1729 / 150000 | consumed_samples: 110656 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785950E+05 | tokens_per_sec_per_gpu: 9.040547E+03 | global_batch_size: 64 | lm_loss: 1.087803E+01 | lr: 2.593E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 9727144.000 [ip-26-0-154-121:0]:06/21/2023 17:39:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1730 / 150000 | consumed_samples: 110720 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783427E+05 | tokens_per_sec_per_gpu: 9.036605E+03 | global_batch_size: 64 | lm_loss: 1.086848E+01 | lr: 2.595E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 14773097.000 [ip-26-0-154-121:0]:06/21/2023 17:39:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1731 / 150000 | consumed_samples: 110784 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783091E+05 | tokens_per_sec_per_gpu: 9.036080E+03 | global_batch_size: 64 | lm_loss: 1.087594E+01 | lr: 2.596E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 16448150.000 [ip-26-0-154-121:0]:06/21/2023 17:39:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1732 / 150000 | consumed_samples: 110848 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781791E+05 | tokens_per_sec_per_gpu: 9.034048E+03 | global_batch_size: 64 | lm_loss: 1.087394E+01 | lr: 2.598E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 12608050.000 [ip-26-0-154-121:0]:06/21/2023 17:39:09 [INFO|DP=0|PP=0|TP=0]: iteration: 1733 / 150000 | consumed_samples: 110912 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783657E+05 | tokens_per_sec_per_gpu: 9.036964E+03 | global_batch_size: 64 | lm_loss: 1.087527E+01 | lr: 2.599E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 6683444.500 [ip-26-0-154-121:0]:06/21/2023 17:39:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1734 / 150000 | consumed_samples: 110976 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784483E+05 | tokens_per_sec_per_gpu: 9.038255E+03 | global_batch_size: 64 | lm_loss: 1.087451E+01 | lr: 2.601E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3727886.750 [ip-26-0-154-121:0]:06/21/2023 17:39:11 [INFO|DP=0|PP=0|TP=0]: iteration: 1735 / 150000 | consumed_samples: 111040 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782682E+05 | tokens_per_sec_per_gpu: 9.035441E+03 | global_batch_size: 64 | lm_loss: 1.087493E+01 | lr: 2.602E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 2778917.250 [ip-26-0-154-121:0]:06/21/2023 17:39:12 [INFO|DP=0|PP=0|TP=0]: iteration: 1736 / 150000 | consumed_samples: 111104 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785423E+05 | tokens_per_sec_per_gpu: 9.039724E+03 | global_batch_size: 64 | lm_loss: 1.087457E+01 | lr: 2.604E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 2840903.500 [ip-26-0-154-121:0]:06/21/2023 17:39:12 [INFO|DP=0|PP=0|TP=0]: iteration: 1737 / 150000 | consumed_samples: 111168 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783464E+05 | tokens_per_sec_per_gpu: 9.036662E+03 | global_batch_size: 64 | lm_loss: 1.086869E+01 | lr: 2.606E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3282955.000 [ip-26-0-154-121:0]:06/21/2023 17:39:13 [INFO|DP=0|PP=0|TP=0]: iteration: 1738 / 150000 | consumed_samples: 111232 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786897E+05 | tokens_per_sec_per_gpu: 9.042027E+03 | global_batch_size: 64 | lm_loss: 1.087365E+01 | lr: 2.607E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3511682.750 [ip-26-0-154-121:0]:06/21/2023 17:39:14 [INFO|DP=0|PP=0|TP=0]: iteration: 1739 / 150000 | consumed_samples: 111296 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785489E+05 | tokens_per_sec_per_gpu: 9.039826E+03 | global_batch_size: 64 | lm_loss: 1.087266E+01 | lr: 2.608E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3521516.250 [ip-26-0-154-121:0]:06/21/2023 17:39:15 [INFO|DP=0|PP=0|TP=0]: iteration: 1740 / 150000 | consumed_samples: 111360 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789681E+05 | tokens_per_sec_per_gpu: 9.046376E+03 | global_batch_size: 64 | lm_loss: 1.087167E+01 | lr: 2.610E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 4162767.250 [ip-26-0-154-121:0]:06/21/2023 17:39:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1741 / 150000 | consumed_samples: 111424 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786338E+05 | tokens_per_sec_per_gpu: 9.041154E+03 | global_batch_size: 64 | lm_loss: 1.086989E+01 | lr: 2.611E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 4463031.500 [ip-26-0-154-121:0]:06/21/2023 17:39:17 [INFO|DP=0|PP=0|TP=0]: iteration: 1742 / 150000 | consumed_samples: 111488 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784489E+05 | tokens_per_sec_per_gpu: 9.038264E+03 | global_batch_size: 64 | lm_loss: 1.087314E+01 | lr: 2.613E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 5489248.500 [ip-26-0-154-121:0]:06/21/2023 17:39:18 [INFO|DP=0|PP=0|TP=0]: iteration: 1743 / 150000 | consumed_samples: 111552 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785460E+05 | tokens_per_sec_per_gpu: 9.039781E+03 | global_batch_size: 64 | lm_loss: 1.087371E+01 | lr: 2.614E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 6864934.500 [ip-26-0-154-121:0]:06/21/2023 17:39:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1744 / 150000 | consumed_samples: 111616 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785882E+05 | tokens_per_sec_per_gpu: 9.040440E+03 | global_batch_size: 64 | lm_loss: 1.087302E+01 | lr: 2.616E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 7711860.000 [ip-26-0-154-121:0]:06/21/2023 17:39:20 [INFO|DP=0|PP=0|TP=0]: iteration: 1745 / 150000 | consumed_samples: 111680 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783488E+05 | tokens_per_sec_per_gpu: 9.036700E+03 | global_batch_size: 64 | lm_loss: 1.086723E+01 | lr: 2.617E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 7724211.500 [ip-26-0-154-121:0]:06/21/2023 17:39:21 [INFO|DP=0|PP=0|TP=0]: iteration: 1746 / 150000 | consumed_samples: 111744 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783473E+05 | tokens_per_sec_per_gpu: 9.036676E+03 | global_batch_size: 64 | lm_loss: 1.086949E+01 | lr: 2.619E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 7558375.500 [ip-26-0-154-121:0]:06/21/2023 17:39:22 [INFO|DP=0|PP=0|TP=0]: iteration: 1747 / 150000 | consumed_samples: 111808 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784159E+05 | tokens_per_sec_per_gpu: 9.037748E+03 | global_batch_size: 64 | lm_loss: 1.086800E+01 | lr: 2.620E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 6462978.000 [ip-26-0-154-121:0]:06/21/2023 17:39:22 [INFO|DP=0|PP=0|TP=0]: iteration: 1748 / 150000 | consumed_samples: 111872 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789275E+05 | tokens_per_sec_per_gpu: 9.045743E+03 | global_batch_size: 64 | lm_loss: 1.087055E+01 | lr: 2.622E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 5463032.500 [ip-26-0-154-121:0]:06/21/2023 17:39:23 [INFO|DP=0|PP=0|TP=0]: iteration: 1749 / 150000 | consumed_samples: 111936 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785232E+05 | tokens_per_sec_per_gpu: 9.039424E+03 | global_batch_size: 64 | lm_loss: 1.086949E+01 | lr: 2.623E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 4470817.500 [ip-26-0-154-121:0]:06/21/2023 17:39:24 [INFO|DP=0|PP=0|TP=0]: iteration: 1750 / 150000 | consumed_samples: 112000 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789076E+05 | tokens_per_sec_per_gpu: 9.045431E+03 | global_batch_size: 64 | lm_loss: 1.087082E+01 | lr: 2.625E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 4209938.000 [ip-26-0-154-121:0]:06/21/2023 17:39:25 [INFO|DP=0|PP=0|TP=0]: iteration: 1751 / 150000 | consumed_samples: 112064 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786038E+05 | tokens_per_sec_per_gpu: 9.040685E+03 | global_batch_size: 64 | lm_loss: 1.087032E+01 | lr: 2.627E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4576490.500 [ip-26-0-154-121:0]:06/21/2023 17:39:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1752 / 150000 | consumed_samples: 112128 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786147E+05 | tokens_per_sec_per_gpu: 9.040854E+03 | global_batch_size: 64 | lm_loss: 1.086938E+01 | lr: 2.628E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 8290916.500 [ip-26-0-154-121:0]:06/21/2023 17:39:27 [INFO|DP=0|PP=0|TP=0]: iteration: 1753 / 150000 | consumed_samples: 112192 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787552E+05 | tokens_per_sec_per_gpu: 9.043050E+03 | global_batch_size: 64 | lm_loss: 1.087080E+01 | lr: 2.629E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 20033182.000 [ip-26-0-154-121:0]:06/21/2023 17:39:28 [INFO|DP=0|PP=0|TP=0]: iteration: 1754 / 150000 | consumed_samples: 112256 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786795E+05 | tokens_per_sec_per_gpu: 9.041867E+03 | global_batch_size: 64 | lm_loss: 1.086966E+01 | lr: 2.631E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 20609482.000 [ip-26-0-154-121:0]:06/21/2023 17:39:29 [INFO|DP=0|PP=0|TP=0]: iteration: 1755 / 150000 | consumed_samples: 112320 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789784E+05 | tokens_per_sec_per_gpu: 9.046538E+03 | global_batch_size: 64 | lm_loss: 1.087107E+01 | lr: 2.632E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 11072209.000 [ip-26-0-154-121:0]:06/21/2023 17:39:30 [INFO|DP=0|PP=0|TP=0]: iteration: 1756 / 150000 | consumed_samples: 112384 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786244E+05 | tokens_per_sec_per_gpu: 9.041006E+03 | global_batch_size: 64 | lm_loss: 1.087156E+01 | lr: 2.634E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 11658495.000 [ip-26-0-154-121:0]:06/21/2023 17:39:31 [INFO|DP=0|PP=0|TP=0]: iteration: 1757 / 150000 | consumed_samples: 112448 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786728E+05 | tokens_per_sec_per_gpu: 9.041763E+03 | global_batch_size: 64 | lm_loss: 1.087234E+01 | lr: 2.635E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 44870344.000 [ip-26-0-154-121:0]:06/21/2023 17:39:32 [INFO|DP=0|PP=0|TP=0]: iteration: 1758 / 150000 | consumed_samples: 112512 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787283E+05 | tokens_per_sec_per_gpu: 9.042629E+03 | global_batch_size: 64 | lm_loss: 1.087170E+01 | lr: 2.637E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 44653228.000 [ip-26-0-154-121:0]:06/21/2023 17:39:32 [INFO|DP=0|PP=0|TP=0]: iteration: 1759 / 150000 | consumed_samples: 112576 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787823E+05 | tokens_per_sec_per_gpu: 9.043474E+03 | global_batch_size: 64 | lm_loss: 1.087076E+01 | lr: 2.638E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 15076023.000 [ip-26-0-154-121:0]:06/21/2023 17:39:33 [INFO|DP=0|PP=0|TP=0]: iteration: 1760 / 150000 | consumed_samples: 112640 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786972E+05 | tokens_per_sec_per_gpu: 9.042143E+03 | global_batch_size: 64 | lm_loss: 1.087150E+01 | lr: 2.640E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 5576236.500 [ip-26-0-154-121:0]:06/21/2023 17:39:34 [INFO|DP=0|PP=0|TP=0]: iteration: 1761 / 150000 | consumed_samples: 112704 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787927E+05 | tokens_per_sec_per_gpu: 9.043636E+03 | global_batch_size: 64 | lm_loss: 1.086794E+01 | lr: 2.641E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 7094419.000 [ip-26-0-154-121:0]:06/21/2023 17:39:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1762 / 150000 | consumed_samples: 112768 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788126E+05 | tokens_per_sec_per_gpu: 9.043948E+03 | global_batch_size: 64 | lm_loss: 1.086757E+01 | lr: 2.643E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 34713020.000 [ip-26-0-154-121:0]:06/21/2023 17:39:36 [INFO|DP=0|PP=0|TP=0]: iteration: 1763 / 150000 | consumed_samples: 112832 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788012E+05 | tokens_per_sec_per_gpu: 9.043769E+03 | global_batch_size: 64 | lm_loss: 1.085774E+01 | lr: 2.644E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 26304834.000 [ip-26-0-154-121:0]:06/21/2023 17:39:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1764 / 150000 | consumed_samples: 112896 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785396E+05 | tokens_per_sec_per_gpu: 9.039681E+03 | global_batch_size: 64 | lm_loss: 1.086722E+01 | lr: 2.646E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 16359142.000 [ip-26-0-154-121:0]:06/21/2023 17:39:38 [INFO|DP=0|PP=0|TP=0]: iteration: 1765 / 150000 | consumed_samples: 112960 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785040E+05 | tokens_per_sec_per_gpu: 9.039125E+03 | global_batch_size: 64 | lm_loss: 1.086598E+01 | lr: 2.648E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 9228101.000 [ip-26-0-154-121:0]:06/21/2023 17:39:39 [INFO|DP=0|PP=0|TP=0]: iteration: 1766 / 150000 | consumed_samples: 113024 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786197E+05 | tokens_per_sec_per_gpu: 9.040932E+03 | global_batch_size: 64 | lm_loss: 1.086575E+01 | lr: 2.649E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4835495.500 [ip-26-0-154-121:0]:06/21/2023 17:39:40 [INFO|DP=0|PP=0|TP=0]: iteration: 1767 / 150000 | consumed_samples: 113088 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783347E+05 | tokens_per_sec_per_gpu: 9.036479E+03 | global_batch_size: 64 | lm_loss: 1.086117E+01 | lr: 2.650E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3134792.750 [ip-26-0-154-121:0]:06/21/2023 17:39:41 [INFO|DP=0|PP=0|TP=0]: iteration: 1768 / 150000 | consumed_samples: 113152 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785649E+05 | tokens_per_sec_per_gpu: 9.040076E+03 | global_batch_size: 64 | lm_loss: 1.086861E+01 | lr: 2.652E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 22662570.000 [ip-26-0-154-121:0]:06/21/2023 17:39:41 [INFO|DP=0|PP=0|TP=0]: iteration: 1769 / 150000 | consumed_samples: 113216 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787765E+05 | tokens_per_sec_per_gpu: 9.043383E+03 | global_batch_size: 64 | lm_loss: 1.086740E+01 | lr: 2.653E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 8246043.500 [ip-26-0-154-121:0]:06/21/2023 17:39:42 [INFO|DP=0|PP=0|TP=0]: iteration: 1770 / 150000 | consumed_samples: 113280 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787945E+05 | tokens_per_sec_per_gpu: 9.043664E+03 | global_batch_size: 64 | lm_loss: 1.087051E+01 | lr: 2.655E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 6381725.500 [ip-26-0-154-121:0]:06/21/2023 17:39:43 [INFO|DP=0|PP=0|TP=0]: iteration: 1771 / 150000 | consumed_samples: 113344 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784576E+05 | tokens_per_sec_per_gpu: 9.038400E+03 | global_batch_size: 64 | lm_loss: 1.087067E+01 | lr: 2.656E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 10284564.000 [ip-26-0-154-121:0]:06/21/2023 17:39:44 [INFO|DP=0|PP=0|TP=0]: iteration: 1772 / 150000 | consumed_samples: 113408 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783873E+05 | tokens_per_sec_per_gpu: 9.037301E+03 | global_batch_size: 64 | lm_loss: 1.086883E+01 | lr: 2.658E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 16288061.000 [ip-26-0-154-121:0]:06/21/2023 17:39:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1773 / 150000 | consumed_samples: 113472 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785501E+05 | tokens_per_sec_per_gpu: 9.039845E+03 | global_batch_size: 64 | lm_loss: 1.086896E+01 | lr: 2.659E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 8280785.500 [ip-26-0-154-121:0]:06/21/2023 17:39:46 [INFO|DP=0|PP=0|TP=0]: iteration: 1774 / 150000 | consumed_samples: 113536 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788811E+05 | tokens_per_sec_per_gpu: 9.045017E+03 | global_batch_size: 64 | lm_loss: 1.086715E+01 | lr: 2.661E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 2894220.750 [ip-26-0-154-121:0]:06/21/2023 17:39:47 [INFO|DP=0|PP=0|TP=0]: iteration: 1775 / 150000 | consumed_samples: 113600 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.778962E+05 | tokens_per_sec_per_gpu: 9.029628E+03 | global_batch_size: 64 | lm_loss: 1.086818E+01 | lr: 2.662E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 6550612.000 [ip-26-0-154-121:0]:06/21/2023 17:39:48 [INFO|DP=0|PP=0|TP=0]: iteration: 1776 / 150000 | consumed_samples: 113664 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789791E+05 | tokens_per_sec_per_gpu: 9.046548E+03 | global_batch_size: 64 | lm_loss: 1.086758E+01 | lr: 2.664E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 11264460.000 [ip-26-0-154-121:0]:06/21/2023 17:39:49 [INFO|DP=0|PP=0|TP=0]: iteration: 1777 / 150000 | consumed_samples: 113728 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789525E+05 | tokens_per_sec_per_gpu: 9.046133E+03 | global_batch_size: 64 | lm_loss: 1.085864E+01 | lr: 2.665E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 12835924.000 [ip-26-0-154-121:0]:06/21/2023 17:39:50 [INFO|DP=0|PP=0|TP=0]: iteration: 1778 / 150000 | consumed_samples: 113792 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788791E+05 | tokens_per_sec_per_gpu: 9.044986E+03 | global_batch_size: 64 | lm_loss: 1.086366E+01 | lr: 2.667E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 8464727.000 [ip-26-0-154-121:0]:06/21/2023 17:39:51 [INFO|DP=0|PP=0|TP=0]: iteration: 1779 / 150000 | consumed_samples: 113856 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787112E+05 | tokens_per_sec_per_gpu: 9.042362E+03 | global_batch_size: 64 | lm_loss: 1.086580E+01 | lr: 2.668E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3623387.250 [ip-26-0-154-121:0]:06/21/2023 17:39:51 [INFO|DP=0|PP=0|TP=0]: iteration: 1780 / 150000 | consumed_samples: 113920 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787899E+05 | tokens_per_sec_per_gpu: 9.043593E+03 | global_batch_size: 64 | lm_loss: 1.086502E+01 | lr: 2.670E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 2591457.750 [ip-26-0-154-121:0]:06/21/2023 17:39:52 [INFO|DP=0|PP=0|TP=0]: iteration: 1781 / 150000 | consumed_samples: 113984 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787350E+05 | tokens_per_sec_per_gpu: 9.042734E+03 | global_batch_size: 64 | lm_loss: 1.086713E+01 | lr: 2.672E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 5170438.500 [ip-26-0-154-121:0]:06/21/2023 17:39:53 [INFO|DP=0|PP=0|TP=0]: iteration: 1782 / 150000 | consumed_samples: 114048 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786556E+05 | tokens_per_sec_per_gpu: 9.041494E+03 | global_batch_size: 64 | lm_loss: 1.086800E+01 | lr: 2.673E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 13484783.000 [ip-26-0-154-121:0]:06/21/2023 17:39:54 [INFO|DP=0|PP=0|TP=0]: iteration: 1783 / 150000 | consumed_samples: 114112 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790530E+05 | tokens_per_sec_per_gpu: 9.047703E+03 | global_batch_size: 64 | lm_loss: 1.086388E+01 | lr: 2.674E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 19581286.000 [ip-26-0-154-121:0]:06/21/2023 17:39:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1784 / 150000 | consumed_samples: 114176 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784303E+05 | tokens_per_sec_per_gpu: 9.037974E+03 | global_batch_size: 64 | lm_loss: 1.086214E+01 | lr: 2.676E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 16528177.000 [ip-26-0-154-121:0]:06/21/2023 17:39:56 [INFO|DP=0|PP=0|TP=0]: iteration: 1785 / 150000 | consumed_samples: 114240 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787788E+05 | tokens_per_sec_per_gpu: 9.043419E+03 | global_batch_size: 64 | lm_loss: 1.086350E+01 | lr: 2.677E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 12447349.000 [ip-26-0-154-121:0]:06/21/2023 17:39:57 [INFO|DP=0|PP=0|TP=0]: iteration: 1786 / 150000 | consumed_samples: 114304 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786670E+05 | tokens_per_sec_per_gpu: 9.041672E+03 | global_batch_size: 64 | lm_loss: 1.086718E+01 | lr: 2.679E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 6959146.500 [ip-26-0-154-121:0]:06/21/2023 17:39:58 [INFO|DP=0|PP=0|TP=0]: iteration: 1787 / 150000 | consumed_samples: 114368 | elapsed_time_per_iteration_ms: 908.8 | tokens_per_sec: 5.768770E+05 | tokens_per_sec_per_gpu: 9.013703E+03 | global_batch_size: 64 | lm_loss: 1.086408E+01 | lr: 2.680E-04 | model_tflops_per_gpu: 104.12 | hardware_tflops_per_gpu: 104.12 | grad_norm: 3034621.000 [ip-26-0-154-121:0]:06/21/2023 17:39:59 [INFO|DP=0|PP=0|TP=0]: iteration: 1788 / 150000 | consumed_samples: 114432 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785057E+05 | tokens_per_sec_per_gpu: 9.039151E+03 | global_batch_size: 64 | lm_loss: 1.086363E+01 | lr: 2.682E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 2532789.750 [ip-26-0-154-121:0]:06/21/2023 17:40:00 [INFO|DP=0|PP=0|TP=0]: iteration: 1789 / 150000 | consumed_samples: 114496 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787110E+05 | tokens_per_sec_per_gpu: 9.042360E+03 | global_batch_size: 64 | lm_loss: 1.086442E+01 | lr: 2.683E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 19884478.000 [ip-26-0-154-121:0]:06/21/2023 17:40:01 [INFO|DP=0|PP=0|TP=0]: iteration: 1790 / 150000 | consumed_samples: 114560 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787045E+05 | tokens_per_sec_per_gpu: 9.042258E+03 | global_batch_size: 64 | lm_loss: 1.086628E+01 | lr: 2.685E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 6914987.500 [ip-26-0-154-121:0]:06/21/2023 17:40:01 [INFO|DP=0|PP=0|TP=0]: iteration: 1791 / 150000 | consumed_samples: 114624 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787829E+05 | tokens_per_sec_per_gpu: 9.043483E+03 | global_batch_size: 64 | lm_loss: 1.086608E+01 | lr: 2.686E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 5622180.000 [ip-26-0-154-121:0]:06/21/2023 17:40:02 [INFO|DP=0|PP=0|TP=0]: iteration: 1792 / 150000 | consumed_samples: 114688 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781827E+05 | tokens_per_sec_per_gpu: 9.034105E+03 | global_batch_size: 64 | lm_loss: 1.086563E+01 | lr: 2.688E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 11488510.000 [ip-26-0-154-121:0]:06/21/2023 17:40:03 [INFO|DP=0|PP=0|TP=0]: iteration: 1793 / 150000 | consumed_samples: 114752 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787755E+05 | tokens_per_sec_per_gpu: 9.043367E+03 | global_batch_size: 64 | lm_loss: 1.086454E+01 | lr: 2.689E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 14413511.000 [ip-26-0-154-121:0]:06/21/2023 17:40:04 [INFO|DP=0|PP=0|TP=0]: iteration: 1794 / 150000 | consumed_samples: 114816 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789934E+05 | tokens_per_sec_per_gpu: 9.046772E+03 | global_batch_size: 64 | lm_loss: 1.086435E+01 | lr: 2.691E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 6650702.000 [ip-26-0-154-121:0]:06/21/2023 17:40:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1795 / 150000 | consumed_samples: 114880 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782125E+05 | tokens_per_sec_per_gpu: 9.034571E+03 | global_batch_size: 64 | lm_loss: 1.086287E+01 | lr: 2.693E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 6969879.500 [ip-26-0-154-121:0]:06/21/2023 17:40:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1796 / 150000 | consumed_samples: 114944 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788777E+05 | tokens_per_sec_per_gpu: 9.044964E+03 | global_batch_size: 64 | lm_loss: 1.086130E+01 | lr: 2.694E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 8469457.000 [ip-26-0-154-121:0]:06/21/2023 17:40:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1797 / 150000 | consumed_samples: 115008 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786182E+05 | tokens_per_sec_per_gpu: 9.040909E+03 | global_batch_size: 64 | lm_loss: 1.086236E+01 | lr: 2.695E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 8054020.500 [ip-26-0-154-121:0]:06/21/2023 17:40:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1798 / 150000 | consumed_samples: 115072 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788434E+05 | tokens_per_sec_per_gpu: 9.044428E+03 | global_batch_size: 64 | lm_loss: 1.086306E+01 | lr: 2.697E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 5728985.000 [ip-26-0-154-121:0]:06/21/2023 17:40:09 [INFO|DP=0|PP=0|TP=0]: iteration: 1799 / 150000 | consumed_samples: 115136 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789419E+05 | tokens_per_sec_per_gpu: 9.045967E+03 | global_batch_size: 64 | lm_loss: 1.086011E+01 | lr: 2.698E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 2574826.250 [ip-26-0-154-121:0]:06/21/2023 17:40:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1800 / 150000 | consumed_samples: 115200 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783321E+05 | tokens_per_sec_per_gpu: 9.036439E+03 | global_batch_size: 64 | lm_loss: 1.086544E+01 | lr: 2.700E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 2415744.250 [ip-26-0-154-121:0]:06/21/2023 17:40:11 [INFO|DP=0|PP=0|TP=0]: iteration: 1801 / 150000 | consumed_samples: 115264 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782919E+05 | tokens_per_sec_per_gpu: 9.035811E+03 | global_batch_size: 64 | lm_loss: 1.086505E+01 | lr: 2.701E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 3191632.500 [ip-26-0-154-121:0]:06/21/2023 17:40:11 [INFO|DP=0|PP=0|TP=0]: iteration: 1802 / 150000 | consumed_samples: 115328 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785538E+05 | tokens_per_sec_per_gpu: 9.039903E+03 | global_batch_size: 64 | lm_loss: 1.086420E+01 | lr: 2.703E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3367500.750 [ip-26-0-154-121:0]:06/21/2023 17:40:12 [INFO|DP=0|PP=0|TP=0]: iteration: 1803 / 150000 | consumed_samples: 115392 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790681E+05 | tokens_per_sec_per_gpu: 9.047939E+03 | global_batch_size: 64 | lm_loss: 1.086231E+01 | lr: 2.704E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 2290899.500 [ip-26-0-154-121:0]:06/21/2023 17:40:13 [INFO|DP=0|PP=0|TP=0]: iteration: 1804 / 150000 | consumed_samples: 115456 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787357E+05 | tokens_per_sec_per_gpu: 9.042746E+03 | global_batch_size: 64 | lm_loss: 1.086144E+01 | lr: 2.706E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 2197336.500 [ip-26-0-154-121:0]:06/21/2023 17:40:14 [INFO|DP=0|PP=0|TP=0]: iteration: 1805 / 150000 | consumed_samples: 115520 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786090E+05 | tokens_per_sec_per_gpu: 9.040766E+03 | global_batch_size: 64 | lm_loss: 1.085473E+01 | lr: 2.707E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4068961.750 [ip-26-0-154-121:0]:06/21/2023 17:40:15 [INFO|DP=0|PP=0|TP=0]: iteration: 1806 / 150000 | consumed_samples: 115584 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783543E+05 | tokens_per_sec_per_gpu: 9.036786E+03 | global_batch_size: 64 | lm_loss: 1.086015E+01 | lr: 2.709E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 13355561.000 [ip-26-0-154-121:0]:06/21/2023 17:40:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1807 / 150000 | consumed_samples: 115648 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785611E+05 | tokens_per_sec_per_gpu: 9.040017E+03 | global_batch_size: 64 | lm_loss: 1.085907E+01 | lr: 2.710E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 36887556.000 [ip-26-0-154-121:0]:06/21/2023 17:40:17 [INFO|DP=0|PP=0|TP=0]: iteration: 1808 / 150000 | consumed_samples: 115712 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784195E+05 | tokens_per_sec_per_gpu: 9.037805E+03 | global_batch_size: 64 | lm_loss: 1.086128E+01 | lr: 2.712E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 41114364.000 [ip-26-0-154-121:0]:06/21/2023 17:40:18 [INFO|DP=0|PP=0|TP=0]: iteration: 1809 / 150000 | consumed_samples: 115776 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783566E+05 | tokens_per_sec_per_gpu: 9.036821E+03 | global_batch_size: 64 | lm_loss: 1.085977E+01 | lr: 2.713E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 33350326.000 [ip-26-0-154-121:0]:06/21/2023 17:40:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1810 / 150000 | consumed_samples: 115840 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787581E+05 | tokens_per_sec_per_gpu: 9.043095E+03 | global_batch_size: 64 | lm_loss: 1.086133E+01 | lr: 2.715E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 11434477.000 [ip-26-0-154-121:0]:06/21/2023 17:40:20 [INFO|DP=0|PP=0|TP=0]: iteration: 1811 / 150000 | consumed_samples: 115904 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786845E+05 | tokens_per_sec_per_gpu: 9.041946E+03 | global_batch_size: 64 | lm_loss: 1.086047E+01 | lr: 2.717E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 4943108.000 [ip-26-0-154-121:0]:06/21/2023 17:40:21 [INFO|DP=0|PP=0|TP=0]: iteration: 1812 / 150000 | consumed_samples: 115968 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781478E+05 | tokens_per_sec_per_gpu: 9.033559E+03 | global_batch_size: 64 | lm_loss: 1.086119E+01 | lr: 2.718E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 4110428.000 [ip-26-0-154-121:0]:06/21/2023 17:40:21 [INFO|DP=0|PP=0|TP=0]: iteration: 1813 / 150000 | consumed_samples: 116032 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787564E+05 | tokens_per_sec_per_gpu: 9.043069E+03 | global_batch_size: 64 | lm_loss: 1.085977E+01 | lr: 2.719E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 7235521.000 [ip-26-0-154-121:0]:06/21/2023 17:40:22 [INFO|DP=0|PP=0|TP=0]: iteration: 1814 / 150000 | consumed_samples: 116096 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788992E+05 | tokens_per_sec_per_gpu: 9.045300E+03 | global_batch_size: 64 | lm_loss: 1.086045E+01 | lr: 2.721E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 28128862.000 [ip-26-0-154-121:0]:06/21/2023 17:40:23 [INFO|DP=0|PP=0|TP=0]: iteration: 1815 / 150000 | consumed_samples: 116160 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787980E+05 | tokens_per_sec_per_gpu: 9.043719E+03 | global_batch_size: 64 | lm_loss: 1.085904E+01 | lr: 2.722E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 45823324.000 [ip-26-0-154-121:0]:06/21/2023 17:40:24 [INFO|DP=0|PP=0|TP=0]: iteration: 1816 / 150000 | consumed_samples: 116224 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785930E+05 | tokens_per_sec_per_gpu: 9.040516E+03 | global_batch_size: 64 | lm_loss: 1.085822E+01 | lr: 2.724E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 25649706.000 [ip-26-0-154-121:0]:06/21/2023 17:40:25 [INFO|DP=0|PP=0|TP=0]: iteration: 1817 / 150000 | consumed_samples: 116288 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788224E+05 | tokens_per_sec_per_gpu: 9.044100E+03 | global_batch_size: 64 | lm_loss: 1.085771E+01 | lr: 2.725E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 5175446.000 [ip-26-0-154-121:0]:06/21/2023 17:40:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1818 / 150000 | consumed_samples: 116352 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784615E+05 | tokens_per_sec_per_gpu: 9.038461E+03 | global_batch_size: 64 | lm_loss: 1.085457E+01 | lr: 2.727E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 2854045.000 [ip-26-0-154-121:0]:06/21/2023 17:40:27 [INFO|DP=0|PP=0|TP=0]: iteration: 1819 / 150000 | consumed_samples: 116416 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783864E+05 | tokens_per_sec_per_gpu: 9.037287E+03 | global_batch_size: 64 | lm_loss: 1.085240E+01 | lr: 2.728E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3257532.250 [ip-26-0-154-121:0]:06/21/2023 17:40:28 [INFO|DP=0|PP=0|TP=0]: iteration: 1820 / 150000 | consumed_samples: 116480 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785087E+05 | tokens_per_sec_per_gpu: 9.039199E+03 | global_batch_size: 64 | lm_loss: 1.085986E+01 | lr: 2.730E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 24919604.000 [ip-26-0-154-121:0]:06/21/2023 17:40:29 [INFO|DP=0|PP=0|TP=0]: iteration: 1821 / 150000 | consumed_samples: 116544 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789631E+05 | tokens_per_sec_per_gpu: 9.046298E+03 | global_batch_size: 64 | lm_loss: 1.086197E+01 | lr: 2.731E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 13298915.000 [ip-26-0-154-121:0]:06/21/2023 17:40:30 [INFO|DP=0|PP=0|TP=0]: iteration: 1822 / 150000 | consumed_samples: 116608 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782370E+05 | tokens_per_sec_per_gpu: 9.034953E+03 | global_batch_size: 64 | lm_loss: 1.085989E+01 | lr: 2.733E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 11089332.000 [ip-26-0-154-121:0]:06/21/2023 17:40:30 [INFO|DP=0|PP=0|TP=0]: iteration: 1823 / 150000 | consumed_samples: 116672 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788629E+05 | tokens_per_sec_per_gpu: 9.044733E+03 | global_batch_size: 64 | lm_loss: 1.086020E+01 | lr: 2.734E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 17516890.000 [ip-26-0-154-121:0]:06/21/2023 17:40:31 [INFO|DP=0|PP=0|TP=0]: iteration: 1824 / 150000 | consumed_samples: 116736 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785649E+05 | tokens_per_sec_per_gpu: 9.040076E+03 | global_batch_size: 64 | lm_loss: 1.086032E+01 | lr: 2.736E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 17106202.000 [ip-26-0-154-121:0]:06/21/2023 17:40:32 [INFO|DP=0|PP=0|TP=0]: iteration: 1825 / 150000 | consumed_samples: 116800 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783680E+05 | tokens_per_sec_per_gpu: 9.036999E+03 | global_batch_size: 64 | lm_loss: 1.085784E+01 | lr: 2.738E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 9533163.000 [ip-26-0-154-121:0]:06/21/2023 17:40:33 [INFO|DP=0|PP=0|TP=0]: iteration: 1826 / 150000 | consumed_samples: 116864 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787241E+05 | tokens_per_sec_per_gpu: 9.042565E+03 | global_batch_size: 64 | lm_loss: 1.085829E+01 | lr: 2.739E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 4878858.500 [ip-26-0-154-121:0]:06/21/2023 17:40:34 [INFO|DP=0|PP=0|TP=0]: iteration: 1827 / 150000 | consumed_samples: 116928 | elapsed_time_per_iteration_ms: 905.2 | tokens_per_sec: 5.791764E+05 | tokens_per_sec_per_gpu: 9.049631E+03 | global_batch_size: 64 | lm_loss: 1.085779E+01 | lr: 2.740E-04 | model_tflops_per_gpu: 104.53 | hardware_tflops_per_gpu: 104.53 | grad_norm: 3128722.250 [ip-26-0-154-121:0]:06/21/2023 17:40:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1828 / 150000 | consumed_samples: 116992 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787107E+05 | tokens_per_sec_per_gpu: 9.042355E+03 | global_batch_size: 64 | lm_loss: 1.085737E+01 | lr: 2.742E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 14871096.000 [ip-26-0-154-121:0]:06/21/2023 17:40:36 [INFO|DP=0|PP=0|TP=0]: iteration: 1829 / 150000 | consumed_samples: 117056 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786227E+05 | tokens_per_sec_per_gpu: 9.040980E+03 | global_batch_size: 64 | lm_loss: 1.085431E+01 | lr: 2.743E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 6533467.500 [ip-26-0-154-121:0]:06/21/2023 17:40:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1830 / 150000 | consumed_samples: 117120 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783114E+05 | tokens_per_sec_per_gpu: 9.036115E+03 | global_batch_size: 64 | lm_loss: 1.084995E+01 | lr: 2.745E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3742133.750 [ip-26-0-154-121:0]:06/21/2023 17:40:38 [INFO|DP=0|PP=0|TP=0]: iteration: 1831 / 150000 | consumed_samples: 117184 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785824E+05 | tokens_per_sec_per_gpu: 9.040350E+03 | global_batch_size: 64 | lm_loss: 1.085640E+01 | lr: 2.746E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4731066.000 [ip-26-0-154-121:0]:06/21/2023 17:40:39 [INFO|DP=0|PP=0|TP=0]: iteration: 1832 / 150000 | consumed_samples: 117248 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780002E+05 | tokens_per_sec_per_gpu: 9.031254E+03 | global_batch_size: 64 | lm_loss: 1.085683E+01 | lr: 2.748E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 11255805.000 [ip-26-0-154-121:0]:06/21/2023 17:40:40 [INFO|DP=0|PP=0|TP=0]: iteration: 1833 / 150000 | consumed_samples: 117312 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781931E+05 | tokens_per_sec_per_gpu: 9.034267E+03 | global_batch_size: 64 | lm_loss: 1.085609E+01 | lr: 2.749E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 16412816.000 [ip-26-0-154-121:0]:06/21/2023 17:40:40 [INFO|DP=0|PP=0|TP=0]: iteration: 1834 / 150000 | consumed_samples: 117376 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785032E+05 | tokens_per_sec_per_gpu: 9.039113E+03 | global_batch_size: 64 | lm_loss: 1.085837E+01 | lr: 2.751E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 4659444.000 [ip-26-0-154-121:0]:06/21/2023 17:40:41 [INFO|DP=0|PP=0|TP=0]: iteration: 1835 / 150000 | consumed_samples: 117440 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786676E+05 | tokens_per_sec_per_gpu: 9.041682E+03 | global_batch_size: 64 | lm_loss: 1.085687E+01 | lr: 2.752E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3571983.750 [ip-26-0-154-121:0]:06/21/2023 17:40:42 [INFO|DP=0|PP=0|TP=0]: iteration: 1836 / 150000 | consumed_samples: 117504 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784561E+05 | tokens_per_sec_per_gpu: 9.038376E+03 | global_batch_size: 64 | lm_loss: 1.085172E+01 | lr: 2.754E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3062787.250 [ip-26-0-154-121:0]:06/21/2023 17:40:43 [INFO|DP=0|PP=0|TP=0]: iteration: 1837 / 150000 | consumed_samples: 117568 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781081E+05 | tokens_per_sec_per_gpu: 9.032939E+03 | global_batch_size: 64 | lm_loss: 1.085877E+01 | lr: 2.755E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 6703079.000 [ip-26-0-154-121:0]:06/21/2023 17:40:44 [INFO|DP=0|PP=0|TP=0]: iteration: 1838 / 150000 | consumed_samples: 117632 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784653E+05 | tokens_per_sec_per_gpu: 9.038521E+03 | global_batch_size: 64 | lm_loss: 1.085751E+01 | lr: 2.757E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 11692972.000 [ip-26-0-154-121:0]:06/21/2023 17:40:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1839 / 150000 | consumed_samples: 117696 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787790E+05 | tokens_per_sec_per_gpu: 9.043421E+03 | global_batch_size: 64 | lm_loss: 1.085739E+01 | lr: 2.759E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 13346406.000 [ip-26-0-154-121:0]:06/21/2023 17:40:46 [INFO|DP=0|PP=0|TP=0]: iteration: 1840 / 150000 | consumed_samples: 117760 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787779E+05 | tokens_per_sec_per_gpu: 9.043405E+03 | global_batch_size: 64 | lm_loss: 1.085634E+01 | lr: 2.760E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 7937479.000 [ip-26-0-154-121:0]:06/21/2023 17:40:47 [INFO|DP=0|PP=0|TP=0]: iteration: 1841 / 150000 | consumed_samples: 117824 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786692E+05 | tokens_per_sec_per_gpu: 9.041706E+03 | global_batch_size: 64 | lm_loss: 1.085764E+01 | lr: 2.761E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 6148996.000 [ip-26-0-154-121:0]:06/21/2023 17:40:48 [INFO|DP=0|PP=0|TP=0]: iteration: 1842 / 150000 | consumed_samples: 117888 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789423E+05 | tokens_per_sec_per_gpu: 9.045974E+03 | global_batch_size: 64 | lm_loss: 1.085456E+01 | lr: 2.763E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 4444954.000 [ip-26-0-154-121:0]:06/21/2023 17:40:49 [INFO|DP=0|PP=0|TP=0]: iteration: 1843 / 150000 | consumed_samples: 117952 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787482E+05 | tokens_per_sec_per_gpu: 9.042941E+03 | global_batch_size: 64 | lm_loss: 1.085541E+01 | lr: 2.764E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 5320864.500 [ip-26-0-154-121:0]:06/21/2023 17:40:50 [INFO|DP=0|PP=0|TP=0]: iteration: 1844 / 150000 | consumed_samples: 118016 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787174E+05 | tokens_per_sec_per_gpu: 9.042460E+03 | global_batch_size: 64 | lm_loss: 1.085446E+01 | lr: 2.766E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 9315230.000 [ip-26-0-154-121:0]:06/21/2023 17:40:50 [INFO|DP=0|PP=0|TP=0]: iteration: 1845 / 150000 | consumed_samples: 118080 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787301E+05 | tokens_per_sec_per_gpu: 9.042658E+03 | global_batch_size: 64 | lm_loss: 1.085536E+01 | lr: 2.767E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 14586727.000 [ip-26-0-154-121:0]:06/21/2023 17:40:51 [INFO|DP=0|PP=0|TP=0]: iteration: 1846 / 150000 | consumed_samples: 118144 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785828E+05 | tokens_per_sec_per_gpu: 9.040357E+03 | global_batch_size: 64 | lm_loss: 1.084898E+01 | lr: 2.769E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 17263248.000 [ip-26-0-154-121:0]:06/21/2023 17:40:52 [INFO|DP=0|PP=0|TP=0]: iteration: 1847 / 150000 | consumed_samples: 118208 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.783975E+05 | tokens_per_sec_per_gpu: 9.037461E+03 | global_batch_size: 64 | lm_loss: 1.085483E+01 | lr: 2.770E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 12531168.000 [ip-26-0-154-121:0]:06/21/2023 17:40:53 [INFO|DP=0|PP=0|TP=0]: iteration: 1848 / 150000 | consumed_samples: 118272 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789128E+05 | tokens_per_sec_per_gpu: 9.045512E+03 | global_batch_size: 64 | lm_loss: 1.082925E+01 | lr: 2.772E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 4426290.000 [ip-26-0-154-121:0]:06/21/2023 17:40:54 [INFO|DP=0|PP=0|TP=0]: iteration: 1849 / 150000 | consumed_samples: 118336 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787423E+05 | tokens_per_sec_per_gpu: 9.042848E+03 | global_batch_size: 64 | lm_loss: 1.085010E+01 | lr: 2.773E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 2394286.750 [ip-26-0-154-121:0]:06/21/2023 17:40:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1850 / 150000 | consumed_samples: 118400 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786104E+05 | tokens_per_sec_per_gpu: 9.040787E+03 | global_batch_size: 64 | lm_loss: 1.085623E+01 | lr: 2.775E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3817467.250 [ip-26-0-154-121:0]:06/21/2023 17:40:56 [INFO|DP=0|PP=0|TP=0]: iteration: 1851 / 150000 | consumed_samples: 118464 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784659E+05 | tokens_per_sec_per_gpu: 9.038530E+03 | global_batch_size: 64 | lm_loss: 1.085625E+01 | lr: 2.776E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 15455661.000 [ip-26-0-154-121:0]:06/21/2023 17:40:57 [INFO|DP=0|PP=0|TP=0]: iteration: 1852 / 150000 | consumed_samples: 118528 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785358E+05 | tokens_per_sec_per_gpu: 9.039622E+03 | global_batch_size: 64 | lm_loss: 1.085653E+01 | lr: 2.778E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 30409996.000 [ip-26-0-154-121:0]:06/21/2023 17:40:58 [INFO|DP=0|PP=0|TP=0]: iteration: 1853 / 150000 | consumed_samples: 118592 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787520E+05 | tokens_per_sec_per_gpu: 9.043000E+03 | global_batch_size: 64 | lm_loss: 1.085628E+01 | lr: 2.779E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 29878140.000 [ip-26-0-154-121:0]:06/21/2023 17:40:59 [INFO|DP=0|PP=0|TP=0]: iteration: 1854 / 150000 | consumed_samples: 118656 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786891E+05 | tokens_per_sec_per_gpu: 9.042017E+03 | global_batch_size: 64 | lm_loss: 1.085676E+01 | lr: 2.781E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 25821508.000 [ip-26-0-154-121:0]:06/21/2023 17:41:00 [INFO|DP=0|PP=0|TP=0]: iteration: 1855 / 150000 | consumed_samples: 118720 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789704E+05 | tokens_per_sec_per_gpu: 9.046412E+03 | global_batch_size: 64 | lm_loss: 1.085541E+01 | lr: 2.783E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 11997894.000 [ip-26-0-154-121:0]:06/21/2023 17:41:00 [INFO|DP=0|PP=0|TP=0]: iteration: 1856 / 150000 | consumed_samples: 118784 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785309E+05 | tokens_per_sec_per_gpu: 9.039546E+03 | global_batch_size: 64 | lm_loss: 1.085482E+01 | lr: 2.784E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 4548322.500 [ip-26-0-154-121:0]:06/21/2023 17:41:01 [INFO|DP=0|PP=0|TP=0]: iteration: 1857 / 150000 | consumed_samples: 118848 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786227E+05 | tokens_per_sec_per_gpu: 9.040980E+03 | global_batch_size: 64 | lm_loss: 1.085236E+01 | lr: 2.785E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 2435185.250 [ip-26-0-154-121:0]:06/21/2023 17:41:02 [INFO|DP=0|PP=0|TP=0]: iteration: 1858 / 150000 | consumed_samples: 118912 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786809E+05 | tokens_per_sec_per_gpu: 9.041889E+03 | global_batch_size: 64 | lm_loss: 1.085000E+01 | lr: 2.787E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2893207.500 [ip-26-0-154-121:0]:06/21/2023 17:41:03 [INFO|DP=0|PP=0|TP=0]: iteration: 1859 / 150000 | consumed_samples: 118976 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783397E+05 | tokens_per_sec_per_gpu: 9.036557E+03 | global_batch_size: 64 | lm_loss: 1.085069E+01 | lr: 2.788E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 12935161.000 [ip-26-0-154-121:0]:06/21/2023 17:41:04 [INFO|DP=0|PP=0|TP=0]: iteration: 1860 / 150000 | consumed_samples: 119040 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785407E+05 | tokens_per_sec_per_gpu: 9.039698E+03 | global_batch_size: 64 | lm_loss: 1.084906E+01 | lr: 2.790E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 30771444.000 [ip-26-0-154-121:0]:06/21/2023 17:41:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1861 / 150000 | consumed_samples: 119104 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785606E+05 | tokens_per_sec_per_gpu: 9.040010E+03 | global_batch_size: 64 | lm_loss: 1.084994E+01 | lr: 2.791E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 10510933.000 [ip-26-0-154-121:0]:06/21/2023 17:41:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1862 / 150000 | consumed_samples: 119168 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784320E+05 | tokens_per_sec_per_gpu: 9.038000E+03 | global_batch_size: 64 | lm_loss: 1.085017E+01 | lr: 2.793E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 15468236.000 [ip-26-0-154-121:0]:06/21/2023 17:41:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1863 / 150000 | consumed_samples: 119232 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788334E+05 | tokens_per_sec_per_gpu: 9.044271E+03 | global_batch_size: 64 | lm_loss: 1.084965E+01 | lr: 2.794E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 17386924.000 [ip-26-0-154-121:0]:06/21/2023 17:41:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1864 / 150000 | consumed_samples: 119296 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788110E+05 | tokens_per_sec_per_gpu: 9.043921E+03 | global_batch_size: 64 | lm_loss: 1.084687E+01 | lr: 2.796E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 12910005.000 [ip-26-0-154-121:0]:06/21/2023 17:41:09 [INFO|DP=0|PP=0|TP=0]: iteration: 1865 / 150000 | consumed_samples: 119360 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785484E+05 | tokens_per_sec_per_gpu: 9.039819E+03 | global_batch_size: 64 | lm_loss: 1.084986E+01 | lr: 2.797E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 7939160.500 [ip-26-0-154-121:0]:06/21/2023 17:41:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1866 / 150000 | consumed_samples: 119424 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786992E+05 | tokens_per_sec_per_gpu: 9.042174E+03 | global_batch_size: 64 | lm_loss: 1.084998E+01 | lr: 2.799E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 4582887.500 [ip-26-0-154-121:0]:06/21/2023 17:41:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1867 / 150000 | consumed_samples: 119488 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786373E+05 | tokens_per_sec_per_gpu: 9.041208E+03 | global_batch_size: 64 | lm_loss: 1.084795E+01 | lr: 2.800E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2703348.000 [ip-26-0-154-121:0]:06/21/2023 17:41:11 [INFO|DP=0|PP=0|TP=0]: iteration: 1868 / 150000 | consumed_samples: 119552 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789207E+05 | tokens_per_sec_per_gpu: 9.045636E+03 | global_batch_size: 64 | lm_loss: 1.085443E+01 | lr: 2.802E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 2680128.500 [ip-26-0-154-121:0]:06/21/2023 17:41:12 [INFO|DP=0|PP=0|TP=0]: iteration: 1869 / 150000 | consumed_samples: 119616 | elapsed_time_per_iteration_ms: 904.9 | tokens_per_sec: 5.794059E+05 | tokens_per_sec_per_gpu: 9.053217E+03 | global_batch_size: 64 | lm_loss: 1.084965E+01 | lr: 2.804E-04 | model_tflops_per_gpu: 104.57 | hardware_tflops_per_gpu: 104.57 | grad_norm: 14972053.000 [ip-26-0-154-121:0]:06/21/2023 17:41:13 [INFO|DP=0|PP=0|TP=0]: iteration: 1870 / 150000 | consumed_samples: 119680 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782869E+05 | tokens_per_sec_per_gpu: 9.035733E+03 | global_batch_size: 64 | lm_loss: 1.085186E+01 | lr: 2.805E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 11627199.000 [ip-26-0-154-121:0]:06/21/2023 17:41:14 [INFO|DP=0|PP=0|TP=0]: iteration: 1871 / 150000 | consumed_samples: 119744 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786873E+05 | tokens_per_sec_per_gpu: 9.041989E+03 | global_batch_size: 64 | lm_loss: 1.085382E+01 | lr: 2.806E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 5287129.500 [ip-26-0-154-121:0]:06/21/2023 17:41:15 [INFO|DP=0|PP=0|TP=0]: iteration: 1872 / 150000 | consumed_samples: 119808 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787467E+05 | tokens_per_sec_per_gpu: 9.042917E+03 | global_batch_size: 64 | lm_loss: 1.085396E+01 | lr: 2.808E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 5789446.000 [ip-26-0-154-121:0]:06/21/2023 17:41:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1873 / 150000 | consumed_samples: 119872 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789047E+05 | tokens_per_sec_per_gpu: 9.045386E+03 | global_batch_size: 64 | lm_loss: 1.085258E+01 | lr: 2.809E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 9977467.000 [ip-26-0-154-121:0]:06/21/2023 17:41:17 [INFO|DP=0|PP=0|TP=0]: iteration: 1874 / 150000 | consumed_samples: 119936 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788523E+05 | tokens_per_sec_per_gpu: 9.044567E+03 | global_batch_size: 64 | lm_loss: 1.085090E+01 | lr: 2.811E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 15992692.000 [ip-26-0-154-121:0]:06/21/2023 17:41:18 [INFO|DP=0|PP=0|TP=0]: iteration: 1875 / 150000 | consumed_samples: 120000 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787153E+05 | tokens_per_sec_per_gpu: 9.042427E+03 | global_batch_size: 64 | lm_loss: 1.085209E+01 | lr: 2.812E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 13965611.000 [ip-26-0-154-121:0]:06/21/2023 17:41:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1876 / 150000 | consumed_samples: 120064 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786958E+05 | tokens_per_sec_per_gpu: 9.042122E+03 | global_batch_size: 64 | lm_loss: 1.084789E+01 | lr: 2.814E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 7921900.000 [ip-26-0-154-121:0]:06/21/2023 17:41:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1877 / 150000 | consumed_samples: 120128 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787590E+05 | tokens_per_sec_per_gpu: 9.043110E+03 | global_batch_size: 64 | lm_loss: 1.084966E+01 | lr: 2.815E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 4345379.000 [ip-26-0-154-121:0]:06/21/2023 17:41:20 [INFO|DP=0|PP=0|TP=0]: iteration: 1878 / 150000 | consumed_samples: 120192 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786090E+05 | tokens_per_sec_per_gpu: 9.040766E+03 | global_batch_size: 64 | lm_loss: 1.085079E+01 | lr: 2.817E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3044700.750 [ip-26-0-154-121:0]:06/21/2023 17:41:21 [INFO|DP=0|PP=0|TP=0]: iteration: 1879 / 150000 | consumed_samples: 120256 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786719E+05 | tokens_per_sec_per_gpu: 9.041749E+03 | global_batch_size: 64 | lm_loss: 1.085052E+01 | lr: 2.818E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2599896.500 [ip-26-0-154-121:0]:06/21/2023 17:41:22 [INFO|DP=0|PP=0|TP=0]: iteration: 1880 / 150000 | consumed_samples: 120320 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787275E+05 | tokens_per_sec_per_gpu: 9.042617E+03 | global_batch_size: 64 | lm_loss: 1.085195E+01 | lr: 2.820E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 2329189.250 [ip-26-0-154-121:0]:06/21/2023 17:41:23 [INFO|DP=0|PP=0|TP=0]: iteration: 1881 / 150000 | consumed_samples: 120384 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785889E+05 | tokens_per_sec_per_gpu: 9.040452E+03 | global_batch_size: 64 | lm_loss: 1.085122E+01 | lr: 2.821E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 2087813.000 [ip-26-0-154-121:0]:06/21/2023 17:41:24 [INFO|DP=0|PP=0|TP=0]: iteration: 1882 / 150000 | consumed_samples: 120448 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787909E+05 | tokens_per_sec_per_gpu: 9.043607E+03 | global_batch_size: 64 | lm_loss: 1.084879E+01 | lr: 2.823E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 2336227.750 [ip-26-0-154-121:0]:06/21/2023 17:41:25 [INFO|DP=0|PP=0|TP=0]: iteration: 1883 / 150000 | consumed_samples: 120512 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786457E+05 | tokens_per_sec_per_gpu: 9.041339E+03 | global_batch_size: 64 | lm_loss: 1.084876E+01 | lr: 2.824E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2623440.250 [ip-26-0-154-121:0]:06/21/2023 17:41:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1884 / 150000 | consumed_samples: 120576 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787078E+05 | tokens_per_sec_per_gpu: 9.042310E+03 | global_batch_size: 64 | lm_loss: 1.084906E+01 | lr: 2.826E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 2716350.000 [ip-26-0-154-121:0]:06/21/2023 17:41:27 [INFO|DP=0|PP=0|TP=0]: iteration: 1885 / 150000 | consumed_samples: 120640 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787517E+05 | tokens_per_sec_per_gpu: 9.042995E+03 | global_batch_size: 64 | lm_loss: 1.084926E+01 | lr: 2.828E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 2322041.500 [ip-26-0-154-121:0]:06/21/2023 17:41:28 [INFO|DP=0|PP=0|TP=0]: iteration: 1886 / 150000 | consumed_samples: 120704 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787403E+05 | tokens_per_sec_per_gpu: 9.042817E+03 | global_batch_size: 64 | lm_loss: 1.085071E+01 | lr: 2.829E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 2020017.125 [ip-26-0-154-121:0]:06/21/2023 17:41:29 [INFO|DP=0|PP=0|TP=0]: iteration: 1887 / 150000 | consumed_samples: 120768 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783559E+05 | tokens_per_sec_per_gpu: 9.036812E+03 | global_batch_size: 64 | lm_loss: 1.084926E+01 | lr: 2.830E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1950567.250 [ip-26-0-154-121:0]:06/21/2023 17:41:29 [INFO|DP=0|PP=0|TP=0]: iteration: 1888 / 150000 | consumed_samples: 120832 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786718E+05 | tokens_per_sec_per_gpu: 9.041746E+03 | global_batch_size: 64 | lm_loss: 1.084845E+01 | lr: 2.832E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1918064.000 [ip-26-0-154-121:0]:06/21/2023 17:41:30 [INFO|DP=0|PP=0|TP=0]: iteration: 1889 / 150000 | consumed_samples: 120896 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786258E+05 | tokens_per_sec_per_gpu: 9.041028E+03 | global_batch_size: 64 | lm_loss: 1.084938E+01 | lr: 2.833E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1902887.500 [ip-26-0-154-121:0]:06/21/2023 17:41:31 [INFO|DP=0|PP=0|TP=0]: iteration: 1890 / 150000 | consumed_samples: 120960 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785968E+05 | tokens_per_sec_per_gpu: 9.040576E+03 | global_batch_size: 64 | lm_loss: 1.084753E+01 | lr: 2.835E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1912509.875 [ip-26-0-154-121:0]:06/21/2023 17:41:32 [INFO|DP=0|PP=0|TP=0]: iteration: 1891 / 150000 | consumed_samples: 121024 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788072E+05 | tokens_per_sec_per_gpu: 9.043862E+03 | global_batch_size: 64 | lm_loss: 1.084923E+01 | lr: 2.836E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1944467.375 [ip-26-0-154-121:0]:06/21/2023 17:41:33 [INFO|DP=0|PP=0|TP=0]: iteration: 1892 / 150000 | consumed_samples: 121088 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785367E+05 | tokens_per_sec_per_gpu: 9.039636E+03 | global_batch_size: 64 | lm_loss: 1.084907E+01 | lr: 2.838E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1954058.000 [ip-26-0-154-121:0]:06/21/2023 17:41:34 [INFO|DP=0|PP=0|TP=0]: iteration: 1893 / 150000 | consumed_samples: 121152 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786261E+05 | tokens_per_sec_per_gpu: 9.041032E+03 | global_batch_size: 64 | lm_loss: 1.084917E+01 | lr: 2.839E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1912910.375 [ip-26-0-154-121:0]:06/21/2023 17:41:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1894 / 150000 | consumed_samples: 121216 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785093E+05 | tokens_per_sec_per_gpu: 9.039208E+03 | global_batch_size: 64 | lm_loss: 1.084616E+01 | lr: 2.841E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1859073.875 [ip-26-0-154-121:0]:06/21/2023 17:41:36 [INFO|DP=0|PP=0|TP=0]: iteration: 1895 / 150000 | consumed_samples: 121280 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781268E+05 | tokens_per_sec_per_gpu: 9.033231E+03 | global_batch_size: 64 | lm_loss: 1.084506E+01 | lr: 2.842E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 1823396.500 [ip-26-0-154-121:0]:06/21/2023 17:41:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1896 / 150000 | consumed_samples: 121344 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784554E+05 | tokens_per_sec_per_gpu: 9.038366E+03 | global_batch_size: 64 | lm_loss: 1.084438E+01 | lr: 2.844E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1796469.875 [ip-26-0-154-121:0]:06/21/2023 17:41:38 [INFO|DP=0|PP=0|TP=0]: iteration: 1897 / 150000 | consumed_samples: 121408 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788695E+05 | tokens_per_sec_per_gpu: 9.044836E+03 | global_batch_size: 64 | lm_loss: 1.084803E+01 | lr: 2.845E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1784389.250 [ip-26-0-154-121:0]:06/21/2023 17:41:39 [INFO|DP=0|PP=0|TP=0]: iteration: 1898 / 150000 | consumed_samples: 121472 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787351E+05 | tokens_per_sec_per_gpu: 9.042736E+03 | global_batch_size: 64 | lm_loss: 1.084660E+01 | lr: 2.847E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1785442.500 [ip-26-0-154-121:0]:06/21/2023 17:41:39 [INFO|DP=0|PP=0|TP=0]: iteration: 1899 / 150000 | consumed_samples: 121536 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788000E+05 | tokens_per_sec_per_gpu: 9.043750E+03 | global_batch_size: 64 | lm_loss: 1.084851E+01 | lr: 2.849E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1793064.625 [ip-26-0-154-121:0]:06/21/2023 17:41:40 [INFO|DP=0|PP=0|TP=0]: iteration: 1900 / 150000 | consumed_samples: 121600 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787371E+05 | tokens_per_sec_per_gpu: 9.042767E+03 | global_batch_size: 64 | lm_loss: 1.084620E+01 | lr: 2.850E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1802378.875 [ip-26-0-154-121:0]:06/21/2023 17:41:41 [INFO|DP=0|PP=0|TP=0]: iteration: 1901 / 150000 | consumed_samples: 121664 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788235E+05 | tokens_per_sec_per_gpu: 9.044117E+03 | global_batch_size: 64 | lm_loss: 1.084617E+01 | lr: 2.851E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1794713.250 [ip-26-0-154-121:0]:06/21/2023 17:41:42 [INFO|DP=0|PP=0|TP=0]: iteration: 1902 / 150000 | consumed_samples: 121728 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786157E+05 | tokens_per_sec_per_gpu: 9.040871E+03 | global_batch_size: 64 | lm_loss: 1.084441E+01 | lr: 2.853E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1794768.250 [ip-26-0-154-121:0]:06/21/2023 17:41:43 [INFO|DP=0|PP=0|TP=0]: iteration: 1903 / 150000 | consumed_samples: 121792 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785401E+05 | tokens_per_sec_per_gpu: 9.039688E+03 | global_batch_size: 64 | lm_loss: 1.084492E+01 | lr: 2.854E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1792250.000 [ip-26-0-154-121:0]:06/21/2023 17:41:44 [INFO|DP=0|PP=0|TP=0]: iteration: 1904 / 150000 | consumed_samples: 121856 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788399E+05 | tokens_per_sec_per_gpu: 9.044374E+03 | global_batch_size: 64 | lm_loss: 1.084621E+01 | lr: 2.856E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1792936.250 [ip-26-0-154-121:0]:06/21/2023 17:41:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1905 / 150000 | consumed_samples: 121920 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788088E+05 | tokens_per_sec_per_gpu: 9.043888E+03 | global_batch_size: 64 | lm_loss: 1.084503E+01 | lr: 2.857E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1782934.875 [ip-26-0-154-121:0]:06/21/2023 17:41:46 [INFO|DP=0|PP=0|TP=0]: iteration: 1906 / 150000 | consumed_samples: 121984 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787825E+05 | tokens_per_sec_per_gpu: 9.043476E+03 | global_batch_size: 64 | lm_loss: 1.084556E+01 | lr: 2.859E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1770871.875 [ip-26-0-154-121:0]:06/21/2023 17:41:47 [INFO|DP=0|PP=0|TP=0]: iteration: 1907 / 150000 | consumed_samples: 122048 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783774E+05 | tokens_per_sec_per_gpu: 9.037147E+03 | global_batch_size: 64 | lm_loss: 1.084609E+01 | lr: 2.860E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1759868.125 [ip-26-0-154-121:0]:06/21/2023 17:41:48 [INFO|DP=0|PP=0|TP=0]: iteration: 1908 / 150000 | consumed_samples: 122112 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782224E+05 | tokens_per_sec_per_gpu: 9.034725E+03 | global_batch_size: 64 | lm_loss: 1.084692E+01 | lr: 2.862E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 1753892.875 [ip-26-0-154-121:0]:06/21/2023 17:41:49 [INFO|DP=0|PP=0|TP=0]: iteration: 1909 / 150000 | consumed_samples: 122176 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787787E+05 | tokens_per_sec_per_gpu: 9.043417E+03 | global_batch_size: 64 | lm_loss: 1.084546E+01 | lr: 2.863E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1753353.125 [ip-26-0-154-121:0]:06/21/2023 17:41:49 [INFO|DP=0|PP=0|TP=0]: iteration: 1910 / 150000 | consumed_samples: 122240 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786597E+05 | tokens_per_sec_per_gpu: 9.041558E+03 | global_batch_size: 64 | lm_loss: 1.084573E+01 | lr: 2.865E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1758767.250 [ip-26-0-154-121:0]:06/21/2023 17:41:50 [INFO|DP=0|PP=0|TP=0]: iteration: 1911 / 150000 | consumed_samples: 122304 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786976E+05 | tokens_per_sec_per_gpu: 9.042151E+03 | global_batch_size: 64 | lm_loss: 1.084431E+01 | lr: 2.866E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1763911.125 [ip-26-0-154-121:0]:06/21/2023 17:41:51 [INFO|DP=0|PP=0|TP=0]: iteration: 1912 / 150000 | consumed_samples: 122368 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787845E+05 | tokens_per_sec_per_gpu: 9.043507E+03 | global_batch_size: 64 | lm_loss: 1.084237E+01 | lr: 2.868E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1766695.000 [ip-26-0-154-121:0]:06/21/2023 17:41:52 [INFO|DP=0|PP=0|TP=0]: iteration: 1913 / 150000 | consumed_samples: 122432 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785051E+05 | tokens_per_sec_per_gpu: 9.039142E+03 | global_batch_size: 64 | lm_loss: 1.084434E+01 | lr: 2.870E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1768733.500 [ip-26-0-154-121:0]:06/21/2023 17:41:53 [INFO|DP=0|PP=0|TP=0]: iteration: 1914 / 150000 | consumed_samples: 122496 | elapsed_time_per_iteration_ms: 987.1 | tokens_per_sec: 5.311480E+05 | tokens_per_sec_per_gpu: 8.299187E+03 | global_batch_size: 64 | lm_loss: 1.084423E+01 | lr: 2.871E-04 | model_tflops_per_gpu: 95.86 | hardware_tflops_per_gpu: 95.86 | grad_norm: 1762247.875 [ip-26-0-154-121:0]:06/21/2023 17:41:54 [INFO|DP=0|PP=0|TP=0]: iteration: 1915 / 150000 | consumed_samples: 122560 | elapsed_time_per_iteration_ms: 997.4 | tokens_per_sec: 5.256362E+05 | tokens_per_sec_per_gpu: 8.213066E+03 | global_batch_size: 64 | lm_loss: 1.083950E+01 | lr: 2.872E-04 | model_tflops_per_gpu: 94.87 | hardware_tflops_per_gpu: 94.87 | grad_norm: 1753095.000 [ip-26-0-154-121:0]:06/21/2023 17:41:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1916 / 150000 | consumed_samples: 122624 | elapsed_time_per_iteration_ms: 931.4 | tokens_per_sec: 5.629096E+05 | tokens_per_sec_per_gpu: 8.795463E+03 | global_batch_size: 64 | lm_loss: 1.084370E+01 | lr: 2.874E-04 | model_tflops_per_gpu: 101.60 | hardware_tflops_per_gpu: 101.60 | grad_norm: 1756317.375 [ip-26-0-154-121:0]:06/21/2023 17:41:56 [INFO|DP=0|PP=0|TP=0]: iteration: 1917 / 150000 | consumed_samples: 122688 | elapsed_time_per_iteration_ms: 911.0 | tokens_per_sec: 5.754849E+05 | tokens_per_sec_per_gpu: 8.991951E+03 | global_batch_size: 64 | lm_loss: 1.084157E+01 | lr: 2.875E-04 | model_tflops_per_gpu: 103.87 | hardware_tflops_per_gpu: 103.87 | grad_norm: 1761453.500 [ip-26-0-154-121:0]:06/21/2023 17:41:57 [INFO|DP=0|PP=0|TP=0]: iteration: 1918 / 150000 | consumed_samples: 122752 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778540E+05 | tokens_per_sec_per_gpu: 9.028968E+03 | global_batch_size: 64 | lm_loss: 1.084245E+01 | lr: 2.877E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 1758739.500 [ip-26-0-154-121:0]:06/21/2023 17:41:58 [INFO|DP=0|PP=0|TP=0]: iteration: 1919 / 150000 | consumed_samples: 122816 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785784E+05 | tokens_per_sec_per_gpu: 9.040288E+03 | global_batch_size: 64 | lm_loss: 1.084259E+01 | lr: 2.878E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1783884.375 [ip-26-0-154-121:0]:06/21/2023 17:41:59 [INFO|DP=0|PP=0|TP=0]: iteration: 1920 / 150000 | consumed_samples: 122880 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784138E+05 | tokens_per_sec_per_gpu: 9.037715E+03 | global_batch_size: 64 | lm_loss: 1.084287E+01 | lr: 2.880E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1791767.000 [ip-26-0-154-121:0]:06/21/2023 17:42:00 [INFO|DP=0|PP=0|TP=0]: iteration: 1921 / 150000 | consumed_samples: 122944 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786142E+05 | tokens_per_sec_per_gpu: 9.040847E+03 | global_batch_size: 64 | lm_loss: 1.084392E+01 | lr: 2.881E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1799140.875 [ip-26-0-154-121:0]:06/21/2023 17:42:01 [INFO|DP=0|PP=0|TP=0]: iteration: 1922 / 150000 | consumed_samples: 123008 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786102E+05 | tokens_per_sec_per_gpu: 9.040785E+03 | global_batch_size: 64 | lm_loss: 1.084372E+01 | lr: 2.883E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1862197.750 [ip-26-0-154-121:0]:06/21/2023 17:42:01 [INFO|DP=0|PP=0|TP=0]: iteration: 1923 / 150000 | consumed_samples: 123072 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789120E+05 | tokens_per_sec_per_gpu: 9.045500E+03 | global_batch_size: 64 | lm_loss: 1.084288E+01 | lr: 2.884E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 1890386.125 [ip-26-0-154-121:0]:06/21/2023 17:42:02 [INFO|DP=0|PP=0|TP=0]: iteration: 1924 / 150000 | consumed_samples: 123136 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785760E+05 | tokens_per_sec_per_gpu: 9.040250E+03 | global_batch_size: 64 | lm_loss: 1.084075E+01 | lr: 2.886E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1887760.750 [ip-26-0-154-121:0]:06/21/2023 17:42:03 [INFO|DP=0|PP=0|TP=0]: iteration: 1925 / 150000 | consumed_samples: 123200 | elapsed_time_per_iteration_ms: 944.0 | tokens_per_sec: 5.553650E+05 | tokens_per_sec_per_gpu: 8.677578E+03 | global_batch_size: 64 | lm_loss: 1.084242E+01 | lr: 2.887E-04 | model_tflops_per_gpu: 100.24 | hardware_tflops_per_gpu: 100.24 | grad_norm: 2005116.500 [ip-26-0-154-121:0]:06/21/2023 17:42:04 [INFO|DP=0|PP=0|TP=0]: iteration: 1926 / 150000 | consumed_samples: 123264 | elapsed_time_per_iteration_ms: 923.2 | tokens_per_sec: 5.679235E+05 | tokens_per_sec_per_gpu: 8.873805E+03 | global_batch_size: 64 | lm_loss: 1.084195E+01 | lr: 2.889E-04 | model_tflops_per_gpu: 102.50 | hardware_tflops_per_gpu: 102.50 | grad_norm: 2511479.500 [ip-26-0-154-121:0]:06/21/2023 17:42:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1927 / 150000 | consumed_samples: 123328 | elapsed_time_per_iteration_ms: 912.0 | tokens_per_sec: 5.749079E+05 | tokens_per_sec_per_gpu: 8.982936E+03 | global_batch_size: 64 | lm_loss: 1.083777E+01 | lr: 2.890E-04 | model_tflops_per_gpu: 103.76 | hardware_tflops_per_gpu: 103.76 | grad_norm: 5910744.500 [ip-26-0-154-121:0]:06/21/2023 17:42:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1928 / 150000 | consumed_samples: 123392 | elapsed_time_per_iteration_ms: 909.0 | tokens_per_sec: 5.767473E+05 | tokens_per_sec_per_gpu: 9.011677E+03 | global_batch_size: 64 | lm_loss: 1.084125E+01 | lr: 2.892E-04 | model_tflops_per_gpu: 104.09 | hardware_tflops_per_gpu: 104.09 | grad_norm: 123642640.000 [ip-26-0-154-121:0]:06/21/2023 17:42:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1929 / 150000 | consumed_samples: 123456 | elapsed_time_per_iteration_ms: 1102.8 | tokens_per_sec: 4.754109E+05 | tokens_per_sec_per_gpu: 7.428296E+03 | global_batch_size: 64 | lm_loss: 1.083776E+01 | lr: 2.894E-04 | model_tflops_per_gpu: 85.81 | hardware_tflops_per_gpu: 85.81 | grad_norm: 116604952.000 [ip-26-0-154-121:0]:06/21/2023 17:42:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1930 / 150000 | consumed_samples: 123520 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779697E+05 | tokens_per_sec_per_gpu: 9.030776E+03 | global_batch_size: 64 | lm_loss: 1.083751E+01 | lr: 2.895E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 32200878.000 [ip-26-0-154-121:0]:06/21/2023 17:42:09 [INFO|DP=0|PP=0|TP=0]: iteration: 1931 / 150000 | consumed_samples: 123584 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784799E+05 | tokens_per_sec_per_gpu: 9.038749E+03 | global_batch_size: 64 | lm_loss: 1.084052E+01 | lr: 2.896E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 21708344.000 [ip-26-0-154-121:0]:06/21/2023 17:42:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1932 / 150000 | consumed_samples: 123648 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787216E+05 | tokens_per_sec_per_gpu: 9.042524E+03 | global_batch_size: 64 | lm_loss: 1.083756E+01 | lr: 2.898E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 6113453.000 [ip-26-0-154-121:0]:06/21/2023 17:42:11 [INFO|DP=0|PP=0|TP=0]: iteration: 1933 / 150000 | consumed_samples: 123712 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785446E+05 | tokens_per_sec_per_gpu: 9.039760E+03 | global_batch_size: 64 | lm_loss: 1.084054E+01 | lr: 2.899E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 6528787.000 [ip-26-0-154-121:0]:06/21/2023 17:42:12 [INFO|DP=0|PP=0|TP=0]: iteration: 1934 / 150000 | consumed_samples: 123776 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784483E+05 | tokens_per_sec_per_gpu: 9.038255E+03 | global_batch_size: 64 | lm_loss: 1.084032E+01 | lr: 2.901E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 15521283.000 [ip-26-0-154-121:0]:06/21/2023 17:42:13 [INFO|DP=0|PP=0|TP=0]: iteration: 1935 / 150000 | consumed_samples: 123840 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785984E+05 | tokens_per_sec_per_gpu: 9.040599E+03 | global_batch_size: 64 | lm_loss: 1.083902E+01 | lr: 2.902E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 55945600.000 [ip-26-0-154-121:0]:06/21/2023 17:42:14 [INFO|DP=0|PP=0|TP=0]: iteration: 1936 / 150000 | consumed_samples: 123904 | elapsed_time_per_iteration_ms: 925.4 | tokens_per_sec: 5.665585E+05 | tokens_per_sec_per_gpu: 8.852477E+03 | global_batch_size: 64 | lm_loss: 1.084135E+01 | lr: 2.904E-04 | model_tflops_per_gpu: 102.26 | hardware_tflops_per_gpu: 102.26 | grad_norm: 33970764.000 [ip-26-0-154-121:0]:06/21/2023 17:42:14 [INFO|DP=0|PP=0|TP=0]: iteration: 1937 / 150000 | consumed_samples: 123968 | elapsed_time_per_iteration_ms: 920.1 | tokens_per_sec: 5.698033E+05 | tokens_per_sec_per_gpu: 8.903177E+03 | global_batch_size: 64 | lm_loss: 1.084049E+01 | lr: 2.905E-04 | model_tflops_per_gpu: 102.84 | hardware_tflops_per_gpu: 102.84 | grad_norm: 3084550.500 [ip-26-0-154-121:0]:06/21/2023 17:42:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1938 / 150000 | consumed_samples: 124032 | elapsed_time_per_iteration_ms: 1049.1 | tokens_per_sec: 4.997338E+05 | tokens_per_sec_per_gpu: 7.808341E+03 | global_batch_size: 64 | lm_loss: 1.084118E+01 | lr: 2.907E-04 | model_tflops_per_gpu: 90.20 | hardware_tflops_per_gpu: 90.20 | grad_norm: 2005794.875 [ip-26-0-154-121:0]:06/21/2023 17:42:16 [INFO|DP=0|PP=0|TP=0]: iteration: 1939 / 150000 | consumed_samples: 124096 | elapsed_time_per_iteration_ms: 911.5 | tokens_per_sec: 5.752232E+05 | tokens_per_sec_per_gpu: 8.987863E+03 | global_batch_size: 64 | lm_loss: 1.083677E+01 | lr: 2.908E-04 | model_tflops_per_gpu: 103.82 | hardware_tflops_per_gpu: 103.82 | grad_norm: 23155052.000 [ip-26-0-154-121:0]:06/21/2023 17:42:18 [INFO|DP=0|PP=0|TP=0]: iteration: 1940 / 150000 | consumed_samples: 124160 | elapsed_time_per_iteration_ms: 1244.8 | tokens_per_sec: 4.211662E+05 | tokens_per_sec_per_gpu: 6.580722E+03 | global_batch_size: 64 | lm_loss: 1.084029E+01 | lr: 2.910E-04 | model_tflops_per_gpu: 76.01 | hardware_tflops_per_gpu: 76.01 | grad_norm: 8980213.000 [ip-26-0-154-121:0]:06/21/2023 17:42:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1941 / 150000 | consumed_samples: 124224 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785460E+05 | tokens_per_sec_per_gpu: 9.039781E+03 | global_batch_size: 64 | lm_loss: 1.083979E+01 | lr: 2.911E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 8917871.000 [ip-26-0-154-121:0]:06/21/2023 17:42:19 [INFO|DP=0|PP=0|TP=0]: iteration: 1942 / 150000 | consumed_samples: 124288 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786297E+05 | tokens_per_sec_per_gpu: 9.041089E+03 | global_batch_size: 64 | lm_loss: 1.083799E+01 | lr: 2.913E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 18358012.000 [ip-26-0-154-121:0]:06/21/2023 17:42:20 [INFO|DP=0|PP=0|TP=0]: iteration: 1943 / 150000 | consumed_samples: 124352 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786025E+05 | tokens_per_sec_per_gpu: 9.040664E+03 | global_batch_size: 64 | lm_loss: 1.084059E+01 | lr: 2.915E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 17204422.000 [ip-26-0-154-121:0]:06/21/2023 17:42:21 [INFO|DP=0|PP=0|TP=0]: iteration: 1944 / 150000 | consumed_samples: 124416 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786922E+05 | tokens_per_sec_per_gpu: 9.042065E+03 | global_batch_size: 64 | lm_loss: 1.083829E+01 | lr: 2.916E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 8554766.000 [ip-26-0-154-121:0]:06/21/2023 17:42:22 [INFO|DP=0|PP=0|TP=0]: iteration: 1945 / 150000 | consumed_samples: 124480 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787773E+05 | tokens_per_sec_per_gpu: 9.043395E+03 | global_batch_size: 64 | lm_loss: 1.083996E+01 | lr: 2.917E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 4728974.500 [ip-26-0-154-121:0]:06/21/2023 17:42:23 [INFO|DP=0|PP=0|TP=0]: iteration: 1946 / 150000 | consumed_samples: 124544 | elapsed_time_per_iteration_ms: 915.1 | tokens_per_sec: 5.729306E+05 | tokens_per_sec_per_gpu: 8.952040E+03 | global_batch_size: 64 | lm_loss: 1.084070E+01 | lr: 2.919E-04 | model_tflops_per_gpu: 103.41 | hardware_tflops_per_gpu: 103.41 | grad_norm: 5518903.500 [ip-26-0-154-121:0]:06/21/2023 17:42:24 [INFO|DP=0|PP=0|TP=0]: iteration: 1947 / 150000 | consumed_samples: 124608 | elapsed_time_per_iteration_ms: 922.0 | tokens_per_sec: 5.686705E+05 | tokens_per_sec_per_gpu: 8.885477E+03 | global_batch_size: 64 | lm_loss: 1.084013E+01 | lr: 2.920E-04 | model_tflops_per_gpu: 102.64 | hardware_tflops_per_gpu: 102.64 | grad_norm: 10978046.000 [ip-26-0-154-121:0]:06/21/2023 17:42:25 [INFO|DP=0|PP=0|TP=0]: iteration: 1948 / 150000 | consumed_samples: 124672 | elapsed_time_per_iteration_ms: 1171.0 | tokens_per_sec: 4.477240E+05 | tokens_per_sec_per_gpu: 6.995687E+03 | global_batch_size: 64 | lm_loss: 1.083903E+01 | lr: 2.922E-04 | model_tflops_per_gpu: 80.81 | hardware_tflops_per_gpu: 80.81 | grad_norm: 4711225.500 [ip-26-0-154-121:0]:06/21/2023 17:42:26 [INFO|DP=0|PP=0|TP=0]: iteration: 1949 / 150000 | consumed_samples: 124736 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784363E+05 | tokens_per_sec_per_gpu: 9.038067E+03 | global_batch_size: 64 | lm_loss: 1.084073E+01 | lr: 2.923E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3150703.000 [ip-26-0-154-121:0]:06/21/2023 17:42:27 [INFO|DP=0|PP=0|TP=0]: iteration: 1950 / 150000 | consumed_samples: 124800 | elapsed_time_per_iteration_ms: 917.7 | tokens_per_sec: 5.713138E+05 | tokens_per_sec_per_gpu: 8.926778E+03 | global_batch_size: 64 | lm_loss: 1.083984E+01 | lr: 2.925E-04 | model_tflops_per_gpu: 103.11 | hardware_tflops_per_gpu: 103.11 | grad_norm: 5098319.000 [ip-26-0-154-121:0]:06/21/2023 17:42:28 [INFO|DP=0|PP=0|TP=0]: iteration: 1951 / 150000 | consumed_samples: 124864 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785017E+05 | tokens_per_sec_per_gpu: 9.039089E+03 | global_batch_size: 64 | lm_loss: 1.083411E+01 | lr: 2.926E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 12646212.000 [ip-26-0-154-121:0]:06/21/2023 17:42:29 [INFO|DP=0|PP=0|TP=0]: iteration: 1952 / 150000 | consumed_samples: 124928 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786361E+05 | tokens_per_sec_per_gpu: 9.041189E+03 | global_batch_size: 64 | lm_loss: 1.083878E+01 | lr: 2.928E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2629695.750 [ip-26-0-154-121:0]:06/21/2023 17:42:30 [INFO|DP=0|PP=0|TP=0]: iteration: 1953 / 150000 | consumed_samples: 124992 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787642E+05 | tokens_per_sec_per_gpu: 9.043191E+03 | global_batch_size: 64 | lm_loss: 1.083845E+01 | lr: 2.929E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 2564310.750 [ip-26-0-154-121:0]:06/21/2023 17:42:31 [INFO|DP=0|PP=0|TP=0]: iteration: 1954 / 150000 | consumed_samples: 125056 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786509E+05 | tokens_per_sec_per_gpu: 9.041420E+03 | global_batch_size: 64 | lm_loss: 1.084080E+01 | lr: 2.931E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2618529.500 [ip-26-0-154-121:0]:06/21/2023 17:42:32 [INFO|DP=0|PP=0|TP=0]: iteration: 1955 / 150000 | consumed_samples: 125120 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786987E+05 | tokens_per_sec_per_gpu: 9.042167E+03 | global_batch_size: 64 | lm_loss: 1.084106E+01 | lr: 2.932E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 17666926.000 [ip-26-0-154-121:0]:06/21/2023 17:42:32 [INFO|DP=0|PP=0|TP=0]: iteration: 1956 / 150000 | consumed_samples: 125184 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784375E+05 | tokens_per_sec_per_gpu: 9.038086E+03 | global_batch_size: 64 | lm_loss: 1.084100E+01 | lr: 2.934E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 10317580.000 [ip-26-0-154-121:0]:06/21/2023 17:42:33 [INFO|DP=0|PP=0|TP=0]: iteration: 1957 / 150000 | consumed_samples: 125248 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785620E+05 | tokens_per_sec_per_gpu: 9.040031E+03 | global_batch_size: 64 | lm_loss: 1.083993E+01 | lr: 2.935E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 11293408.000 [ip-26-0-154-121:0]:06/21/2023 17:42:34 [INFO|DP=0|PP=0|TP=0]: iteration: 1958 / 150000 | consumed_samples: 125312 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788341E+05 | tokens_per_sec_per_gpu: 9.044283E+03 | global_batch_size: 64 | lm_loss: 1.083930E+01 | lr: 2.937E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 11307072.000 [ip-26-0-154-121:0]:06/21/2023 17:42:35 [INFO|DP=0|PP=0|TP=0]: iteration: 1959 / 150000 | consumed_samples: 125376 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788692E+05 | tokens_per_sec_per_gpu: 9.044831E+03 | global_batch_size: 64 | lm_loss: 1.083916E+01 | lr: 2.939E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 6781888.000 [ip-26-0-154-121:0]:06/21/2023 17:42:36 [INFO|DP=0|PP=0|TP=0]: iteration: 1960 / 150000 | consumed_samples: 125440 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785518E+05 | tokens_per_sec_per_gpu: 9.039872E+03 | global_batch_size: 64 | lm_loss: 1.083903E+01 | lr: 2.940E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 2653076.000 [ip-26-0-154-121:0]:06/21/2023 17:42:37 [INFO|DP=0|PP=0|TP=0]: iteration: 1961 / 150000 | consumed_samples: 125504 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786874E+05 | tokens_per_sec_per_gpu: 9.041991E+03 | global_batch_size: 64 | lm_loss: 1.083834E+01 | lr: 2.941E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1971938.875 [ip-26-0-154-121:0]:06/21/2023 17:42:38 [INFO|DP=0|PP=0|TP=0]: iteration: 1962 / 150000 | consumed_samples: 125568 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788821E+05 | tokens_per_sec_per_gpu: 9.045033E+03 | global_batch_size: 64 | lm_loss: 1.083705E+01 | lr: 2.943E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 4124418.500 [ip-26-0-154-121:0]:06/21/2023 17:42:39 [INFO|DP=0|PP=0|TP=0]: iteration: 1963 / 150000 | consumed_samples: 125632 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786548E+05 | tokens_per_sec_per_gpu: 9.041482E+03 | global_batch_size: 64 | lm_loss: 1.083854E+01 | lr: 2.944E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 29083502.000 [ip-26-0-154-121:0]:06/21/2023 17:42:40 [INFO|DP=0|PP=0|TP=0]: iteration: 1964 / 150000 | consumed_samples: 125696 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786815E+05 | tokens_per_sec_per_gpu: 9.041898E+03 | global_batch_size: 64 | lm_loss: 1.083896E+01 | lr: 2.946E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 25407318.000 [ip-26-0-154-121:0]:06/21/2023 17:42:41 [INFO|DP=0|PP=0|TP=0]: iteration: 1965 / 150000 | consumed_samples: 125760 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784182E+05 | tokens_per_sec_per_gpu: 9.037784E+03 | global_batch_size: 64 | lm_loss: 1.083739E+01 | lr: 2.947E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 20643006.000 [ip-26-0-154-121:0]:06/21/2023 17:42:42 [INFO|DP=0|PP=0|TP=0]: iteration: 1966 / 150000 | consumed_samples: 125824 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789777E+05 | tokens_per_sec_per_gpu: 9.046526E+03 | global_batch_size: 64 | lm_loss: 1.083834E+01 | lr: 2.949E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 16303418.000 [ip-26-0-154-121:0]:06/21/2023 17:42:42 [INFO|DP=0|PP=0|TP=0]: iteration: 1967 / 150000 | consumed_samples: 125888 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786698E+05 | tokens_per_sec_per_gpu: 9.041715E+03 | global_batch_size: 64 | lm_loss: 1.083835E+01 | lr: 2.950E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 10911086.000 [ip-26-0-154-121:0]:06/21/2023 17:42:43 [INFO|DP=0|PP=0|TP=0]: iteration: 1968 / 150000 | consumed_samples: 125952 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786331E+05 | tokens_per_sec_per_gpu: 9.041142E+03 | global_batch_size: 64 | lm_loss: 1.083854E+01 | lr: 2.952E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 6654792.000 [ip-26-0-154-121:0]:06/21/2023 17:42:44 [INFO|DP=0|PP=0|TP=0]: iteration: 1969 / 150000 | consumed_samples: 126016 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.778877E+05 | tokens_per_sec_per_gpu: 9.029495E+03 | global_batch_size: 64 | lm_loss: 1.083834E+01 | lr: 2.953E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 4233074.500 [ip-26-0-154-121:0]:06/21/2023 17:42:45 [INFO|DP=0|PP=0|TP=0]: iteration: 1970 / 150000 | consumed_samples: 126080 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787228E+05 | tokens_per_sec_per_gpu: 9.042543E+03 | global_batch_size: 64 | lm_loss: 1.083943E+01 | lr: 2.955E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3065852.000 [ip-26-0-154-121:0]:06/21/2023 17:42:46 [INFO|DP=0|PP=0|TP=0]: iteration: 1971 / 150000 | consumed_samples: 126144 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787252E+05 | tokens_per_sec_per_gpu: 9.042581E+03 | global_batch_size: 64 | lm_loss: 1.083591E+01 | lr: 2.956E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 2119666.250 [ip-26-0-154-121:0]:06/21/2023 17:42:47 [INFO|DP=0|PP=0|TP=0]: iteration: 1972 / 150000 | consumed_samples: 126208 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789818E+05 | tokens_per_sec_per_gpu: 9.046591E+03 | global_batch_size: 64 | lm_loss: 1.083270E+01 | lr: 2.958E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 3662187.000 [ip-26-0-154-121:0]:06/21/2023 17:42:48 [INFO|DP=0|PP=0|TP=0]: iteration: 1973 / 150000 | consumed_samples: 126272 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787931E+05 | tokens_per_sec_per_gpu: 9.043643E+03 | global_batch_size: 64 | lm_loss: 1.083680E+01 | lr: 2.960E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 4513810.000 [ip-26-0-154-121:0]:06/21/2023 17:42:49 [INFO|DP=0|PP=0|TP=0]: iteration: 1974 / 150000 | consumed_samples: 126336 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785003E+05 | tokens_per_sec_per_gpu: 9.039068E+03 | global_batch_size: 64 | lm_loss: 1.083784E+01 | lr: 2.961E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3095762.750 [ip-26-0-154-121:0]:06/21/2023 17:42:50 [INFO|DP=0|PP=0|TP=0]: iteration: 1975 / 150000 | consumed_samples: 126400 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780914E+05 | tokens_per_sec_per_gpu: 9.032678E+03 | global_batch_size: 64 | lm_loss: 1.083738E+01 | lr: 2.962E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 4577821.000 [ip-26-0-154-121:0]:06/21/2023 17:42:51 [INFO|DP=0|PP=0|TP=0]: iteration: 1976 / 150000 | consumed_samples: 126464 | elapsed_time_per_iteration_ms: 909.4 | tokens_per_sec: 5.765406E+05 | tokens_per_sec_per_gpu: 9.008447E+03 | global_batch_size: 64 | lm_loss: 1.083673E+01 | lr: 2.964E-04 | model_tflops_per_gpu: 104.06 | hardware_tflops_per_gpu: 104.06 | grad_norm: 10520505.000 [ip-26-0-154-121:0]:06/21/2023 17:42:52 [INFO|DP=0|PP=0|TP=0]: iteration: 1977 / 150000 | consumed_samples: 126528 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786035E+05 | tokens_per_sec_per_gpu: 9.040680E+03 | global_batch_size: 64 | lm_loss: 1.083705E+01 | lr: 2.965E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 33358530.000 [ip-26-0-154-121:0]:06/21/2023 17:42:52 [INFO|DP=0|PP=0|TP=0]: iteration: 1978 / 150000 | consumed_samples: 126592 | elapsed_time_per_iteration_ms: 909.3 | tokens_per_sec: 5.765905E+05 | tokens_per_sec_per_gpu: 9.009226E+03 | global_batch_size: 64 | lm_loss: 1.083470E+01 | lr: 2.967E-04 | model_tflops_per_gpu: 104.07 | hardware_tflops_per_gpu: 104.07 | grad_norm: 47624696.000 [ip-26-0-154-121:0]:06/21/2023 17:42:53 [INFO|DP=0|PP=0|TP=0]: iteration: 1979 / 150000 | consumed_samples: 126656 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785783E+05 | tokens_per_sec_per_gpu: 9.040285E+03 | global_batch_size: 64 | lm_loss: 1.083581E+01 | lr: 2.968E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 31469674.000 [ip-26-0-154-121:0]:06/21/2023 17:42:54 [INFO|DP=0|PP=0|TP=0]: iteration: 1980 / 150000 | consumed_samples: 126720 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786209E+05 | tokens_per_sec_per_gpu: 9.040951E+03 | global_batch_size: 64 | lm_loss: 1.083660E+01 | lr: 2.970E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 7264182.000 [ip-26-0-154-121:0]:06/21/2023 17:42:55 [INFO|DP=0|PP=0|TP=0]: iteration: 1981 / 150000 | consumed_samples: 126784 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784669E+05 | tokens_per_sec_per_gpu: 9.038545E+03 | global_batch_size: 64 | lm_loss: 1.083296E+01 | lr: 2.971E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1798570.125 [ip-26-0-154-121:0]:06/21/2023 17:42:56 [INFO|DP=0|PP=0|TP=0]: iteration: 1982 / 150000 | consumed_samples: 126848 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786296E+05 | tokens_per_sec_per_gpu: 9.041087E+03 | global_batch_size: 64 | lm_loss: 1.083650E+01 | lr: 2.973E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1682626.875 [ip-26-0-154-121:0]:06/21/2023 17:42:57 [INFO|DP=0|PP=0|TP=0]: iteration: 1983 / 150000 | consumed_samples: 126912 | elapsed_time_per_iteration_ms: 928.6 | tokens_per_sec: 5.645900E+05 | tokens_per_sec_per_gpu: 8.821719E+03 | global_batch_size: 64 | lm_loss: 1.083672E+01 | lr: 2.974E-04 | model_tflops_per_gpu: 101.90 | hardware_tflops_per_gpu: 101.90 | grad_norm: 1726366.250 [ip-26-0-154-121:0]:06/21/2023 17:42:58 [INFO|DP=0|PP=0|TP=0]: iteration: 1984 / 150000 | consumed_samples: 126976 | elapsed_time_per_iteration_ms: 911.9 | tokens_per_sec: 5.749234E+05 | tokens_per_sec_per_gpu: 8.983178E+03 | global_batch_size: 64 | lm_loss: 1.083289E+01 | lr: 2.976E-04 | model_tflops_per_gpu: 103.77 | hardware_tflops_per_gpu: 103.77 | grad_norm: 2060112.875 [ip-26-0-154-121:0]:06/21/2023 17:42:59 [INFO|DP=0|PP=0|TP=0]: iteration: 1985 / 150000 | consumed_samples: 127040 | elapsed_time_per_iteration_ms: 1005.8 | tokens_per_sec: 5.212873E+05 | tokens_per_sec_per_gpu: 8.145114E+03 | global_batch_size: 64 | lm_loss: 1.083788E+01 | lr: 2.977E-04 | model_tflops_per_gpu: 94.09 | hardware_tflops_per_gpu: 94.09 | grad_norm: 2182271.000 [ip-26-0-154-121:0]:06/21/2023 17:43:00 [INFO|DP=0|PP=0|TP=0]: iteration: 1986 / 150000 | consumed_samples: 127104 | elapsed_time_per_iteration_ms: 937.4 | tokens_per_sec: 5.593037E+05 | tokens_per_sec_per_gpu: 8.739120E+03 | global_batch_size: 64 | lm_loss: 1.083573E+01 | lr: 2.979E-04 | model_tflops_per_gpu: 100.95 | hardware_tflops_per_gpu: 100.95 | grad_norm: 1767471.625 [ip-26-0-154-121:0]:06/21/2023 17:43:01 [INFO|DP=0|PP=0|TP=0]: iteration: 1987 / 150000 | consumed_samples: 127168 | elapsed_time_per_iteration_ms: 932.4 | tokens_per_sec: 5.622760E+05 | tokens_per_sec_per_gpu: 8.785563E+03 | global_batch_size: 64 | lm_loss: 1.083554E+01 | lr: 2.981E-04 | model_tflops_per_gpu: 101.48 | hardware_tflops_per_gpu: 101.48 | grad_norm: 1659338.250 [ip-26-0-154-121:0]:06/21/2023 17:43:02 [INFO|DP=0|PP=0|TP=0]: iteration: 1988 / 150000 | consumed_samples: 127232 | elapsed_time_per_iteration_ms: 937.4 | tokens_per_sec: 5.593054E+05 | tokens_per_sec_per_gpu: 8.739147E+03 | global_batch_size: 64 | lm_loss: 1.083510E+01 | lr: 2.982E-04 | model_tflops_per_gpu: 100.95 | hardware_tflops_per_gpu: 100.95 | grad_norm: 1641969.500 [ip-26-0-154-121:0]:06/21/2023 17:43:03 [INFO|DP=0|PP=0|TP=0]: iteration: 1989 / 150000 | consumed_samples: 127296 | elapsed_time_per_iteration_ms: 992.8 | tokens_per_sec: 5.281020E+05 | tokens_per_sec_per_gpu: 8.251594E+03 | global_batch_size: 64 | lm_loss: 1.083519E+01 | lr: 2.983E-04 | model_tflops_per_gpu: 95.32 | hardware_tflops_per_gpu: 95.32 | grad_norm: 1633129.000 [ip-26-0-154-121:0]:06/21/2023 17:43:04 [INFO|DP=0|PP=0|TP=0]: iteration: 1990 / 150000 | consumed_samples: 127360 | elapsed_time_per_iteration_ms: 929.8 | tokens_per_sec: 5.638988E+05 | tokens_per_sec_per_gpu: 8.810919E+03 | global_batch_size: 64 | lm_loss: 1.082846E+01 | lr: 2.985E-04 | model_tflops_per_gpu: 101.78 | hardware_tflops_per_gpu: 101.78 | grad_norm: 1635296.875 [ip-26-0-154-121:0]:06/21/2023 17:43:05 [INFO|DP=0|PP=0|TP=0]: iteration: 1991 / 150000 | consumed_samples: 127424 | elapsed_time_per_iteration_ms: 966.5 | tokens_per_sec: 5.424609E+05 | tokens_per_sec_per_gpu: 8.475952E+03 | global_batch_size: 64 | lm_loss: 1.083590E+01 | lr: 2.986E-04 | model_tflops_per_gpu: 97.91 | hardware_tflops_per_gpu: 97.91 | grad_norm: 1658224.875 [ip-26-0-154-121:0]:06/21/2023 17:43:06 [INFO|DP=0|PP=0|TP=0]: iteration: 1992 / 150000 | consumed_samples: 127488 | elapsed_time_per_iteration_ms: 1175.6 | tokens_per_sec: 4.459602E+05 | tokens_per_sec_per_gpu: 6.968129E+03 | global_batch_size: 64 | lm_loss: 1.083521E+01 | lr: 2.988E-04 | model_tflops_per_gpu: 80.49 | hardware_tflops_per_gpu: 80.49 | grad_norm: 1678967.875 [ip-26-0-154-121:0]:06/21/2023 17:43:07 [INFO|DP=0|PP=0|TP=0]: iteration: 1993 / 150000 | consumed_samples: 127552 | elapsed_time_per_iteration_ms: 917.0 | tokens_per_sec: 5.717649E+05 | tokens_per_sec_per_gpu: 8.933827E+03 | global_batch_size: 64 | lm_loss: 1.083502E+01 | lr: 2.989E-04 | model_tflops_per_gpu: 103.20 | hardware_tflops_per_gpu: 103.20 | grad_norm: 1676979.500 [ip-26-0-154-121:0]:06/21/2023 17:43:08 [INFO|DP=0|PP=0|TP=0]: iteration: 1994 / 150000 | consumed_samples: 127616 | elapsed_time_per_iteration_ms: 931.2 | tokens_per_sec: 5.629967E+05 | tokens_per_sec_per_gpu: 8.796823E+03 | global_batch_size: 64 | lm_loss: 1.083476E+01 | lr: 2.991E-04 | model_tflops_per_gpu: 101.61 | hardware_tflops_per_gpu: 101.61 | grad_norm: 1644341.750 [ip-26-0-154-121:0]:06/21/2023 17:43:09 [INFO|DP=0|PP=0|TP=0]: iteration: 1995 / 150000 | consumed_samples: 127680 | elapsed_time_per_iteration_ms: 925.7 | tokens_per_sec: 5.663615E+05 | tokens_per_sec_per_gpu: 8.849399E+03 | global_batch_size: 64 | lm_loss: 1.083463E+01 | lr: 2.992E-04 | model_tflops_per_gpu: 102.22 | hardware_tflops_per_gpu: 102.22 | grad_norm: 1629022.250 [ip-26-0-154-121:0]:06/21/2023 17:43:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1996 / 150000 | consumed_samples: 127744 | elapsed_time_per_iteration_ms: 924.0 | tokens_per_sec: 5.674127E+05 | tokens_per_sec_per_gpu: 8.865824E+03 | global_batch_size: 64 | lm_loss: 1.083501E+01 | lr: 2.994E-04 | model_tflops_per_gpu: 102.41 | hardware_tflops_per_gpu: 102.41 | grad_norm: 1623858.375 [ip-26-0-154-121:0]:06/21/2023 17:43:10 [INFO|DP=0|PP=0|TP=0]: iteration: 1997 / 150000 | consumed_samples: 127808 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784876E+05 | tokens_per_sec_per_gpu: 9.038868E+03 | global_batch_size: 64 | lm_loss: 1.083513E+01 | lr: 2.995E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1614483.250 [ip-26-0-154-121:0]:06/21/2023 17:43:11 [INFO|DP=0|PP=0|TP=0]: iteration: 1998 / 150000 | consumed_samples: 127872 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784089E+05 | tokens_per_sec_per_gpu: 9.037639E+03 | global_batch_size: 64 | lm_loss: 1.083299E+01 | lr: 2.997E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1604948.500 [ip-26-0-154-121:0]:06/21/2023 17:43:12 [INFO|DP=0|PP=0|TP=0]: iteration: 1999 / 150000 | consumed_samples: 127936 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785151E+05 | tokens_per_sec_per_gpu: 9.039298E+03 | global_batch_size: 64 | lm_loss: 1.083456E+01 | lr: 2.998E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1595821.875 [ip-26-0-154-121:0]:06/21/2023 17:43:13 [INFO|DP=0|PP=0|TP=0]: iteration: 2000 / 150000 | consumed_samples: 128000 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789378E+05 | tokens_per_sec_per_gpu: 9.045902E+03 | global_batch_size: 64 | lm_loss: 1.083425E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 1591511.750 [ip-26-0-154-121:0]:06/21/2023 17:43:14 [INFO|DP=0|PP=0|TP=0]: iteration: 2001 / 150000 | consumed_samples: 128064 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785399E+05 | tokens_per_sec_per_gpu: 9.039686E+03 | global_batch_size: 64 | lm_loss: 1.083420E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1594334.750 [ip-26-0-154-121:0]:06/21/2023 17:43:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2002 / 150000 | consumed_samples: 128128 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785271E+05 | tokens_per_sec_per_gpu: 9.039486E+03 | global_batch_size: 64 | lm_loss: 1.083391E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1602771.375 [ip-26-0-154-121:0]:06/21/2023 17:43:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2003 / 150000 | consumed_samples: 128192 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787261E+05 | tokens_per_sec_per_gpu: 9.042596E+03 | global_batch_size: 64 | lm_loss: 1.083359E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1607363.250 [ip-26-0-154-121:0]:06/21/2023 17:43:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2004 / 150000 | consumed_samples: 128256 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790442E+05 | tokens_per_sec_per_gpu: 9.047565E+03 | global_batch_size: 64 | lm_loss: 1.083385E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 1606820.250 [ip-26-0-154-121:0]:06/21/2023 17:43:18 [INFO|DP=0|PP=0|TP=0]: iteration: 2005 / 150000 | consumed_samples: 128320 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784681E+05 | tokens_per_sec_per_gpu: 9.038564E+03 | global_batch_size: 64 | lm_loss: 1.083429E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1599299.625 [ip-26-0-154-121:0]:06/21/2023 17:43:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2006 / 150000 | consumed_samples: 128384 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785713E+05 | tokens_per_sec_per_gpu: 9.040176E+03 | global_batch_size: 64 | lm_loss: 1.083356E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1587590.250 [ip-26-0-154-121:0]:06/21/2023 17:43:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2007 / 150000 | consumed_samples: 128448 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788402E+05 | tokens_per_sec_per_gpu: 9.044378E+03 | global_batch_size: 64 | lm_loss: 1.083367E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1579660.500 [ip-26-0-154-121:0]:06/21/2023 17:43:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2008 / 150000 | consumed_samples: 128512 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784285E+05 | tokens_per_sec_per_gpu: 9.037946E+03 | global_batch_size: 64 | lm_loss: 1.083357E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1574489.875 [ip-26-0-154-121:0]:06/21/2023 17:43:21 [INFO|DP=0|PP=0|TP=0]: iteration: 2009 / 150000 | consumed_samples: 128576 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781811E+05 | tokens_per_sec_per_gpu: 9.034079E+03 | global_batch_size: 64 | lm_loss: 1.083226E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 1570767.875 [ip-26-0-154-121:0]:06/21/2023 17:43:22 [INFO|DP=0|PP=0|TP=0]: iteration: 2010 / 150000 | consumed_samples: 128640 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785624E+05 | tokens_per_sec_per_gpu: 9.040038E+03 | global_batch_size: 64 | lm_loss: 1.083158E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1571020.625 [ip-26-0-154-121:0]:06/21/2023 17:43:23 [INFO|DP=0|PP=0|TP=0]: iteration: 2011 / 150000 | consumed_samples: 128704 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785500E+05 | tokens_per_sec_per_gpu: 9.039843E+03 | global_batch_size: 64 | lm_loss: 1.083237E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1570491.250 [ip-26-0-154-121:0]:06/21/2023 17:43:24 [INFO|DP=0|PP=0|TP=0]: iteration: 2012 / 150000 | consumed_samples: 128768 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782887E+05 | tokens_per_sec_per_gpu: 9.035761E+03 | global_batch_size: 64 | lm_loss: 1.083219E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 1567053.125 [ip-26-0-154-121:0]:06/21/2023 17:43:25 [INFO|DP=0|PP=0|TP=0]: iteration: 2013 / 150000 | consumed_samples: 128832 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777173E+05 | tokens_per_sec_per_gpu: 9.026833E+03 | global_batch_size: 64 | lm_loss: 1.083216E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 1565082.625 [ip-26-0-154-121:0]:06/21/2023 17:43:26 [INFO|DP=0|PP=0|TP=0]: iteration: 2014 / 150000 | consumed_samples: 128896 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784860E+05 | tokens_per_sec_per_gpu: 9.038844E+03 | global_batch_size: 64 | lm_loss: 1.083200E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1566641.500 [ip-26-0-154-121:0]:06/21/2023 17:43:27 [INFO|DP=0|PP=0|TP=0]: iteration: 2015 / 150000 | consumed_samples: 128960 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782600E+05 | tokens_per_sec_per_gpu: 9.035312E+03 | global_batch_size: 64 | lm_loss: 1.083297E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 1572766.625 [ip-26-0-154-121:0]:06/21/2023 17:43:28 [INFO|DP=0|PP=0|TP=0]: iteration: 2016 / 150000 | consumed_samples: 129024 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786474E+05 | tokens_per_sec_per_gpu: 9.041365E+03 | global_batch_size: 64 | lm_loss: 1.083176E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1571676.750 [ip-26-0-154-121:0]:06/21/2023 17:43:29 [INFO|DP=0|PP=0|TP=0]: iteration: 2017 / 150000 | consumed_samples: 129088 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783138E+05 | tokens_per_sec_per_gpu: 9.036153E+03 | global_batch_size: 64 | lm_loss: 1.083042E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 1563621.125 [ip-26-0-154-121:0]:06/21/2023 17:43:30 [INFO|DP=0|PP=0|TP=0]: iteration: 2018 / 150000 | consumed_samples: 129152 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785201E+05 | tokens_per_sec_per_gpu: 9.039377E+03 | global_batch_size: 64 | lm_loss: 1.083142E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1558107.750 [ip-26-0-154-121:0]:06/21/2023 17:43:30 [INFO|DP=0|PP=0|TP=0]: iteration: 2019 / 150000 | consumed_samples: 129216 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785218E+05 | tokens_per_sec_per_gpu: 9.039403E+03 | global_batch_size: 64 | lm_loss: 1.083174E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1556564.500 [ip-26-0-154-121:0]:06/21/2023 17:43:31 [INFO|DP=0|PP=0|TP=0]: iteration: 2020 / 150000 | consumed_samples: 129280 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784868E+05 | tokens_per_sec_per_gpu: 9.038856E+03 | global_batch_size: 64 | lm_loss: 1.083077E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1556791.375 [ip-26-0-154-121:0]:06/21/2023 17:43:32 [INFO|DP=0|PP=0|TP=0]: iteration: 2021 / 150000 | consumed_samples: 129344 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787758E+05 | tokens_per_sec_per_gpu: 9.043371E+03 | global_batch_size: 64 | lm_loss: 1.083104E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1558232.625 [ip-26-0-154-121:0]:06/21/2023 17:43:33 [INFO|DP=0|PP=0|TP=0]: iteration: 2022 / 150000 | consumed_samples: 129408 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785626E+05 | tokens_per_sec_per_gpu: 9.040040E+03 | global_batch_size: 64 | lm_loss: 1.083187E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1561935.750 [ip-26-0-154-121:0]:06/21/2023 17:43:34 [INFO|DP=0|PP=0|TP=0]: iteration: 2023 / 150000 | consumed_samples: 129472 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782198E+05 | tokens_per_sec_per_gpu: 9.034685E+03 | global_batch_size: 64 | lm_loss: 1.083068E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 1562203.250 [ip-26-0-154-121:0]:06/21/2023 17:43:35 [INFO|DP=0|PP=0|TP=0]: iteration: 2024 / 150000 | consumed_samples: 129536 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780419E+05 | tokens_per_sec_per_gpu: 9.031904E+03 | global_batch_size: 64 | lm_loss: 1.083113E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 1553642.750 [ip-26-0-154-121:0]:06/21/2023 17:43:36 [INFO|DP=0|PP=0|TP=0]: iteration: 2025 / 150000 | consumed_samples: 129600 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781993E+05 | tokens_per_sec_per_gpu: 9.034364E+03 | global_batch_size: 64 | lm_loss: 1.083045E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 1548640.500 [ip-26-0-154-121:0]:06/21/2023 17:43:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2026 / 150000 | consumed_samples: 129664 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782297E+05 | tokens_per_sec_per_gpu: 9.034839E+03 | global_batch_size: 64 | lm_loss: 1.082944E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 1553105.625 [ip-26-0-154-121:0]:06/21/2023 17:43:38 [INFO|DP=0|PP=0|TP=0]: iteration: 2027 / 150000 | consumed_samples: 129728 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789228E+05 | tokens_per_sec_per_gpu: 9.045669E+03 | global_batch_size: 64 | lm_loss: 1.082863E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 1562042.750 [ip-26-0-154-121:0]:06/21/2023 17:43:39 [INFO|DP=0|PP=0|TP=0]: iteration: 2028 / 150000 | consumed_samples: 129792 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781060E+05 | tokens_per_sec_per_gpu: 9.032906E+03 | global_batch_size: 64 | lm_loss: 1.082976E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 1561811.875 [ip-26-0-154-121:0]:06/21/2023 17:43:40 [INFO|DP=0|PP=0|TP=0]: iteration: 2029 / 150000 | consumed_samples: 129856 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783842E+05 | tokens_per_sec_per_gpu: 9.037254E+03 | global_batch_size: 64 | lm_loss: 1.082927E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1554141.875 [ip-26-0-154-121:0]:06/21/2023 17:43:40 [INFO|DP=0|PP=0|TP=0]: iteration: 2030 / 150000 | consumed_samples: 129920 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782627E+05 | tokens_per_sec_per_gpu: 9.035355E+03 | global_batch_size: 64 | lm_loss: 1.082988E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 1543335.375 [ip-26-0-154-121:0]:06/21/2023 17:43:41 [INFO|DP=0|PP=0|TP=0]: iteration: 2031 / 150000 | consumed_samples: 129984 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782200E+05 | tokens_per_sec_per_gpu: 9.034687E+03 | global_batch_size: 64 | lm_loss: 1.082988E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 1542370.250 [ip-26-0-154-121:0]:06/21/2023 17:43:42 [INFO|DP=0|PP=0|TP=0]: iteration: 2032 / 150000 | consumed_samples: 130048 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779042E+05 | tokens_per_sec_per_gpu: 9.029754E+03 | global_batch_size: 64 | lm_loss: 1.082954E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 1543520.500 [ip-26-0-154-121:0]:06/21/2023 17:43:43 [INFO|DP=0|PP=0|TP=0]: iteration: 2033 / 150000 | consumed_samples: 130112 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787545E+05 | tokens_per_sec_per_gpu: 9.043038E+03 | global_batch_size: 64 | lm_loss: 1.082810E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1545309.500 [ip-26-0-154-121:0]:06/21/2023 17:43:44 [INFO|DP=0|PP=0|TP=0]: iteration: 2034 / 150000 | consumed_samples: 130176 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.782036E+05 | tokens_per_sec_per_gpu: 9.034431E+03 | global_batch_size: 64 | lm_loss: 1.082972E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 1544171.000 [ip-26-0-154-121:0]:06/21/2023 17:43:45 [INFO|DP=0|PP=0|TP=0]: iteration: 2035 / 150000 | consumed_samples: 130240 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785271E+05 | tokens_per_sec_per_gpu: 9.039486E+03 | global_batch_size: 64 | lm_loss: 1.082911E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1534217.250 [ip-26-0-154-121:0]:06/21/2023 17:43:46 [INFO|DP=0|PP=0|TP=0]: iteration: 2036 / 150000 | consumed_samples: 130304 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.775846E+05 | tokens_per_sec_per_gpu: 9.024759E+03 | global_batch_size: 64 | lm_loss: 1.082970E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 1526105.125 [ip-26-0-154-121:0]:06/21/2023 17:43:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2037 / 150000 | consumed_samples: 130368 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785105E+05 | tokens_per_sec_per_gpu: 9.039227E+03 | global_batch_size: 64 | lm_loss: 1.082937E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1526419.125 [ip-26-0-154-121:0]:06/21/2023 17:43:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2038 / 150000 | consumed_samples: 130432 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783010E+05 | tokens_per_sec_per_gpu: 9.035954E+03 | global_batch_size: 64 | lm_loss: 1.082898E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 1525916.000 [ip-26-0-154-121:0]:06/21/2023 17:43:49 [INFO|DP=0|PP=0|TP=0]: iteration: 2039 / 150000 | consumed_samples: 130496 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783558E+05 | tokens_per_sec_per_gpu: 9.036809E+03 | global_batch_size: 64 | lm_loss: 1.082707E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1534105.750 [ip-26-0-154-121:0]:06/21/2023 17:43:49 [INFO|DP=0|PP=0|TP=0]: iteration: 2040 / 150000 | consumed_samples: 130560 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789641E+05 | tokens_per_sec_per_gpu: 9.046314E+03 | global_batch_size: 64 | lm_loss: 1.082792E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 1547245.875 [ip-26-0-154-121:0]:06/21/2023 17:43:50 [INFO|DP=0|PP=0|TP=0]: iteration: 2041 / 150000 | consumed_samples: 130624 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786603E+05 | tokens_per_sec_per_gpu: 9.041568E+03 | global_batch_size: 64 | lm_loss: 1.082847E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1547017.500 [ip-26-0-154-121:0]:06/21/2023 17:43:51 [INFO|DP=0|PP=0|TP=0]: iteration: 2042 / 150000 | consumed_samples: 130688 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782094E+05 | tokens_per_sec_per_gpu: 9.034521E+03 | global_batch_size: 64 | lm_loss: 1.082858E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 1542439.500 [ip-26-0-154-121:0]:06/21/2023 17:43:52 [INFO|DP=0|PP=0|TP=0]: iteration: 2043 / 150000 | consumed_samples: 130752 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790440E+05 | tokens_per_sec_per_gpu: 9.047563E+03 | global_batch_size: 64 | lm_loss: 1.082863E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 1550810.000 [ip-26-0-154-121:0]:06/21/2023 17:43:53 [INFO|DP=0|PP=0|TP=0]: iteration: 2044 / 150000 | consumed_samples: 130816 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774655E+05 | tokens_per_sec_per_gpu: 9.022898E+03 | global_batch_size: 64 | lm_loss: 1.082658E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 1573306.250 [ip-26-0-154-121:0]:06/21/2023 17:43:54 [INFO|DP=0|PP=0|TP=0]: iteration: 2045 / 150000 | consumed_samples: 130880 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780884E+05 | tokens_per_sec_per_gpu: 9.032631E+03 | global_batch_size: 64 | lm_loss: 1.082764E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 1593285.625 [ip-26-0-154-121:0]:06/21/2023 17:43:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2046 / 150000 | consumed_samples: 130944 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785090E+05 | tokens_per_sec_per_gpu: 9.039203E+03 | global_batch_size: 64 | lm_loss: 1.082809E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1751948.375 [ip-26-0-154-121:0]:06/21/2023 17:43:56 [INFO|DP=0|PP=0|TP=0]: iteration: 2047 / 150000 | consumed_samples: 131008 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779446E+05 | tokens_per_sec_per_gpu: 9.030385E+03 | global_batch_size: 64 | lm_loss: 1.082833E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 2244669.250 [ip-26-0-154-121:0]:06/21/2023 17:43:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2048 / 150000 | consumed_samples: 131072 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786931E+05 | tokens_per_sec_per_gpu: 9.042079E+03 | global_batch_size: 64 | lm_loss: 1.082829E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 4425542.000 [ip-26-0-154-121:0]:06/21/2023 17:43:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2049 / 150000 | consumed_samples: 131136 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789905E+05 | tokens_per_sec_per_gpu: 9.046726E+03 | global_batch_size: 64 | lm_loss: 1.082751E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 36635028.000 [ip-26-0-154-121:0]:06/21/2023 17:43:59 [INFO|DP=0|PP=0|TP=0]: iteration: 2050 / 150000 | consumed_samples: 131200 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780666E+05 | tokens_per_sec_per_gpu: 9.032291E+03 | global_batch_size: 64 | lm_loss: 1.082904E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 13032001.000 [ip-26-0-154-121:0]:06/21/2023 17:43:59 [INFO|DP=0|PP=0|TP=0]: iteration: 2051 / 150000 | consumed_samples: 131264 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785506E+05 | tokens_per_sec_per_gpu: 9.039853E+03 | global_batch_size: 64 | lm_loss: 1.082884E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 44210312.000 [ip-26-0-154-121:0]:06/21/2023 17:44:00 [INFO|DP=0|PP=0|TP=0]: iteration: 2052 / 150000 | consumed_samples: 131328 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779878E+05 | tokens_per_sec_per_gpu: 9.031059E+03 | global_batch_size: 64 | lm_loss: 1.082551E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 3546995.500 [ip-26-0-154-121:0]:06/21/2023 17:44:01 [INFO|DP=0|PP=0|TP=0]: iteration: 2053 / 150000 | consumed_samples: 131392 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784010E+05 | tokens_per_sec_per_gpu: 9.037515E+03 | global_batch_size: 64 | lm_loss: 1.082709E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 2261450.250 [ip-26-0-154-121:0]:06/21/2023 17:44:02 [INFO|DP=0|PP=0|TP=0]: iteration: 2054 / 150000 | consumed_samples: 131456 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781285E+05 | tokens_per_sec_per_gpu: 9.033258E+03 | global_batch_size: 64 | lm_loss: 1.082649E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 34150728.000 [ip-26-0-154-121:0]:06/21/2023 17:44:03 [INFO|DP=0|PP=0|TP=0]: iteration: 2055 / 150000 | consumed_samples: 131520 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783392E+05 | tokens_per_sec_per_gpu: 9.036550E+03 | global_batch_size: 64 | lm_loss: 1.082615E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 11238705.000 [ip-26-0-154-121:0]:06/21/2023 17:44:04 [INFO|DP=0|PP=0|TP=0]: iteration: 2056 / 150000 | consumed_samples: 131584 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786848E+05 | tokens_per_sec_per_gpu: 9.041951E+03 | global_batch_size: 64 | lm_loss: 1.082655E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 40562764.000 [ip-26-0-154-121:0]:06/21/2023 17:44:05 [INFO|DP=0|PP=0|TP=0]: iteration: 2057 / 150000 | consumed_samples: 131648 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783683E+05 | tokens_per_sec_per_gpu: 9.037004E+03 | global_batch_size: 64 | lm_loss: 1.082452E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 6272506.000 [ip-26-0-154-121:0]:06/21/2023 17:44:06 [INFO|DP=0|PP=0|TP=0]: iteration: 2058 / 150000 | consumed_samples: 131712 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784576E+05 | tokens_per_sec_per_gpu: 9.038400E+03 | global_batch_size: 64 | lm_loss: 1.082487E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1995881.125 [ip-26-0-154-121:0]:06/21/2023 17:44:07 [INFO|DP=0|PP=0|TP=0]: iteration: 2059 / 150000 | consumed_samples: 131776 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787174E+05 | tokens_per_sec_per_gpu: 9.042460E+03 | global_batch_size: 64 | lm_loss: 1.082537E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 9807795.000 [ip-26-0-154-121:0]:06/21/2023 17:44:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2060 / 150000 | consumed_samples: 131840 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784564E+05 | tokens_per_sec_per_gpu: 9.038381E+03 | global_batch_size: 64 | lm_loss: 1.082733E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 5354283.000 [ip-26-0-154-121:0]:06/21/2023 17:44:09 [INFO|DP=0|PP=0|TP=0]: iteration: 2061 / 150000 | consumed_samples: 131904 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790010E+05 | tokens_per_sec_per_gpu: 9.046891E+03 | global_batch_size: 64 | lm_loss: 1.082886E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 3472793.750 [ip-26-0-154-121:0]:06/21/2023 17:44:09 [INFO|DP=0|PP=0|TP=0]: iteration: 2062 / 150000 | consumed_samples: 131968 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.776232E+05 | tokens_per_sec_per_gpu: 9.025363E+03 | global_batch_size: 64 | lm_loss: 1.082857E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 12386681.000 [ip-26-0-154-121:0]:06/21/2023 17:44:10 [INFO|DP=0|PP=0|TP=0]: iteration: 2063 / 150000 | consumed_samples: 132032 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783788E+05 | tokens_per_sec_per_gpu: 9.037168E+03 | global_batch_size: 64 | lm_loss: 1.082638E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 14143365.000 [ip-26-0-154-121:0]:06/21/2023 17:44:11 [INFO|DP=0|PP=0|TP=0]: iteration: 2064 / 150000 | consumed_samples: 132096 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788600E+05 | tokens_per_sec_per_gpu: 9.044688E+03 | global_batch_size: 64 | lm_loss: 1.082745E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1981344.125 [ip-26-0-154-121:0]:06/21/2023 17:44:12 [INFO|DP=0|PP=0|TP=0]: iteration: 2065 / 150000 | consumed_samples: 132160 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785672E+05 | tokens_per_sec_per_gpu: 9.040112E+03 | global_batch_size: 64 | lm_loss: 1.082745E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1687096.625 [ip-26-0-154-121:0]:06/21/2023 17:44:13 [INFO|DP=0|PP=0|TP=0]: iteration: 2066 / 150000 | consumed_samples: 132224 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775645E+05 | tokens_per_sec_per_gpu: 9.024446E+03 | global_batch_size: 64 | lm_loss: 1.082618E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 5858496.000 [ip-26-0-154-121:0]:06/21/2023 17:44:14 [INFO|DP=0|PP=0|TP=0]: iteration: 2067 / 150000 | consumed_samples: 132288 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783397E+05 | tokens_per_sec_per_gpu: 9.036557E+03 | global_batch_size: 64 | lm_loss: 1.082522E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 28214164.000 [ip-26-0-154-121:0]:06/21/2023 17:44:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2068 / 150000 | consumed_samples: 132352 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785248E+05 | tokens_per_sec_per_gpu: 9.039451E+03 | global_batch_size: 64 | lm_loss: 1.082545E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 21762860.000 [ip-26-0-154-121:0]:06/21/2023 17:44:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2069 / 150000 | consumed_samples: 132416 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785807E+05 | tokens_per_sec_per_gpu: 9.040324E+03 | global_batch_size: 64 | lm_loss: 1.082524E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 11629790.000 [ip-26-0-154-121:0]:06/21/2023 17:44:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2070 / 150000 | consumed_samples: 132480 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785684E+05 | tokens_per_sec_per_gpu: 9.040131E+03 | global_batch_size: 64 | lm_loss: 1.082484E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 2887098.000 [ip-26-0-154-121:0]:06/21/2023 17:44:18 [INFO|DP=0|PP=0|TP=0]: iteration: 2071 / 150000 | consumed_samples: 132544 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786370E+05 | tokens_per_sec_per_gpu: 9.041204E+03 | global_batch_size: 64 | lm_loss: 1.081489E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1819681.375 [ip-26-0-154-121:0]:06/21/2023 17:44:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2072 / 150000 | consumed_samples: 132608 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781475E+05 | tokens_per_sec_per_gpu: 9.033554E+03 | global_batch_size: 64 | lm_loss: 1.082217E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 1783025.500 [ip-26-0-154-121:0]:06/21/2023 17:44:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2073 / 150000 | consumed_samples: 132672 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785264E+05 | tokens_per_sec_per_gpu: 9.039474E+03 | global_batch_size: 64 | lm_loss: 1.082624E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 4702655.500 [ip-26-0-154-121:0]:06/21/2023 17:44:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2074 / 150000 | consumed_samples: 132736 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777246E+05 | tokens_per_sec_per_gpu: 9.026947E+03 | global_batch_size: 64 | lm_loss: 1.082720E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 26687080.000 [ip-26-0-154-121:0]:06/21/2023 17:44:21 [INFO|DP=0|PP=0|TP=0]: iteration: 2075 / 150000 | consumed_samples: 132800 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776841E+05 | tokens_per_sec_per_gpu: 9.026314E+03 | global_batch_size: 64 | lm_loss: 1.082763E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 46257508.000 [ip-26-0-154-121:0]:06/21/2023 17:44:22 [INFO|DP=0|PP=0|TP=0]: iteration: 2076 / 150000 | consumed_samples: 132864 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781768E+05 | tokens_per_sec_per_gpu: 9.034013E+03 | global_batch_size: 64 | lm_loss: 1.082738E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 43989716.000 [ip-26-0-154-121:0]:06/21/2023 17:44:23 [INFO|DP=0|PP=0|TP=0]: iteration: 2077 / 150000 | consumed_samples: 132928 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777374E+05 | tokens_per_sec_per_gpu: 9.027146E+03 | global_batch_size: 64 | lm_loss: 1.082684E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 21674350.000 [ip-26-0-154-121:0]:06/21/2023 17:44:24 [INFO|DP=0|PP=0|TP=0]: iteration: 2078 / 150000 | consumed_samples: 132992 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781706E+05 | tokens_per_sec_per_gpu: 9.033915E+03 | global_batch_size: 64 | lm_loss: 1.082821E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 10062229.000 [ip-26-0-154-121:0]:06/21/2023 17:44:25 [INFO|DP=0|PP=0|TP=0]: iteration: 2079 / 150000 | consumed_samples: 133056 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779916E+05 | tokens_per_sec_per_gpu: 9.031118E+03 | global_batch_size: 64 | lm_loss: 1.082680E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 5707248.500 [ip-26-0-154-121:0]:06/21/2023 17:44:26 [INFO|DP=0|PP=0|TP=0]: iteration: 2080 / 150000 | consumed_samples: 133120 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785328E+05 | tokens_per_sec_per_gpu: 9.039574E+03 | global_batch_size: 64 | lm_loss: 1.082631E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3302697.500 [ip-26-0-154-121:0]:06/21/2023 17:44:27 [INFO|DP=0|PP=0|TP=0]: iteration: 2081 / 150000 | consumed_samples: 133184 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788914E+05 | tokens_per_sec_per_gpu: 9.045178E+03 | global_batch_size: 64 | lm_loss: 1.082692E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1809157.375 [ip-26-0-154-121:0]:06/21/2023 17:44:28 [INFO|DP=0|PP=0|TP=0]: iteration: 2082 / 150000 | consumed_samples: 133248 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785591E+05 | tokens_per_sec_per_gpu: 9.039986E+03 | global_batch_size: 64 | lm_loss: 1.082768E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1703080.625 [ip-26-0-154-121:0]:06/21/2023 17:44:29 [INFO|DP=0|PP=0|TP=0]: iteration: 2083 / 150000 | consumed_samples: 133312 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786136E+05 | tokens_per_sec_per_gpu: 9.040837E+03 | global_batch_size: 64 | lm_loss: 1.082574E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 2056376.875 [ip-26-0-154-121:0]:06/21/2023 17:44:29 [INFO|DP=0|PP=0|TP=0]: iteration: 2084 / 150000 | consumed_samples: 133376 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786719E+05 | tokens_per_sec_per_gpu: 9.041749E+03 | global_batch_size: 64 | lm_loss: 1.082478E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2416163.750 [ip-26-0-154-121:0]:06/21/2023 17:44:30 [INFO|DP=0|PP=0|TP=0]: iteration: 2085 / 150000 | consumed_samples: 133440 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787315E+05 | tokens_per_sec_per_gpu: 9.042679E+03 | global_batch_size: 64 | lm_loss: 1.081993E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3689738.500 [ip-26-0-154-121:0]:06/21/2023 17:44:31 [INFO|DP=0|PP=0|TP=0]: iteration: 2086 / 150000 | consumed_samples: 133504 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784717E+05 | tokens_per_sec_per_gpu: 9.038621E+03 | global_batch_size: 64 | lm_loss: 1.082344E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 9519133.000 [ip-26-0-154-121:0]:06/21/2023 17:44:32 [INFO|DP=0|PP=0|TP=0]: iteration: 2087 / 150000 | consumed_samples: 133568 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785451E+05 | tokens_per_sec_per_gpu: 9.039767E+03 | global_batch_size: 64 | lm_loss: 1.082341E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 8732093.000 [ip-26-0-154-121:0]:06/21/2023 17:44:33 [INFO|DP=0|PP=0|TP=0]: iteration: 2088 / 150000 | consumed_samples: 133632 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786772E+05 | tokens_per_sec_per_gpu: 9.041832E+03 | global_batch_size: 64 | lm_loss: 1.082280E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 5483699.000 [ip-26-0-154-121:0]:06/21/2023 17:44:34 [INFO|DP=0|PP=0|TP=0]: iteration: 2089 / 150000 | consumed_samples: 133696 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784337E+05 | tokens_per_sec_per_gpu: 9.038026E+03 | global_batch_size: 64 | lm_loss: 1.082329E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 2811201.250 [ip-26-0-154-121:0]:06/21/2023 17:44:35 [INFO|DP=0|PP=0|TP=0]: iteration: 2090 / 150000 | consumed_samples: 133760 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786628E+05 | tokens_per_sec_per_gpu: 9.041606E+03 | global_batch_size: 64 | lm_loss: 1.082265E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1947434.125 [ip-26-0-154-121:0]:06/21/2023 17:44:36 [INFO|DP=0|PP=0|TP=0]: iteration: 2091 / 150000 | consumed_samples: 133824 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788105E+05 | tokens_per_sec_per_gpu: 9.043914E+03 | global_batch_size: 64 | lm_loss: 1.082363E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 2758043.750 [ip-26-0-154-121:0]:06/21/2023 17:44:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2092 / 150000 | consumed_samples: 133888 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786331E+05 | tokens_per_sec_per_gpu: 9.041142E+03 | global_batch_size: 64 | lm_loss: 1.082365E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3583369.500 [ip-26-0-154-121:0]:06/21/2023 17:44:38 [INFO|DP=0|PP=0|TP=0]: iteration: 2093 / 150000 | consumed_samples: 133952 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787211E+05 | tokens_per_sec_per_gpu: 9.042517E+03 | global_batch_size: 64 | lm_loss: 1.082422E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 2124407.250 [ip-26-0-154-121:0]:06/21/2023 17:44:39 [INFO|DP=0|PP=0|TP=0]: iteration: 2094 / 150000 | consumed_samples: 134016 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783013E+05 | tokens_per_sec_per_gpu: 9.035959E+03 | global_batch_size: 64 | lm_loss: 1.082352E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 1887444.750 [ip-26-0-154-121:0]:06/21/2023 17:44:39 [INFO|DP=0|PP=0|TP=0]: iteration: 2095 / 150000 | consumed_samples: 134080 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783210E+05 | tokens_per_sec_per_gpu: 9.036265E+03 | global_batch_size: 64 | lm_loss: 1.082304E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 1707692.750 [ip-26-0-154-121:0]:06/21/2023 17:44:40 [INFO|DP=0|PP=0|TP=0]: iteration: 2096 / 150000 | consumed_samples: 134144 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783118E+05 | tokens_per_sec_per_gpu: 9.036123E+03 | global_batch_size: 64 | lm_loss: 1.082516E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 1572761.875 [ip-26-0-154-121:0]:06/21/2023 17:44:41 [INFO|DP=0|PP=0|TP=0]: iteration: 2097 / 150000 | consumed_samples: 134208 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783780E+05 | tokens_per_sec_per_gpu: 9.037156E+03 | global_batch_size: 64 | lm_loss: 1.082579E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1531292.250 [ip-26-0-154-121:0]:06/21/2023 17:44:42 [INFO|DP=0|PP=0|TP=0]: iteration: 2098 / 150000 | consumed_samples: 134272 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781236E+05 | tokens_per_sec_per_gpu: 9.033182E+03 | global_batch_size: 64 | lm_loss: 1.082464E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 1491057.375 [ip-26-0-154-121:0]:06/21/2023 17:44:43 [INFO|DP=0|PP=0|TP=0]: iteration: 2099 / 150000 | consumed_samples: 134336 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787360E+05 | tokens_per_sec_per_gpu: 9.042750E+03 | global_batch_size: 64 | lm_loss: 1.082488E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1454773.625 [ip-26-0-154-121:0]:06/21/2023 17:44:44 [INFO|DP=0|PP=0|TP=0]: iteration: 2100 / 150000 | consumed_samples: 134400 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778430E+05 | tokens_per_sec_per_gpu: 9.028797E+03 | global_batch_size: 64 | lm_loss: 1.082383E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 1449581.500 [ip-26-0-154-121:0]:06/21/2023 17:44:45 [INFO|DP=0|PP=0|TP=0]: iteration: 2101 / 150000 | consumed_samples: 134464 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785273E+05 | tokens_per_sec_per_gpu: 9.039489E+03 | global_batch_size: 64 | lm_loss: 1.081675E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1443283.000 [ip-26-0-154-121:0]:06/21/2023 17:44:46 [INFO|DP=0|PP=0|TP=0]: iteration: 2102 / 150000 | consumed_samples: 134528 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777996E+05 | tokens_per_sec_per_gpu: 9.028119E+03 | global_batch_size: 64 | lm_loss: 1.082388E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 1441607.375 [ip-26-0-154-121:0]:06/21/2023 17:44:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2103 / 150000 | consumed_samples: 134592 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780926E+05 | tokens_per_sec_per_gpu: 9.032697E+03 | global_batch_size: 64 | lm_loss: 1.082435E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 1447627.875 [ip-26-0-154-121:0]:06/21/2023 17:44:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2104 / 150000 | consumed_samples: 134656 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783451E+05 | tokens_per_sec_per_gpu: 9.036643E+03 | global_batch_size: 64 | lm_loss: 1.082350E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 1451502.125 [ip-26-0-154-121:0]:06/21/2023 17:44:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2105 / 150000 | consumed_samples: 134720 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.776035E+05 | tokens_per_sec_per_gpu: 9.025055E+03 | global_batch_size: 64 | lm_loss: 1.082358E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 1447105.000 [ip-26-0-154-121:0]:06/21/2023 17:44:49 [INFO|DP=0|PP=0|TP=0]: iteration: 2106 / 150000 | consumed_samples: 134784 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.775902E+05 | tokens_per_sec_per_gpu: 9.024846E+03 | global_batch_size: 64 | lm_loss: 1.082383E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 1438764.875 [ip-26-0-154-121:0]:06/21/2023 17:44:50 [INFO|DP=0|PP=0|TP=0]: iteration: 2107 / 150000 | consumed_samples: 134848 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.778931E+05 | tokens_per_sec_per_gpu: 9.029580E+03 | global_batch_size: 64 | lm_loss: 1.082301E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 1432972.250 [ip-26-0-154-121:0]:06/21/2023 17:44:51 [INFO|DP=0|PP=0|TP=0]: iteration: 2108 / 150000 | consumed_samples: 134912 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784667E+05 | tokens_per_sec_per_gpu: 9.038542E+03 | global_batch_size: 64 | lm_loss: 1.082332E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1438895.125 [ip-26-0-154-121:0]:06/21/2023 17:44:52 [INFO|DP=0|PP=0|TP=0]: iteration: 2109 / 150000 | consumed_samples: 134976 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785372E+05 | tokens_per_sec_per_gpu: 9.039643E+03 | global_batch_size: 64 | lm_loss: 1.082313E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1463178.375 [ip-26-0-154-121:0]:06/21/2023 17:44:53 [INFO|DP=0|PP=0|TP=0]: iteration: 2110 / 150000 | consumed_samples: 135040 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787485E+05 | tokens_per_sec_per_gpu: 9.042945E+03 | global_batch_size: 64 | lm_loss: 1.082353E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1484768.000 [ip-26-0-154-121:0]:06/21/2023 17:44:54 [INFO|DP=0|PP=0|TP=0]: iteration: 2111 / 150000 | consumed_samples: 135104 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782889E+05 | tokens_per_sec_per_gpu: 9.035764E+03 | global_batch_size: 64 | lm_loss: 1.082338E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 1477168.625 [ip-26-0-154-121:0]:06/21/2023 17:44:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2112 / 150000 | consumed_samples: 135168 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787156E+05 | tokens_per_sec_per_gpu: 9.042431E+03 | global_batch_size: 64 | lm_loss: 1.082364E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1439693.125 [ip-26-0-154-121:0]:06/21/2023 17:44:56 [INFO|DP=0|PP=0|TP=0]: iteration: 2113 / 150000 | consumed_samples: 135232 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784410E+05 | tokens_per_sec_per_gpu: 9.038140E+03 | global_batch_size: 64 | lm_loss: 1.082368E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1401461.375 [ip-26-0-154-121:0]:06/21/2023 17:44:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2114 / 150000 | consumed_samples: 135296 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786322E+05 | tokens_per_sec_per_gpu: 9.041128E+03 | global_batch_size: 64 | lm_loss: 1.082222E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1386602.000 [ip-26-0-154-121:0]:06/21/2023 17:44:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2115 / 150000 | consumed_samples: 135360 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788107E+05 | tokens_per_sec_per_gpu: 9.043917E+03 | global_batch_size: 64 | lm_loss: 1.082271E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1385285.125 [ip-26-0-154-121:0]:06/21/2023 17:44:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2116 / 150000 | consumed_samples: 135424 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781479E+05 | tokens_per_sec_per_gpu: 9.033562E+03 | global_batch_size: 64 | lm_loss: 1.082244E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 1386890.125 [ip-26-0-154-121:0]:06/21/2023 17:44:59 [INFO|DP=0|PP=0|TP=0]: iteration: 2117 / 150000 | consumed_samples: 135488 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783094E+05 | tokens_per_sec_per_gpu: 9.036084E+03 | global_batch_size: 64 | lm_loss: 1.082216E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 1388888.500 [ip-26-0-154-121:0]:06/21/2023 17:45:00 [INFO|DP=0|PP=0|TP=0]: iteration: 2118 / 150000 | consumed_samples: 135552 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779747E+05 | tokens_per_sec_per_gpu: 9.030855E+03 | global_batch_size: 64 | lm_loss: 1.081450E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 1384005.625 [ip-26-0-154-121:0]:06/21/2023 17:45:01 [INFO|DP=0|PP=0|TP=0]: iteration: 2119 / 150000 | consumed_samples: 135616 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780063E+05 | tokens_per_sec_per_gpu: 9.031349E+03 | global_batch_size: 64 | lm_loss: 1.082236E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 1377510.750 [ip-26-0-154-121:0]:06/21/2023 17:45:02 [INFO|DP=0|PP=0|TP=0]: iteration: 2120 / 150000 | consumed_samples: 135680 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775249E+05 | tokens_per_sec_per_gpu: 9.023827E+03 | global_batch_size: 64 | lm_loss: 1.082223E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 1368757.375 [ip-26-0-154-121:0]:06/21/2023 17:45:03 [INFO|DP=0|PP=0|TP=0]: iteration: 2121 / 150000 | consumed_samples: 135744 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776941E+05 | tokens_per_sec_per_gpu: 9.026471E+03 | global_batch_size: 64 | lm_loss: 1.082159E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 1363338.375 [ip-26-0-154-121:0]:06/21/2023 17:45:04 [INFO|DP=0|PP=0|TP=0]: iteration: 2122 / 150000 | consumed_samples: 135808 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783762E+05 | tokens_per_sec_per_gpu: 9.037128E+03 | global_batch_size: 64 | lm_loss: 1.082200E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1356603.500 [ip-26-0-154-121:0]:06/21/2023 17:45:05 [INFO|DP=0|PP=0|TP=0]: iteration: 2123 / 150000 | consumed_samples: 135872 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779218E+05 | tokens_per_sec_per_gpu: 9.030029E+03 | global_batch_size: 64 | lm_loss: 1.082226E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 1355295.500 [ip-26-0-154-121:0]:06/21/2023 17:45:06 [INFO|DP=0|PP=0|TP=0]: iteration: 2124 / 150000 | consumed_samples: 135936 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787927E+05 | tokens_per_sec_per_gpu: 9.043636E+03 | global_batch_size: 64 | lm_loss: 1.082082E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1345878.375 [ip-26-0-154-121:0]:06/21/2023 17:45:07 [INFO|DP=0|PP=0|TP=0]: iteration: 2125 / 150000 | consumed_samples: 136000 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779071E+05 | tokens_per_sec_per_gpu: 9.029799E+03 | global_batch_size: 64 | lm_loss: 1.081640E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 1344353.250 [ip-26-0-154-121:0]:06/21/2023 17:45:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2126 / 150000 | consumed_samples: 136064 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780300E+05 | tokens_per_sec_per_gpu: 9.031719E+03 | global_batch_size: 64 | lm_loss: 1.082143E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 1344896.000 [ip-26-0-154-121:0]:06/21/2023 17:45:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2127 / 150000 | consumed_samples: 136128 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.774238E+05 | tokens_per_sec_per_gpu: 9.022247E+03 | global_batch_size: 64 | lm_loss: 1.082170E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 1346295.375 [ip-26-0-154-121:0]:06/21/2023 17:45:09 [INFO|DP=0|PP=0|TP=0]: iteration: 2128 / 150000 | consumed_samples: 136192 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780849E+05 | tokens_per_sec_per_gpu: 9.032576E+03 | global_batch_size: 64 | lm_loss: 1.082190E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 1344049.250 [ip-26-0-154-121:0]:06/21/2023 17:45:10 [INFO|DP=0|PP=0|TP=0]: iteration: 2129 / 150000 | consumed_samples: 136256 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782454E+05 | tokens_per_sec_per_gpu: 9.035084E+03 | global_batch_size: 64 | lm_loss: 1.080838E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 1338502.250 [ip-26-0-154-121:0]:06/21/2023 17:45:11 [INFO|DP=0|PP=0|TP=0]: iteration: 2130 / 150000 | consumed_samples: 136320 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779791E+05 | tokens_per_sec_per_gpu: 9.030924E+03 | global_batch_size: 64 | lm_loss: 1.082158E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 1332358.375 [ip-26-0-154-121:0]:06/21/2023 17:45:12 [INFO|DP=0|PP=0|TP=0]: iteration: 2131 / 150000 | consumed_samples: 136384 | elapsed_time_per_iteration_ms: 908.0 | tokens_per_sec: 5.773818E+05 | tokens_per_sec_per_gpu: 9.021591E+03 | global_batch_size: 64 | lm_loss: 1.082115E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.21 | hardware_tflops_per_gpu: 104.21 | grad_norm: 1328321.000 [ip-26-0-154-121:0]:06/21/2023 17:45:13 [INFO|DP=0|PP=0|TP=0]: iteration: 2132 / 150000 | consumed_samples: 136448 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782963E+05 | tokens_per_sec_per_gpu: 9.035880E+03 | global_batch_size: 64 | lm_loss: 1.082155E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 1325392.375 [ip-26-0-154-121:0]:06/21/2023 17:45:14 [INFO|DP=0|PP=0|TP=0]: iteration: 2133 / 150000 | consumed_samples: 136512 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781694E+05 | tokens_per_sec_per_gpu: 9.033896E+03 | global_batch_size: 64 | lm_loss: 1.081888E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 1323007.750 [ip-26-0-154-121:0]:06/21/2023 17:45:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2134 / 150000 | consumed_samples: 136576 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778247E+05 | tokens_per_sec_per_gpu: 9.028510E+03 | global_batch_size: 64 | lm_loss: 1.082084E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 1320541.625 [ip-26-0-154-121:0]:06/21/2023 17:45:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2135 / 150000 | consumed_samples: 136640 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774582E+05 | tokens_per_sec_per_gpu: 9.022785E+03 | global_batch_size: 64 | lm_loss: 1.082145E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.22 | hardware_tflops_per_gpu: 104.22 | grad_norm: 1318122.375 [ip-26-0-154-121:0]:06/21/2023 17:45:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2136 / 150000 | consumed_samples: 136704 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790230E+05 | tokens_per_sec_per_gpu: 9.047234E+03 | global_batch_size: 64 | lm_loss: 1.082103E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 1316281.375 [ip-26-0-154-121:0]:06/21/2023 17:45:18 [INFO|DP=0|PP=0|TP=0]: iteration: 2137 / 150000 | consumed_samples: 136768 | elapsed_time_per_iteration_ms: 908.1 | tokens_per_sec: 5.773616E+05 | tokens_per_sec_per_gpu: 9.021276E+03 | global_batch_size: 64 | lm_loss: 1.081888E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.21 | hardware_tflops_per_gpu: 104.21 | grad_norm: 1316233.750 [ip-26-0-154-121:0]:06/21/2023 17:45:18 [INFO|DP=0|PP=0|TP=0]: iteration: 2138 / 150000 | consumed_samples: 136832 | elapsed_time_per_iteration_ms: 908.3 | tokens_per_sec: 5.772231E+05 | tokens_per_sec_per_gpu: 9.019111E+03 | global_batch_size: 64 | lm_loss: 1.082055E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.18 | hardware_tflops_per_gpu: 104.18 | grad_norm: 1314873.750 [ip-26-0-154-121:0]:06/21/2023 17:45:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2139 / 150000 | consumed_samples: 136896 | elapsed_time_per_iteration_ms: 908.2 | tokens_per_sec: 5.773112E+05 | tokens_per_sec_per_gpu: 9.020487E+03 | global_batch_size: 64 | lm_loss: 1.082063E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.20 | hardware_tflops_per_gpu: 104.20 | grad_norm: 1310326.125 [ip-26-0-154-121:0]:06/21/2023 17:45:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2140 / 150000 | consumed_samples: 136960 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780016E+05 | tokens_per_sec_per_gpu: 9.031275E+03 | global_batch_size: 64 | lm_loss: 1.082052E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 1308410.625 [ip-26-0-154-121:0]:06/21/2023 17:45:21 [INFO|DP=0|PP=0|TP=0]: iteration: 2141 / 150000 | consumed_samples: 137024 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780121E+05 | tokens_per_sec_per_gpu: 9.031439E+03 | global_batch_size: 64 | lm_loss: 1.082045E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 1307021.625 [ip-26-0-154-121:0]:06/21/2023 17:45:22 [INFO|DP=0|PP=0|TP=0]: iteration: 2142 / 150000 | consumed_samples: 137088 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780715E+05 | tokens_per_sec_per_gpu: 9.032367E+03 | global_batch_size: 64 | lm_loss: 1.082016E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 1303226.000 [ip-26-0-154-121:0]:06/21/2023 17:45:23 [INFO|DP=0|PP=0|TP=0]: iteration: 2143 / 150000 | consumed_samples: 137152 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776483E+05 | tokens_per_sec_per_gpu: 9.025754E+03 | global_batch_size: 64 | lm_loss: 1.082050E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 1300881.000 [ip-26-0-154-121:0]:06/21/2023 17:45:24 [INFO|DP=0|PP=0|TP=0]: iteration: 2144 / 150000 | consumed_samples: 137216 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782296E+05 | tokens_per_sec_per_gpu: 9.034837E+03 | global_batch_size: 64 | lm_loss: 1.081776E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 1296226.625 [ip-26-0-154-121:0]:06/21/2023 17:45:25 [INFO|DP=0|PP=0|TP=0]: iteration: 2145 / 150000 | consumed_samples: 137280 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781422E+05 | tokens_per_sec_per_gpu: 9.033471E+03 | global_batch_size: 64 | lm_loss: 1.082024E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 1293477.500 [ip-26-0-154-121:0]:06/21/2023 17:45:26 [INFO|DP=0|PP=0|TP=0]: iteration: 2146 / 150000 | consumed_samples: 137344 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782063E+05 | tokens_per_sec_per_gpu: 9.034474E+03 | global_batch_size: 64 | lm_loss: 1.082019E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 1292584.750 [ip-26-0-154-121:0]:06/21/2023 17:45:27 [INFO|DP=0|PP=0|TP=0]: iteration: 2147 / 150000 | consumed_samples: 137408 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785402E+05 | tokens_per_sec_per_gpu: 9.039691E+03 | global_batch_size: 64 | lm_loss: 1.082007E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1292256.500 [ip-26-0-154-121:0]:06/21/2023 17:45:28 [INFO|DP=0|PP=0|TP=0]: iteration: 2148 / 150000 | consumed_samples: 137472 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781671E+05 | tokens_per_sec_per_gpu: 9.033861E+03 | global_batch_size: 64 | lm_loss: 1.081997E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 1292337.375 [ip-26-0-154-121:0]:06/21/2023 17:45:28 [INFO|DP=0|PP=0|TP=0]: iteration: 2149 / 150000 | consumed_samples: 137536 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785025E+05 | tokens_per_sec_per_gpu: 9.039101E+03 | global_batch_size: 64 | lm_loss: 1.081985E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1287911.875 [ip-26-0-154-121:0]:06/21/2023 17:45:29 [INFO|DP=0|PP=0|TP=0]: iteration: 2150 / 150000 | consumed_samples: 137600 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788216E+05 | tokens_per_sec_per_gpu: 9.044088E+03 | global_batch_size: 64 | lm_loss: 1.081930E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1287503.375 [ip-26-0-154-121:0]:06/21/2023 17:45:30 [INFO|DP=0|PP=0|TP=0]: iteration: 2151 / 150000 | consumed_samples: 137664 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783952E+05 | tokens_per_sec_per_gpu: 9.037425E+03 | global_batch_size: 64 | lm_loss: 1.081944E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1284861.125 [ip-26-0-154-121:0]:06/21/2023 17:45:31 [INFO|DP=0|PP=0|TP=0]: iteration: 2152 / 150000 | consumed_samples: 137728 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786915E+05 | tokens_per_sec_per_gpu: 9.042055E+03 | global_batch_size: 64 | lm_loss: 1.081949E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1286280.250 [ip-26-0-154-121:0]:06/21/2023 17:45:32 [INFO|DP=0|PP=0|TP=0]: iteration: 2153 / 150000 | consumed_samples: 137792 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779951E+05 | tokens_per_sec_per_gpu: 9.031173E+03 | global_batch_size: 64 | lm_loss: 1.081925E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 1283781.000 [ip-26-0-154-121:0]:06/21/2023 17:45:33 [INFO|DP=0|PP=0|TP=0]: iteration: 2154 / 150000 | consumed_samples: 137856 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788710E+05 | tokens_per_sec_per_gpu: 9.044859E+03 | global_batch_size: 64 | lm_loss: 1.081958E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1282593.500 [ip-26-0-154-121:0]:06/21/2023 17:45:34 [INFO|DP=0|PP=0|TP=0]: iteration: 2155 / 150000 | consumed_samples: 137920 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779469E+05 | tokens_per_sec_per_gpu: 9.030420E+03 | global_batch_size: 64 | lm_loss: 1.081975E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 1287207.000 [ip-26-0-154-121:0]:06/21/2023 17:45:35 [INFO|DP=0|PP=0|TP=0]: iteration: 2156 / 150000 | consumed_samples: 137984 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783155E+05 | tokens_per_sec_per_gpu: 9.036180E+03 | global_batch_size: 64 | lm_loss: 1.081849E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 1288644.500 [ip-26-0-154-121:0]:06/21/2023 17:45:36 [INFO|DP=0|PP=0|TP=0]: iteration: 2157 / 150000 | consumed_samples: 138048 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783339E+05 | tokens_per_sec_per_gpu: 9.036467E+03 | global_batch_size: 64 | lm_loss: 1.081961E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 1289081.125 [ip-26-0-154-121:0]:06/21/2023 17:45:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2158 / 150000 | consumed_samples: 138112 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.783995E+05 | tokens_per_sec_per_gpu: 9.037492E+03 | global_batch_size: 64 | lm_loss: 1.081942E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1300223.375 [ip-26-0-154-121:0]:06/21/2023 17:45:38 [INFO|DP=0|PP=0|TP=0]: iteration: 2159 / 150000 | consumed_samples: 138176 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778576E+05 | tokens_per_sec_per_gpu: 9.029025E+03 | global_batch_size: 64 | lm_loss: 1.081942E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 1308982.125 [ip-26-0-154-121:0]:06/21/2023 17:45:38 [INFO|DP=0|PP=0|TP=0]: iteration: 2160 / 150000 | consumed_samples: 138240 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780557E+05 | tokens_per_sec_per_gpu: 9.032120E+03 | global_batch_size: 64 | lm_loss: 1.081917E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 1301221.375 [ip-26-0-154-121:0]:06/21/2023 17:45:39 [INFO|DP=0|PP=0|TP=0]: iteration: 2161 / 150000 | consumed_samples: 138304 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774775E+05 | tokens_per_sec_per_gpu: 9.023086E+03 | global_batch_size: 64 | lm_loss: 1.081918E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 1319685.250 [ip-26-0-154-121:0]:06/21/2023 17:45:40 [INFO|DP=0|PP=0|TP=0]: iteration: 2162 / 150000 | consumed_samples: 138368 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774703E+05 | tokens_per_sec_per_gpu: 9.022974E+03 | global_batch_size: 64 | lm_loss: 1.081884E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 1319336.875 [ip-26-0-154-121:0]:06/21/2023 17:45:41 [INFO|DP=0|PP=0|TP=0]: iteration: 2163 / 150000 | consumed_samples: 138432 | elapsed_time_per_iteration_ms: 908.2 | tokens_per_sec: 5.773015E+05 | tokens_per_sec_per_gpu: 9.020335E+03 | global_batch_size: 64 | lm_loss: 1.081883E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.19 | hardware_tflops_per_gpu: 104.19 | grad_norm: 1320383.875 [ip-26-0-154-121:0]:06/21/2023 17:45:42 [INFO|DP=0|PP=0|TP=0]: iteration: 2164 / 150000 | consumed_samples: 138496 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774810E+05 | tokens_per_sec_per_gpu: 9.023140E+03 | global_batch_size: 64 | lm_loss: 1.081842E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 1305236.750 [ip-26-0-154-121:0]:06/21/2023 17:45:43 [INFO|DP=0|PP=0|TP=0]: iteration: 2165 / 150000 | consumed_samples: 138560 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.778909E+05 | tokens_per_sec_per_gpu: 9.029545E+03 | global_batch_size: 64 | lm_loss: 1.081856E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 1297583.750 [ip-26-0-154-121:0]:06/21/2023 17:45:44 [INFO|DP=0|PP=0|TP=0]: iteration: 2166 / 150000 | consumed_samples: 138624 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781686E+05 | tokens_per_sec_per_gpu: 9.033885E+03 | global_batch_size: 64 | lm_loss: 1.081831E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 1304972.250 [ip-26-0-154-121:0]:06/21/2023 17:45:45 [INFO|DP=0|PP=0|TP=0]: iteration: 2167 / 150000 | consumed_samples: 138688 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778865E+05 | tokens_per_sec_per_gpu: 9.029476E+03 | global_batch_size: 64 | lm_loss: 1.081848E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 1324998.625 [ip-26-0-154-121:0]:06/21/2023 17:45:46 [INFO|DP=0|PP=0|TP=0]: iteration: 2168 / 150000 | consumed_samples: 138752 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778558E+05 | tokens_per_sec_per_gpu: 9.028997E+03 | global_batch_size: 64 | lm_loss: 1.081513E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 1349546.250 [ip-26-0-154-121:0]:06/21/2023 17:45:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2169 / 150000 | consumed_samples: 138816 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780498E+05 | tokens_per_sec_per_gpu: 9.032028E+03 | global_batch_size: 64 | lm_loss: 1.081858E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 1387112.875 [ip-26-0-154-121:0]:06/21/2023 17:45:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2170 / 150000 | consumed_samples: 138880 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784122E+05 | tokens_per_sec_per_gpu: 9.037691E+03 | global_batch_size: 64 | lm_loss: 1.081781E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1420152.750 [ip-26-0-154-121:0]:06/21/2023 17:45:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2171 / 150000 | consumed_samples: 138944 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780106E+05 | tokens_per_sec_per_gpu: 9.031415E+03 | global_batch_size: 64 | lm_loss: 1.081709E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 1495267.000 [ip-26-0-154-121:0]:06/21/2023 17:45:49 [INFO|DP=0|PP=0|TP=0]: iteration: 2172 / 150000 | consumed_samples: 139008 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783211E+05 | tokens_per_sec_per_gpu: 9.036267E+03 | global_batch_size: 64 | lm_loss: 1.081842E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 1761215.250 [ip-26-0-154-121:0]:06/21/2023 17:45:50 [INFO|DP=0|PP=0|TP=0]: iteration: 2173 / 150000 | consumed_samples: 139072 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.776066E+05 | tokens_per_sec_per_gpu: 9.025102E+03 | global_batch_size: 64 | lm_loss: 1.081835E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 2888199.500 [ip-26-0-154-121:0]:06/21/2023 17:45:51 [INFO|DP=0|PP=0|TP=0]: iteration: 2174 / 150000 | consumed_samples: 139136 | elapsed_time_per_iteration_ms: 908.1 | tokens_per_sec: 5.773551E+05 | tokens_per_sec_per_gpu: 9.021174E+03 | global_batch_size: 64 | lm_loss: 1.081802E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.20 | hardware_tflops_per_gpu: 104.20 | grad_norm: 24395578.000 [ip-26-0-154-121:0]:06/21/2023 17:45:52 [INFO|DP=0|PP=0|TP=0]: iteration: 2175 / 150000 | consumed_samples: 139200 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775268E+05 | tokens_per_sec_per_gpu: 9.023856E+03 | global_batch_size: 64 | lm_loss: 1.081843E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 6049989.500 [ip-26-0-154-121:0]:06/21/2023 17:45:53 [INFO|DP=0|PP=0|TP=0]: iteration: 2176 / 150000 | consumed_samples: 139264 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777349E+05 | tokens_per_sec_per_gpu: 9.027108E+03 | global_batch_size: 64 | lm_loss: 1.081786E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 47899196.000 [ip-26-0-154-121:0]:06/21/2023 17:45:54 [INFO|DP=0|PP=0|TP=0]: iteration: 2177 / 150000 | consumed_samples: 139328 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780254E+05 | tokens_per_sec_per_gpu: 9.031648E+03 | global_batch_size: 64 | lm_loss: 1.081747E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 1556888.500 [ip-26-0-154-121:0]:06/21/2023 17:45:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2178 / 150000 | consumed_samples: 139392 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778544E+05 | tokens_per_sec_per_gpu: 9.028975E+03 | global_batch_size: 64 | lm_loss: 1.081794E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 2109610.500 [ip-26-0-154-121:0]:06/21/2023 17:45:56 [INFO|DP=0|PP=0|TP=0]: iteration: 2179 / 150000 | consumed_samples: 139456 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777518E+05 | tokens_per_sec_per_gpu: 9.027372E+03 | global_batch_size: 64 | lm_loss: 1.081821E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 42685040.000 [ip-26-0-154-121:0]:06/21/2023 17:45:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2180 / 150000 | consumed_samples: 139520 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779644E+05 | tokens_per_sec_per_gpu: 9.030693E+03 | global_batch_size: 64 | lm_loss: 1.081811E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 16585812.000 [ip-26-0-154-121:0]:06/21/2023 17:45:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2181 / 150000 | consumed_samples: 139584 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780698E+05 | tokens_per_sec_per_gpu: 9.032341E+03 | global_batch_size: 64 | lm_loss: 1.081800E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 1600525.000 [ip-26-0-154-121:0]:06/21/2023 17:45:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2182 / 150000 | consumed_samples: 139648 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781446E+05 | tokens_per_sec_per_gpu: 9.033509E+03 | global_batch_size: 64 | lm_loss: 1.081797E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 1623303.250 [ip-26-0-154-121:0]:06/21/2023 17:45:59 [INFO|DP=0|PP=0|TP=0]: iteration: 2183 / 150000 | consumed_samples: 139712 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779856E+05 | tokens_per_sec_per_gpu: 9.031026E+03 | global_batch_size: 64 | lm_loss: 1.081813E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 16411913.000 [ip-26-0-154-121:0]:06/21/2023 17:46:00 [INFO|DP=0|PP=0|TP=0]: iteration: 2184 / 150000 | consumed_samples: 139776 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782396E+05 | tokens_per_sec_per_gpu: 9.034994E+03 | global_batch_size: 64 | lm_loss: 1.081815E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 9896587.000 [ip-26-0-154-121:0]:06/21/2023 17:46:01 [INFO|DP=0|PP=0|TP=0]: iteration: 2185 / 150000 | consumed_samples: 139840 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783774E+05 | tokens_per_sec_per_gpu: 9.037147E+03 | global_batch_size: 64 | lm_loss: 1.081784E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 19731830.000 [ip-26-0-154-121:0]:06/21/2023 17:46:02 [INFO|DP=0|PP=0|TP=0]: iteration: 2186 / 150000 | consumed_samples: 139904 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783079E+05 | tokens_per_sec_per_gpu: 9.036061E+03 | global_batch_size: 64 | lm_loss: 1.081747E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 18794196.000 [ip-26-0-154-121:0]:06/21/2023 17:46:03 [INFO|DP=0|PP=0|TP=0]: iteration: 2187 / 150000 | consumed_samples: 139968 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781606E+05 | tokens_per_sec_per_gpu: 9.033759E+03 | global_batch_size: 64 | lm_loss: 1.081606E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 2054739.750 [ip-26-0-154-121:0]:06/21/2023 17:46:04 [INFO|DP=0|PP=0|TP=0]: iteration: 2188 / 150000 | consumed_samples: 140032 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781332E+05 | tokens_per_sec_per_gpu: 9.033331E+03 | global_batch_size: 64 | lm_loss: 1.081702E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 1468297.500 [ip-26-0-154-121:0]:06/21/2023 17:46:05 [INFO|DP=0|PP=0|TP=0]: iteration: 2189 / 150000 | consumed_samples: 140096 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780938E+05 | tokens_per_sec_per_gpu: 9.032716E+03 | global_batch_size: 64 | lm_loss: 1.081783E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 33907380.000 [ip-26-0-154-121:0]:06/21/2023 17:46:06 [INFO|DP=0|PP=0|TP=0]: iteration: 2190 / 150000 | consumed_samples: 140160 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778395E+05 | tokens_per_sec_per_gpu: 9.028743E+03 | global_batch_size: 64 | lm_loss: 1.081782E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 5015576.000 [ip-26-0-154-121:0]:06/21/2023 17:46:07 [INFO|DP=0|PP=0|TP=0]: iteration: 2191 / 150000 | consumed_samples: 140224 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779255E+05 | tokens_per_sec_per_gpu: 9.030086E+03 | global_batch_size: 64 | lm_loss: 1.081826E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4756291.000 [ip-26-0-154-121:0]:06/21/2023 17:46:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2192 / 150000 | consumed_samples: 140288 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779431E+05 | tokens_per_sec_per_gpu: 9.030361E+03 | global_batch_size: 64 | lm_loss: 1.081856E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 7478842.000 [ip-26-0-154-121:0]:06/21/2023 17:46:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2193 / 150000 | consumed_samples: 140352 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780630E+05 | tokens_per_sec_per_gpu: 9.032234E+03 | global_batch_size: 64 | lm_loss: 1.081874E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 11929028.000 [ip-26-0-154-121:0]:06/21/2023 17:46:09 [INFO|DP=0|PP=0|TP=0]: iteration: 2194 / 150000 | consumed_samples: 140416 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784810E+05 | tokens_per_sec_per_gpu: 9.038766E+03 | global_batch_size: 64 | lm_loss: 1.081708E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 12843712.000 [ip-26-0-154-121:0]:06/21/2023 17:46:10 [INFO|DP=0|PP=0|TP=0]: iteration: 2195 / 150000 | consumed_samples: 140480 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780879E+05 | tokens_per_sec_per_gpu: 9.032623E+03 | global_batch_size: 64 | lm_loss: 1.081800E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 9994986.000 [ip-26-0-154-121:0]:06/21/2023 17:46:11 [INFO|DP=0|PP=0|TP=0]: iteration: 2196 / 150000 | consumed_samples: 140544 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.775836E+05 | tokens_per_sec_per_gpu: 9.024745E+03 | global_batch_size: 64 | lm_loss: 1.081717E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 5892155.500 [ip-26-0-154-121:0]:06/21/2023 17:46:12 [INFO|DP=0|PP=0|TP=0]: iteration: 2197 / 150000 | consumed_samples: 140608 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781555E+05 | tokens_per_sec_per_gpu: 9.033680E+03 | global_batch_size: 64 | lm_loss: 1.081730E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 4601023.000 [ip-26-0-154-121:0]:06/21/2023 17:46:13 [INFO|DP=0|PP=0|TP=0]: iteration: 2198 / 150000 | consumed_samples: 140672 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783880E+05 | tokens_per_sec_per_gpu: 9.037313E+03 | global_batch_size: 64 | lm_loss: 1.081645E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 11267401.000 [ip-26-0-154-121:0]:06/21/2023 17:46:14 [INFO|DP=0|PP=0|TP=0]: iteration: 2199 / 150000 | consumed_samples: 140736 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781201E+05 | tokens_per_sec_per_gpu: 9.033127E+03 | global_batch_size: 64 | lm_loss: 1.081612E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 22406116.000 [ip-26-0-154-121:0]:06/21/2023 17:46:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2200 / 150000 | consumed_samples: 140800 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784074E+05 | tokens_per_sec_per_gpu: 9.037615E+03 | global_batch_size: 64 | lm_loss: 1.081586E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 36992600.000 [ip-26-0-154-121:0]:06/21/2023 17:46:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2201 / 150000 | consumed_samples: 140864 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784205E+05 | tokens_per_sec_per_gpu: 9.037820E+03 | global_batch_size: 64 | lm_loss: 1.081564E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 42292516.000 [ip-26-0-154-121:0]:06/21/2023 17:46:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2202 / 150000 | consumed_samples: 140928 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782201E+05 | tokens_per_sec_per_gpu: 9.034690E+03 | global_batch_size: 64 | lm_loss: 1.081570E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 41509296.000 [ip-26-0-154-121:0]:06/21/2023 17:46:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2203 / 150000 | consumed_samples: 140992 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782569E+05 | tokens_per_sec_per_gpu: 9.035265E+03 | global_batch_size: 64 | lm_loss: 1.081556E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 26161456.000 [ip-26-0-154-121:0]:06/21/2023 17:46:18 [INFO|DP=0|PP=0|TP=0]: iteration: 2204 / 150000 | consumed_samples: 141056 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780683E+05 | tokens_per_sec_per_gpu: 9.032317E+03 | global_batch_size: 64 | lm_loss: 1.081542E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 11383339.000 [ip-26-0-154-121:0]:06/21/2023 17:46:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2205 / 150000 | consumed_samples: 141120 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782419E+05 | tokens_per_sec_per_gpu: 9.035030E+03 | global_batch_size: 64 | lm_loss: 1.081582E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 4320819.500 [ip-26-0-154-121:0]:06/21/2023 17:46:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2206 / 150000 | consumed_samples: 141184 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776554E+05 | tokens_per_sec_per_gpu: 9.025866E+03 | global_batch_size: 64 | lm_loss: 1.081642E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 4443960.500 [ip-26-0-154-121:0]:06/21/2023 17:46:21 [INFO|DP=0|PP=0|TP=0]: iteration: 2207 / 150000 | consumed_samples: 141248 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779506E+05 | tokens_per_sec_per_gpu: 9.030477E+03 | global_batch_size: 64 | lm_loss: 1.081729E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 17501542.000 [ip-26-0-154-121:0]:06/21/2023 17:46:22 [INFO|DP=0|PP=0|TP=0]: iteration: 2208 / 150000 | consumed_samples: 141312 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778297E+05 | tokens_per_sec_per_gpu: 9.028589E+03 | global_batch_size: 64 | lm_loss: 1.081852E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 11446801.000 [ip-26-0-154-121:0]:06/21/2023 17:46:23 [INFO|DP=0|PP=0|TP=0]: iteration: 2209 / 150000 | consumed_samples: 141376 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775468E+05 | tokens_per_sec_per_gpu: 9.024169E+03 | global_batch_size: 64 | lm_loss: 1.081866E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 5620827.000 [ip-26-0-154-121:0]:06/21/2023 17:46:24 [INFO|DP=0|PP=0|TP=0]: iteration: 2210 / 150000 | consumed_samples: 141440 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.776962E+05 | tokens_per_sec_per_gpu: 9.026504E+03 | global_batch_size: 64 | lm_loss: 1.081741E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 6386893.000 [ip-26-0-154-121:0]:06/21/2023 17:46:25 [INFO|DP=0|PP=0|TP=0]: iteration: 2211 / 150000 | consumed_samples: 141504 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781195E+05 | tokens_per_sec_per_gpu: 9.033117E+03 | global_batch_size: 64 | lm_loss: 1.081818E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 12197534.000 [ip-26-0-154-121:0]:06/21/2023 17:46:26 [INFO|DP=0|PP=0|TP=0]: iteration: 2212 / 150000 | consumed_samples: 141568 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777730E+05 | tokens_per_sec_per_gpu: 9.027704E+03 | global_batch_size: 64 | lm_loss: 1.081775E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 20691906.000 [ip-26-0-154-121:0]:06/21/2023 17:46:27 [INFO|DP=0|PP=0|TP=0]: iteration: 2213 / 150000 | consumed_samples: 141632 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783348E+05 | tokens_per_sec_per_gpu: 9.036481E+03 | global_batch_size: 64 | lm_loss: 1.081680E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 5074008.000 [ip-26-0-154-121:0]:06/21/2023 17:46:27 [INFO|DP=0|PP=0|TP=0]: iteration: 2214 / 150000 | consumed_samples: 141696 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777503E+05 | tokens_per_sec_per_gpu: 9.027348E+03 | global_batch_size: 64 | lm_loss: 1.081640E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 2286013.250 [ip-26-0-154-121:0]:06/21/2023 17:46:28 [INFO|DP=0|PP=0|TP=0]: iteration: 2215 / 150000 | consumed_samples: 141760 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779811E+05 | tokens_per_sec_per_gpu: 9.030954E+03 | global_batch_size: 64 | lm_loss: 1.081664E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 2224754.000 [ip-26-0-154-121:0]:06/21/2023 17:46:29 [INFO|DP=0|PP=0|TP=0]: iteration: 2216 / 150000 | consumed_samples: 141824 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786923E+05 | tokens_per_sec_per_gpu: 9.042067E+03 | global_batch_size: 64 | lm_loss: 1.081694E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 7679804.000 [ip-26-0-154-121:0]:06/21/2023 17:46:30 [INFO|DP=0|PP=0|TP=0]: iteration: 2217 / 150000 | consumed_samples: 141888 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783698E+05 | tokens_per_sec_per_gpu: 9.037028E+03 | global_batch_size: 64 | lm_loss: 1.081018E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 14790894.000 [ip-26-0-154-121:0]:06/21/2023 17:46:31 [INFO|DP=0|PP=0|TP=0]: iteration: 2218 / 150000 | consumed_samples: 141952 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784341E+05 | tokens_per_sec_per_gpu: 9.038034E+03 | global_batch_size: 64 | lm_loss: 1.081738E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 9025369.000 [ip-26-0-154-121:0]:06/21/2023 17:46:32 [INFO|DP=0|PP=0|TP=0]: iteration: 2219 / 150000 | consumed_samples: 142016 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785166E+05 | tokens_per_sec_per_gpu: 9.039322E+03 | global_batch_size: 64 | lm_loss: 1.081710E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 2390769.750 [ip-26-0-154-121:0]:06/21/2023 17:46:33 [INFO|DP=0|PP=0|TP=0]: iteration: 2220 / 150000 | consumed_samples: 142080 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787278E+05 | tokens_per_sec_per_gpu: 9.042622E+03 | global_batch_size: 64 | lm_loss: 1.081733E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3156054.750 [ip-26-0-154-121:0]:06/21/2023 17:46:34 [INFO|DP=0|PP=0|TP=0]: iteration: 2221 / 150000 | consumed_samples: 142144 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786652E+05 | tokens_per_sec_per_gpu: 9.041644E+03 | global_batch_size: 64 | lm_loss: 1.081663E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3707209.750 [ip-26-0-154-121:0]:06/21/2023 17:46:35 [INFO|DP=0|PP=0|TP=0]: iteration: 2222 / 150000 | consumed_samples: 142208 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787062E+05 | tokens_per_sec_per_gpu: 9.042284E+03 | global_batch_size: 64 | lm_loss: 1.081285E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 5187673.000 [ip-26-0-154-121:0]:06/21/2023 17:46:36 [INFO|DP=0|PP=0|TP=0]: iteration: 2223 / 150000 | consumed_samples: 142272 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780431E+05 | tokens_per_sec_per_gpu: 9.031923E+03 | global_batch_size: 64 | lm_loss: 1.081552E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 6957446.000 [ip-26-0-154-121:0]:06/21/2023 17:46:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2224 / 150000 | consumed_samples: 142336 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784830E+05 | tokens_per_sec_per_gpu: 9.038797E+03 | global_batch_size: 64 | lm_loss: 1.081556E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 4247504.500 [ip-26-0-154-121:0]:06/21/2023 17:46:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2225 / 150000 | consumed_samples: 142400 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785588E+05 | tokens_per_sec_per_gpu: 9.039981E+03 | global_batch_size: 64 | lm_loss: 1.081532E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 2045816.375 [ip-26-0-154-121:0]:06/21/2023 17:46:38 [INFO|DP=0|PP=0|TP=0]: iteration: 2226 / 150000 | consumed_samples: 142464 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787896E+05 | tokens_per_sec_per_gpu: 9.043588E+03 | global_batch_size: 64 | lm_loss: 1.081516E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1542059.750 [ip-26-0-154-121:0]:06/21/2023 17:46:39 [INFO|DP=0|PP=0|TP=0]: iteration: 2227 / 150000 | consumed_samples: 142528 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787890E+05 | tokens_per_sec_per_gpu: 9.043579E+03 | global_batch_size: 64 | lm_loss: 1.081573E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1273904.375 [ip-26-0-154-121:0]:06/21/2023 17:46:40 [INFO|DP=0|PP=0|TP=0]: iteration: 2228 / 150000 | consumed_samples: 142592 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791093E+05 | tokens_per_sec_per_gpu: 9.048582E+03 | global_batch_size: 64 | lm_loss: 1.081539E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 2238244.750 [ip-26-0-154-121:0]:06/21/2023 17:46:41 [INFO|DP=0|PP=0|TP=0]: iteration: 2229 / 150000 | consumed_samples: 142656 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788424E+05 | tokens_per_sec_per_gpu: 9.044412E+03 | global_batch_size: 64 | lm_loss: 1.081613E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 3676157.250 [ip-26-0-154-121:0]:06/21/2023 17:46:42 [INFO|DP=0|PP=0|TP=0]: iteration: 2230 / 150000 | consumed_samples: 142720 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787672E+05 | tokens_per_sec_per_gpu: 9.043238E+03 | global_batch_size: 64 | lm_loss: 1.081617E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 2970588.500 [ip-26-0-154-121:0]:06/21/2023 17:46:43 [INFO|DP=0|PP=0|TP=0]: iteration: 2231 / 150000 | consumed_samples: 142784 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789405E+05 | tokens_per_sec_per_gpu: 9.045945E+03 | global_batch_size: 64 | lm_loss: 1.081633E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 1779730.500 [ip-26-0-154-121:0]:06/21/2023 17:46:44 [INFO|DP=0|PP=0|TP=0]: iteration: 2232 / 150000 | consumed_samples: 142848 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786859E+05 | tokens_per_sec_per_gpu: 9.041967E+03 | global_batch_size: 64 | lm_loss: 1.081639E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1314467.375 [ip-26-0-154-121:0]:06/21/2023 17:46:45 [INFO|DP=0|PP=0|TP=0]: iteration: 2233 / 150000 | consumed_samples: 142912 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789138E+05 | tokens_per_sec_per_gpu: 9.045528E+03 | global_batch_size: 64 | lm_loss: 1.081638E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 1211791.875 [ip-26-0-154-121:0]:06/21/2023 17:46:46 [INFO|DP=0|PP=0|TP=0]: iteration: 2234 / 150000 | consumed_samples: 142976 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789140E+05 | tokens_per_sec_per_gpu: 9.045531E+03 | global_batch_size: 64 | lm_loss: 1.081642E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 1219522.750 [ip-26-0-154-121:0]:06/21/2023 17:46:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2235 / 150000 | consumed_samples: 143040 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787887E+05 | tokens_per_sec_per_gpu: 9.043574E+03 | global_batch_size: 64 | lm_loss: 1.081614E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1287727.250 [ip-26-0-154-121:0]:06/21/2023 17:46:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2236 / 150000 | consumed_samples: 143104 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784853E+05 | tokens_per_sec_per_gpu: 9.038832E+03 | global_batch_size: 64 | lm_loss: 1.081583E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1282694.625 [ip-26-0-154-121:0]:06/21/2023 17:46:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2237 / 150000 | consumed_samples: 143168 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788521E+05 | tokens_per_sec_per_gpu: 9.044564E+03 | global_batch_size: 64 | lm_loss: 1.081488E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1207795.125 [ip-26-0-154-121:0]:06/21/2023 17:46:49 [INFO|DP=0|PP=0|TP=0]: iteration: 2238 / 150000 | consumed_samples: 143232 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786928E+05 | tokens_per_sec_per_gpu: 9.042074E+03 | global_batch_size: 64 | lm_loss: 1.081517E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1158239.375 [ip-26-0-154-121:0]:06/21/2023 17:46:50 [INFO|DP=0|PP=0|TP=0]: iteration: 2239 / 150000 | consumed_samples: 143296 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785539E+05 | tokens_per_sec_per_gpu: 9.039905E+03 | global_batch_size: 64 | lm_loss: 1.081517E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1161836.750 [ip-26-0-154-121:0]:06/21/2023 17:46:51 [INFO|DP=0|PP=0|TP=0]: iteration: 2240 / 150000 | consumed_samples: 143360 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787936E+05 | tokens_per_sec_per_gpu: 9.043650E+03 | global_batch_size: 64 | lm_loss: 1.081551E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1186511.500 [ip-26-0-154-121:0]:06/21/2023 17:46:52 [INFO|DP=0|PP=0|TP=0]: iteration: 2241 / 150000 | consumed_samples: 143424 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784970E+05 | tokens_per_sec_per_gpu: 9.039015E+03 | global_batch_size: 64 | lm_loss: 1.081538E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1177012.250 [ip-26-0-154-121:0]:06/21/2023 17:46:53 [INFO|DP=0|PP=0|TP=0]: iteration: 2242 / 150000 | consumed_samples: 143488 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782901E+05 | tokens_per_sec_per_gpu: 9.035783E+03 | global_batch_size: 64 | lm_loss: 1.081596E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 1155890.500 [ip-26-0-154-121:0]:06/21/2023 17:46:54 [INFO|DP=0|PP=0|TP=0]: iteration: 2243 / 150000 | consumed_samples: 143552 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785197E+05 | tokens_per_sec_per_gpu: 9.039370E+03 | global_batch_size: 64 | lm_loss: 1.081536E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1139341.625 [ip-26-0-154-121:0]:06/21/2023 17:46:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2244 / 150000 | consumed_samples: 143616 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786404E+05 | tokens_per_sec_per_gpu: 9.041256E+03 | global_batch_size: 64 | lm_loss: 1.081499E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1133519.375 [ip-26-0-154-121:0]:06/21/2023 17:46:56 [INFO|DP=0|PP=0|TP=0]: iteration: 2245 / 150000 | consumed_samples: 143680 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789865E+05 | tokens_per_sec_per_gpu: 9.046664E+03 | global_batch_size: 64 | lm_loss: 1.081506E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 1131710.375 [ip-26-0-154-121:0]:06/21/2023 17:46:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2246 / 150000 | consumed_samples: 143744 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783153E+05 | tokens_per_sec_per_gpu: 9.036177E+03 | global_batch_size: 64 | lm_loss: 1.081520E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 1131799.000 [ip-26-0-154-121:0]:06/21/2023 17:46:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2247 / 150000 | consumed_samples: 143808 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784201E+05 | tokens_per_sec_per_gpu: 9.037815E+03 | global_batch_size: 64 | lm_loss: 1.081475E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1139751.875 [ip-26-0-154-121:0]:06/21/2023 17:46:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2248 / 150000 | consumed_samples: 143872 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779493E+05 | tokens_per_sec_per_gpu: 9.030458E+03 | global_batch_size: 64 | lm_loss: 1.081527E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 1151187.625 [ip-26-0-154-121:0]:06/21/2023 17:46:59 [INFO|DP=0|PP=0|TP=0]: iteration: 2249 / 150000 | consumed_samples: 143936 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784434E+05 | tokens_per_sec_per_gpu: 9.038179E+03 | global_batch_size: 64 | lm_loss: 1.081536E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1150900.125 [ip-26-0-154-121:0]:06/21/2023 17:47:00 [INFO|DP=0|PP=0|TP=0]: iteration: 2250 / 150000 | consumed_samples: 144000 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783820E+05 | tokens_per_sec_per_gpu: 9.037218E+03 | global_batch_size: 64 | lm_loss: 1.081538E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1135550.875 [ip-26-0-154-121:0]:06/21/2023 17:47:01 [INFO|DP=0|PP=0|TP=0]: iteration: 2251 / 150000 | consumed_samples: 144064 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782615E+05 | tokens_per_sec_per_gpu: 9.035336E+03 | global_batch_size: 64 | lm_loss: 1.081494E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 1121031.375 [ip-26-0-154-121:0]:06/21/2023 17:47:02 [INFO|DP=0|PP=0|TP=0]: iteration: 2252 / 150000 | consumed_samples: 144128 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786742E+05 | tokens_per_sec_per_gpu: 9.041784E+03 | global_batch_size: 64 | lm_loss: 1.081506E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1123802.375 [ip-26-0-154-121:0]:06/21/2023 17:47:03 [INFO|DP=0|PP=0|TP=0]: iteration: 2253 / 150000 | consumed_samples: 144192 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783873E+05 | tokens_per_sec_per_gpu: 9.037301E+03 | global_batch_size: 64 | lm_loss: 1.081473E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1136467.875 [ip-26-0-154-121:0]:06/21/2023 17:47:04 [INFO|DP=0|PP=0|TP=0]: iteration: 2254 / 150000 | consumed_samples: 144256 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784246E+05 | tokens_per_sec_per_gpu: 9.037884E+03 | global_batch_size: 64 | lm_loss: 1.081366E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1145296.000 [ip-26-0-154-121:0]:06/21/2023 17:47:05 [INFO|DP=0|PP=0|TP=0]: iteration: 2255 / 150000 | consumed_samples: 144320 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787319E+05 | tokens_per_sec_per_gpu: 9.042686E+03 | global_batch_size: 64 | lm_loss: 1.081443E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1146647.000 [ip-26-0-154-121:0]:06/21/2023 17:47:06 [INFO|DP=0|PP=0|TP=0]: iteration: 2256 / 150000 | consumed_samples: 144384 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783406E+05 | tokens_per_sec_per_gpu: 9.036572E+03 | global_batch_size: 64 | lm_loss: 1.081463E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 1132899.750 [ip-26-0-154-121:0]:06/21/2023 17:47:07 [INFO|DP=0|PP=0|TP=0]: iteration: 2257 / 150000 | consumed_samples: 144448 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787663E+05 | tokens_per_sec_per_gpu: 9.043224E+03 | global_batch_size: 64 | lm_loss: 1.081442E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1116414.125 [ip-26-0-154-121:0]:06/21/2023 17:47:07 [INFO|DP=0|PP=0|TP=0]: iteration: 2258 / 150000 | consumed_samples: 144512 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789068E+05 | tokens_per_sec_per_gpu: 9.045419E+03 | global_batch_size: 64 | lm_loss: 1.081370E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1114584.625 [ip-26-0-154-121:0]:06/21/2023 17:47:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2259 / 150000 | consumed_samples: 144576 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790664E+05 | tokens_per_sec_per_gpu: 9.047913E+03 | global_batch_size: 64 | lm_loss: 1.081274E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 1118096.250 [ip-26-0-154-121:0]:06/21/2023 17:47:09 [INFO|DP=0|PP=0|TP=0]: iteration: 2260 / 150000 | consumed_samples: 144640 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789054E+05 | tokens_per_sec_per_gpu: 9.045397E+03 | global_batch_size: 64 | lm_loss: 1.081420E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1116072.750 [ip-26-0-154-121:0]:06/21/2023 17:47:10 [INFO|DP=0|PP=0|TP=0]: iteration: 2261 / 150000 | consumed_samples: 144704 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789576E+05 | tokens_per_sec_per_gpu: 9.046212E+03 | global_batch_size: 64 | lm_loss: 1.081379E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 1111326.750 [ip-26-0-154-121:0]:06/21/2023 17:47:11 [INFO|DP=0|PP=0|TP=0]: iteration: 2262 / 150000 | consumed_samples: 144768 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784933E+05 | tokens_per_sec_per_gpu: 9.038958E+03 | global_batch_size: 64 | lm_loss: 1.081397E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1102077.000 [ip-26-0-154-121:0]:06/21/2023 17:47:12 [INFO|DP=0|PP=0|TP=0]: iteration: 2263 / 150000 | consumed_samples: 144832 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790429E+05 | tokens_per_sec_per_gpu: 9.047546E+03 | global_batch_size: 64 | lm_loss: 1.081427E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 1094946.625 [ip-26-0-154-121:0]:06/21/2023 17:47:13 [INFO|DP=0|PP=0|TP=0]: iteration: 2264 / 150000 | consumed_samples: 144896 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785143E+05 | tokens_per_sec_per_gpu: 9.039287E+03 | global_batch_size: 64 | lm_loss: 1.081451E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1096605.375 [ip-26-0-154-121:0]:06/21/2023 17:47:14 [INFO|DP=0|PP=0|TP=0]: iteration: 2265 / 150000 | consumed_samples: 144960 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787797E+05 | tokens_per_sec_per_gpu: 9.043433E+03 | global_batch_size: 64 | lm_loss: 1.081427E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1102815.750 [ip-26-0-154-121:0]:06/21/2023 17:47:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2266 / 150000 | consumed_samples: 145024 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786119E+05 | tokens_per_sec_per_gpu: 9.040811E+03 | global_batch_size: 64 | lm_loss: 1.081419E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1098534.375 [ip-26-0-154-121:0]:06/21/2023 17:47:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2267 / 150000 | consumed_samples: 145088 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785729E+05 | tokens_per_sec_per_gpu: 9.040202E+03 | global_batch_size: 64 | lm_loss: 1.081378E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1088058.500 [ip-26-0-154-121:0]:06/21/2023 17:47:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2268 / 150000 | consumed_samples: 145152 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789544E+05 | tokens_per_sec_per_gpu: 9.046162E+03 | global_batch_size: 64 | lm_loss: 1.081391E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 1080214.250 [ip-26-0-154-121:0]:06/21/2023 17:47:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2269 / 150000 | consumed_samples: 145216 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786628E+05 | tokens_per_sec_per_gpu: 9.041606E+03 | global_batch_size: 64 | lm_loss: 1.081350E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1077835.750 [ip-26-0-154-121:0]:06/21/2023 17:47:18 [INFO|DP=0|PP=0|TP=0]: iteration: 2270 / 150000 | consumed_samples: 145280 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788676E+05 | tokens_per_sec_per_gpu: 9.044807E+03 | global_batch_size: 64 | lm_loss: 1.081385E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1078898.375 [ip-26-0-154-121:0]:06/21/2023 17:47:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2271 / 150000 | consumed_samples: 145344 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786871E+05 | tokens_per_sec_per_gpu: 9.041986E+03 | global_batch_size: 64 | lm_loss: 1.081385E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1076231.375 [ip-26-0-154-121:0]:06/21/2023 17:47:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2272 / 150000 | consumed_samples: 145408 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783792E+05 | tokens_per_sec_per_gpu: 9.037175E+03 | global_batch_size: 64 | lm_loss: 1.081367E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1074235.625 [ip-26-0-154-121:0]:06/21/2023 17:47:21 [INFO|DP=0|PP=0|TP=0]: iteration: 2273 / 150000 | consumed_samples: 145472 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785305E+05 | tokens_per_sec_per_gpu: 9.039539E+03 | global_batch_size: 64 | lm_loss: 1.081376E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1070276.250 [ip-26-0-154-121:0]:06/21/2023 17:47:22 [INFO|DP=0|PP=0|TP=0]: iteration: 2274 / 150000 | consumed_samples: 145536 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785882E+05 | tokens_per_sec_per_gpu: 9.040440E+03 | global_batch_size: 64 | lm_loss: 1.081347E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1070226.500 [ip-26-0-154-121:0]:06/21/2023 17:47:23 [INFO|DP=0|PP=0|TP=0]: iteration: 2275 / 150000 | consumed_samples: 145600 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786271E+05 | tokens_per_sec_per_gpu: 9.041049E+03 | global_batch_size: 64 | lm_loss: 1.081351E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1074890.000 [ip-26-0-154-121:0]:06/21/2023 17:47:24 [INFO|DP=0|PP=0|TP=0]: iteration: 2276 / 150000 | consumed_samples: 145664 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787133E+05 | tokens_per_sec_per_gpu: 9.042396E+03 | global_batch_size: 64 | lm_loss: 1.081345E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1067980.500 [ip-26-0-154-121:0]:06/21/2023 17:47:25 [INFO|DP=0|PP=0|TP=0]: iteration: 2277 / 150000 | consumed_samples: 145728 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784279E+05 | tokens_per_sec_per_gpu: 9.037936E+03 | global_batch_size: 64 | lm_loss: 1.081330E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1064892.875 [ip-26-0-154-121:0]:06/21/2023 17:47:26 [INFO|DP=0|PP=0|TP=0]: iteration: 2278 / 150000 | consumed_samples: 145792 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786177E+05 | tokens_per_sec_per_gpu: 9.040902E+03 | global_batch_size: 64 | lm_loss: 1.081324E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1071046.625 [ip-26-0-154-121:0]:06/21/2023 17:47:26 [INFO|DP=0|PP=0|TP=0]: iteration: 2279 / 150000 | consumed_samples: 145856 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788564E+05 | tokens_per_sec_per_gpu: 9.044631E+03 | global_batch_size: 64 | lm_loss: 1.081329E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1071175.500 [ip-26-0-154-121:0]:06/21/2023 17:47:27 [INFO|DP=0|PP=0|TP=0]: iteration: 2280 / 150000 | consumed_samples: 145920 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786523E+05 | tokens_per_sec_per_gpu: 9.041442E+03 | global_batch_size: 64 | lm_loss: 1.081320E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1069236.125 [ip-26-0-154-121:0]:06/21/2023 17:47:28 [INFO|DP=0|PP=0|TP=0]: iteration: 2281 / 150000 | consumed_samples: 145984 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788149E+05 | tokens_per_sec_per_gpu: 9.043983E+03 | global_batch_size: 64 | lm_loss: 1.081307E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1065399.125 [ip-26-0-154-121:0]:06/21/2023 17:47:29 [INFO|DP=0|PP=0|TP=0]: iteration: 2282 / 150000 | consumed_samples: 146048 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790184E+05 | tokens_per_sec_per_gpu: 9.047162E+03 | global_batch_size: 64 | lm_loss: 1.081303E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 1060036.250 [ip-26-0-154-121:0]:06/21/2023 17:47:30 [INFO|DP=0|PP=0|TP=0]: iteration: 2283 / 150000 | consumed_samples: 146112 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783707E+05 | tokens_per_sec_per_gpu: 9.037042E+03 | global_batch_size: 64 | lm_loss: 1.081319E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1059071.500 [ip-26-0-154-121:0]:06/21/2023 17:47:31 [INFO|DP=0|PP=0|TP=0]: iteration: 2284 / 150000 | consumed_samples: 146176 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788364E+05 | tokens_per_sec_per_gpu: 9.044319E+03 | global_batch_size: 64 | lm_loss: 1.081105E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1055684.000 [ip-26-0-154-121:0]:06/21/2023 17:47:32 [INFO|DP=0|PP=0|TP=0]: iteration: 2285 / 150000 | consumed_samples: 146240 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788000E+05 | tokens_per_sec_per_gpu: 9.043750E+03 | global_batch_size: 64 | lm_loss: 1.081283E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1049030.750 [ip-26-0-154-121:0]:06/21/2023 17:47:33 [INFO|DP=0|PP=0|TP=0]: iteration: 2286 / 150000 | consumed_samples: 146304 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788873E+05 | tokens_per_sec_per_gpu: 9.045114E+03 | global_batch_size: 64 | lm_loss: 1.081284E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1046891.812 [ip-26-0-154-121:0]:06/21/2023 17:47:34 [INFO|DP=0|PP=0|TP=0]: iteration: 2287 / 150000 | consumed_samples: 146368 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785151E+05 | tokens_per_sec_per_gpu: 9.039298E+03 | global_batch_size: 64 | lm_loss: 1.081152E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1048222.562 [ip-26-0-154-121:0]:06/21/2023 17:47:35 [INFO|DP=0|PP=0|TP=0]: iteration: 2288 / 150000 | consumed_samples: 146432 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786789E+05 | tokens_per_sec_per_gpu: 9.041858E+03 | global_batch_size: 64 | lm_loss: 1.081283E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1051476.000 [ip-26-0-154-121:0]:06/21/2023 17:47:36 [INFO|DP=0|PP=0|TP=0]: iteration: 2289 / 150000 | consumed_samples: 146496 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785145E+05 | tokens_per_sec_per_gpu: 9.039289E+03 | global_batch_size: 64 | lm_loss: 1.081150E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1054298.500 [ip-26-0-154-121:0]:06/21/2023 17:47:36 [INFO|DP=0|PP=0|TP=0]: iteration: 2290 / 150000 | consumed_samples: 146560 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787027E+05 | tokens_per_sec_per_gpu: 9.042229E+03 | global_batch_size: 64 | lm_loss: 1.081276E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1054811.875 [ip-26-0-154-121:0]:06/21/2023 17:47:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2291 / 150000 | consumed_samples: 146624 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790751E+05 | tokens_per_sec_per_gpu: 9.048049E+03 | global_batch_size: 64 | lm_loss: 1.081266E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 1057479.375 [ip-26-0-154-121:0]:06/21/2023 17:47:38 [INFO|DP=0|PP=0|TP=0]: iteration: 2292 / 150000 | consumed_samples: 146688 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788669E+05 | tokens_per_sec_per_gpu: 9.044795E+03 | global_batch_size: 64 | lm_loss: 1.081289E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1059339.250 [ip-26-0-154-121:0]:06/21/2023 17:47:39 [INFO|DP=0|PP=0|TP=0]: iteration: 2293 / 150000 | consumed_samples: 146752 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788276E+05 | tokens_per_sec_per_gpu: 9.044181E+03 | global_batch_size: 64 | lm_loss: 1.081259E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1056408.125 [ip-26-0-154-121:0]:06/21/2023 17:47:40 [INFO|DP=0|PP=0|TP=0]: iteration: 2294 / 150000 | consumed_samples: 146816 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787988E+05 | tokens_per_sec_per_gpu: 9.043731E+03 | global_batch_size: 64 | lm_loss: 1.081270E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1044552.875 [ip-26-0-154-121:0]:06/21/2023 17:47:41 [INFO|DP=0|PP=0|TP=0]: iteration: 2295 / 150000 | consumed_samples: 146880 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787704E+05 | tokens_per_sec_per_gpu: 9.043288E+03 | global_batch_size: 64 | lm_loss: 1.081252E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1033465.062 [ip-26-0-154-121:0]:06/21/2023 17:47:42 [INFO|DP=0|PP=0|TP=0]: iteration: 2296 / 150000 | consumed_samples: 146944 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788791E+05 | tokens_per_sec_per_gpu: 9.044986E+03 | global_batch_size: 64 | lm_loss: 1.081262E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1027150.125 [ip-26-0-154-121:0]:06/21/2023 17:47:43 [INFO|DP=0|PP=0|TP=0]: iteration: 2297 / 150000 | consumed_samples: 147008 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784920E+05 | tokens_per_sec_per_gpu: 9.038937E+03 | global_batch_size: 64 | lm_loss: 1.081237E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1023729.062 [ip-26-0-154-121:0]:06/21/2023 17:47:44 [INFO|DP=0|PP=0|TP=0]: iteration: 2298 / 150000 | consumed_samples: 147072 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783664E+05 | tokens_per_sec_per_gpu: 9.036976E+03 | global_batch_size: 64 | lm_loss: 1.081248E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1023286.625 [ip-26-0-154-121:0]:06/21/2023 17:47:45 [INFO|DP=0|PP=0|TP=0]: iteration: 2299 / 150000 | consumed_samples: 147136 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784529E+05 | tokens_per_sec_per_gpu: 9.038326E+03 | global_batch_size: 64 | lm_loss: 1.081231E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1021158.812 [ip-26-0-154-121:0]:06/21/2023 17:47:46 [INFO|DP=0|PP=0|TP=0]: iteration: 2300 / 150000 | consumed_samples: 147200 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785551E+05 | tokens_per_sec_per_gpu: 9.039924E+03 | global_batch_size: 64 | lm_loss: 1.081201E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1018658.375 [ip-26-0-154-121:0]:06/21/2023 17:47:46 [INFO|DP=0|PP=0|TP=0]: iteration: 2301 / 150000 | consumed_samples: 147264 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784247E+05 | tokens_per_sec_per_gpu: 9.037886E+03 | global_batch_size: 64 | lm_loss: 1.081198E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1016211.688 [ip-26-0-154-121:0]:06/21/2023 17:47:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2302 / 150000 | consumed_samples: 147328 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786622E+05 | tokens_per_sec_per_gpu: 9.041596E+03 | global_batch_size: 64 | lm_loss: 1.081192E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1011834.188 [ip-26-0-154-121:0]:06/21/2023 17:47:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2303 / 150000 | consumed_samples: 147392 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786207E+05 | tokens_per_sec_per_gpu: 9.040949E+03 | global_batch_size: 64 | lm_loss: 1.081193E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1007206.500 [ip-26-0-154-121:0]:06/21/2023 17:47:49 [INFO|DP=0|PP=0|TP=0]: iteration: 2304 / 150000 | consumed_samples: 147456 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788125E+05 | tokens_per_sec_per_gpu: 9.043945E+03 | global_batch_size: 64 | lm_loss: 1.081169E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1002845.125 [ip-26-0-154-121:0]:06/21/2023 17:47:50 [INFO|DP=0|PP=0|TP=0]: iteration: 2305 / 150000 | consumed_samples: 147520 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787840E+05 | tokens_per_sec_per_gpu: 9.043500E+03 | global_batch_size: 64 | lm_loss: 1.081202E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 998359.625 [ip-26-0-154-121:0]:06/21/2023 17:47:51 [INFO|DP=0|PP=0|TP=0]: iteration: 2306 / 150000 | consumed_samples: 147584 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789038E+05 | tokens_per_sec_per_gpu: 9.045371E+03 | global_batch_size: 64 | lm_loss: 1.081207E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 994199.188 [ip-26-0-154-121:0]:06/21/2023 17:47:52 [INFO|DP=0|PP=0|TP=0]: iteration: 2307 / 150000 | consumed_samples: 147648 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781943E+05 | tokens_per_sec_per_gpu: 9.034286E+03 | global_batch_size: 64 | lm_loss: 1.081179E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 988663.875 [ip-26-0-154-121:0]:06/21/2023 17:47:53 [INFO|DP=0|PP=0|TP=0]: iteration: 2308 / 150000 | consumed_samples: 147712 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781695E+05 | tokens_per_sec_per_gpu: 9.033899E+03 | global_batch_size: 64 | lm_loss: 1.081164E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 985908.688 [ip-26-0-154-121:0]:06/21/2023 17:47:54 [INFO|DP=0|PP=0|TP=0]: iteration: 2309 / 150000 | consumed_samples: 147776 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786162E+05 | tokens_per_sec_per_gpu: 9.040878E+03 | global_batch_size: 64 | lm_loss: 1.081185E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 985385.688 [ip-26-0-154-121:0]:06/21/2023 17:47:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2310 / 150000 | consumed_samples: 147840 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783292E+05 | tokens_per_sec_per_gpu: 9.036393E+03 | global_batch_size: 64 | lm_loss: 1.081194E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 984031.250 [ip-26-0-154-121:0]:06/21/2023 17:47:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2311 / 150000 | consumed_samples: 147904 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788286E+05 | tokens_per_sec_per_gpu: 9.044197E+03 | global_batch_size: 64 | lm_loss: 1.081175E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 984092.188 [ip-26-0-154-121:0]:06/21/2023 17:47:56 [INFO|DP=0|PP=0|TP=0]: iteration: 2312 / 150000 | consumed_samples: 147968 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788023E+05 | tokens_per_sec_per_gpu: 9.043786E+03 | global_batch_size: 64 | lm_loss: 1.081164E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 982182.500 [ip-26-0-154-121:0]:06/21/2023 17:47:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2313 / 150000 | consumed_samples: 148032 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786043E+05 | tokens_per_sec_per_gpu: 9.040692E+03 | global_batch_size: 64 | lm_loss: 1.081157E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 978910.938 [ip-26-0-154-121:0]:06/21/2023 17:47:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2314 / 150000 | consumed_samples: 148096 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787354E+05 | tokens_per_sec_per_gpu: 9.042741E+03 | global_batch_size: 64 | lm_loss: 1.081144E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 975825.750 [ip-26-0-154-121:0]:06/21/2023 17:47:59 [INFO|DP=0|PP=0|TP=0]: iteration: 2315 / 150000 | consumed_samples: 148160 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785586E+05 | tokens_per_sec_per_gpu: 9.039979E+03 | global_batch_size: 64 | lm_loss: 1.081135E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 973090.812 [ip-26-0-154-121:0]:06/21/2023 17:48:00 [INFO|DP=0|PP=0|TP=0]: iteration: 2316 / 150000 | consumed_samples: 148224 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786687E+05 | tokens_per_sec_per_gpu: 9.041699E+03 | global_batch_size: 64 | lm_loss: 1.081141E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 970969.438 [ip-26-0-154-121:0]:06/21/2023 17:48:01 [INFO|DP=0|PP=0|TP=0]: iteration: 2317 / 150000 | consumed_samples: 148288 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784974E+05 | tokens_per_sec_per_gpu: 9.039023E+03 | global_batch_size: 64 | lm_loss: 1.081144E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 968023.375 [ip-26-0-154-121:0]:06/21/2023 17:48:02 [INFO|DP=0|PP=0|TP=0]: iteration: 2318 / 150000 | consumed_samples: 148352 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784947E+05 | tokens_per_sec_per_gpu: 9.038980E+03 | global_batch_size: 64 | lm_loss: 1.081088E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 966515.625 [ip-26-0-154-121:0]:06/21/2023 17:48:03 [INFO|DP=0|PP=0|TP=0]: iteration: 2319 / 150000 | consumed_samples: 148416 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790815E+05 | tokens_per_sec_per_gpu: 9.048149E+03 | global_batch_size: 64 | lm_loss: 1.081131E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 962913.312 [ip-26-0-154-121:0]:06/21/2023 17:48:04 [INFO|DP=0|PP=0|TP=0]: iteration: 2320 / 150000 | consumed_samples: 148480 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785303E+05 | tokens_per_sec_per_gpu: 9.039536E+03 | global_batch_size: 64 | lm_loss: 1.081129E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 958644.125 [ip-26-0-154-121:0]:06/21/2023 17:48:05 [INFO|DP=0|PP=0|TP=0]: iteration: 2321 / 150000 | consumed_samples: 148544 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781797E+05 | tokens_per_sec_per_gpu: 9.034058E+03 | global_batch_size: 64 | lm_loss: 1.081134E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 957440.000 [ip-26-0-154-121:0]:06/21/2023 17:48:05 [INFO|DP=0|PP=0|TP=0]: iteration: 2322 / 150000 | consumed_samples: 148608 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788404E+05 | tokens_per_sec_per_gpu: 9.044381E+03 | global_batch_size: 64 | lm_loss: 1.081144E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 958014.938 [ip-26-0-154-121:0]:06/21/2023 17:48:06 [INFO|DP=0|PP=0|TP=0]: iteration: 2323 / 150000 | consumed_samples: 148672 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788308E+05 | tokens_per_sec_per_gpu: 9.044231E+03 | global_batch_size: 64 | lm_loss: 1.081134E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 956836.312 [ip-26-0-154-121:0]:06/21/2023 17:48:07 [INFO|DP=0|PP=0|TP=0]: iteration: 2324 / 150000 | consumed_samples: 148736 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785994E+05 | tokens_per_sec_per_gpu: 9.040616E+03 | global_batch_size: 64 | lm_loss: 1.081149E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 957008.688 [ip-26-0-154-121:0]:06/21/2023 17:48:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2325 / 150000 | consumed_samples: 148800 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788501E+05 | tokens_per_sec_per_gpu: 9.044533E+03 | global_batch_size: 64 | lm_loss: 1.081111E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 954938.562 [ip-26-0-154-121:0]:06/21/2023 17:48:09 [INFO|DP=0|PP=0|TP=0]: iteration: 2326 / 150000 | consumed_samples: 148864 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790121E+05 | tokens_per_sec_per_gpu: 9.047065E+03 | global_batch_size: 64 | lm_loss: 1.081103E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 954395.188 [ip-26-0-154-121:0]:06/21/2023 17:48:10 [INFO|DP=0|PP=0|TP=0]: iteration: 2327 / 150000 | consumed_samples: 148928 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788364E+05 | tokens_per_sec_per_gpu: 9.044319E+03 | global_batch_size: 64 | lm_loss: 1.081056E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 951555.438 [ip-26-0-154-121:0]:06/21/2023 17:48:11 [INFO|DP=0|PP=0|TP=0]: iteration: 2328 / 150000 | consumed_samples: 148992 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788291E+05 | tokens_per_sec_per_gpu: 9.044205E+03 | global_batch_size: 64 | lm_loss: 1.081077E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 951293.062 [ip-26-0-154-121:0]:06/21/2023 17:48:12 [INFO|DP=0|PP=0|TP=0]: iteration: 2329 / 150000 | consumed_samples: 149056 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789172E+05 | tokens_per_sec_per_gpu: 9.045581E+03 | global_batch_size: 64 | lm_loss: 1.081055E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 951581.500 [ip-26-0-154-121:0]:06/21/2023 17:48:13 [INFO|DP=0|PP=0|TP=0]: iteration: 2330 / 150000 | consumed_samples: 149120 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788573E+05 | tokens_per_sec_per_gpu: 9.044645E+03 | global_batch_size: 64 | lm_loss: 1.081067E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 954331.938 [ip-26-0-154-121:0]:06/21/2023 17:48:14 [INFO|DP=0|PP=0|TP=0]: iteration: 2331 / 150000 | consumed_samples: 149184 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782106E+05 | tokens_per_sec_per_gpu: 9.034540E+03 | global_batch_size: 64 | lm_loss: 1.081063E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 954663.375 [ip-26-0-154-121:0]:06/21/2023 17:48:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2332 / 150000 | consumed_samples: 149248 | elapsed_time_per_iteration_ms: 905.2 | tokens_per_sec: 5.792135E+05 | tokens_per_sec_per_gpu: 9.050210E+03 | global_batch_size: 64 | lm_loss: 1.081058E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.54 | hardware_tflops_per_gpu: 104.54 | grad_norm: 954381.812 [ip-26-0-154-121:0]:06/21/2023 17:48:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2333 / 150000 | consumed_samples: 149312 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777006E+05 | tokens_per_sec_per_gpu: 9.026572E+03 | global_batch_size: 64 | lm_loss: 1.081040E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 956637.125 [ip-26-0-154-121:0]:06/21/2023 17:48:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2334 / 150000 | consumed_samples: 149376 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785965E+05 | tokens_per_sec_per_gpu: 9.040571E+03 | global_batch_size: 64 | lm_loss: 1.081017E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 971655.875 [ip-26-0-154-121:0]:06/21/2023 17:48:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2335 / 150000 | consumed_samples: 149440 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786383E+05 | tokens_per_sec_per_gpu: 9.041223E+03 | global_batch_size: 64 | lm_loss: 1.081049E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 976782.938 [ip-26-0-154-121:0]:06/21/2023 17:48:18 [INFO|DP=0|PP=0|TP=0]: iteration: 2336 / 150000 | consumed_samples: 149504 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787254E+05 | tokens_per_sec_per_gpu: 9.042584E+03 | global_batch_size: 64 | lm_loss: 1.081046E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 982049.875 [ip-26-0-154-121:0]:06/21/2023 17:48:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2337 / 150000 | consumed_samples: 149568 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788378E+05 | tokens_per_sec_per_gpu: 9.044340E+03 | global_batch_size: 64 | lm_loss: 1.081059E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 997638.938 [ip-26-0-154-121:0]:06/21/2023 17:48:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2338 / 150000 | consumed_samples: 149632 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786293E+05 | tokens_per_sec_per_gpu: 9.041082E+03 | global_batch_size: 64 | lm_loss: 1.081058E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1027349.000 [ip-26-0-154-121:0]:06/21/2023 17:48:21 [INFO|DP=0|PP=0|TP=0]: iteration: 2339 / 150000 | consumed_samples: 149696 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784542E+05 | tokens_per_sec_per_gpu: 9.038347E+03 | global_batch_size: 64 | lm_loss: 1.081054E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1057563.250 [ip-26-0-154-121:0]:06/21/2023 17:48:22 [INFO|DP=0|PP=0|TP=0]: iteration: 2340 / 150000 | consumed_samples: 149760 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788916E+05 | tokens_per_sec_per_gpu: 9.045181E+03 | global_batch_size: 64 | lm_loss: 1.081012E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1174389.875 [ip-26-0-154-121:0]:06/21/2023 17:48:23 [INFO|DP=0|PP=0|TP=0]: iteration: 2341 / 150000 | consumed_samples: 149824 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788934E+05 | tokens_per_sec_per_gpu: 9.045209E+03 | global_batch_size: 64 | lm_loss: 1.081025E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1224679.375 [ip-26-0-154-121:0]:06/21/2023 17:48:24 [INFO|DP=0|PP=0|TP=0]: iteration: 2342 / 150000 | consumed_samples: 149888 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787348E+05 | tokens_per_sec_per_gpu: 9.042731E+03 | global_batch_size: 64 | lm_loss: 1.081023E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1351385.750 [ip-26-0-154-121:0]:06/21/2023 17:48:25 [INFO|DP=0|PP=0|TP=0]: iteration: 2343 / 150000 | consumed_samples: 149952 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783643E+05 | tokens_per_sec_per_gpu: 9.036942E+03 | global_batch_size: 64 | lm_loss: 1.081016E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1625315.125 [ip-26-0-154-121:0]:06/21/2023 17:48:25 [INFO|DP=0|PP=0|TP=0]: iteration: 2344 / 150000 | consumed_samples: 150016 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.783973E+05 | tokens_per_sec_per_gpu: 9.037458E+03 | global_batch_size: 64 | lm_loss: 1.081021E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1217673.500 [ip-26-0-154-121:0]:06/21/2023 17:48:26 [INFO|DP=0|PP=0|TP=0]: iteration: 2345 / 150000 | consumed_samples: 150080 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784144E+05 | tokens_per_sec_per_gpu: 9.037724E+03 | global_batch_size: 64 | lm_loss: 1.081021E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1214430.875 [ip-26-0-154-121:0]:06/21/2023 17:48:27 [INFO|DP=0|PP=0|TP=0]: iteration: 2346 / 150000 | consumed_samples: 150144 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784249E+05 | tokens_per_sec_per_gpu: 9.037888E+03 | global_batch_size: 64 | lm_loss: 1.081012E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1560246.750 [ip-26-0-154-121:0]:06/21/2023 17:48:28 [INFO|DP=0|PP=0|TP=0]: iteration: 2347 / 150000 | consumed_samples: 150208 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781613E+05 | tokens_per_sec_per_gpu: 9.033771E+03 | global_batch_size: 64 | lm_loss: 1.081009E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 17758390.000 [ip-26-0-154-121:0]:06/21/2023 17:48:29 [INFO|DP=0|PP=0|TP=0]: iteration: 2348 / 150000 | consumed_samples: 150272 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784915E+05 | tokens_per_sec_per_gpu: 9.038930E+03 | global_batch_size: 64 | lm_loss: 1.080990E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 176094384.000 [ip-26-0-154-121:0]:06/21/2023 17:48:30 [INFO|DP=0|PP=0|TP=0]: iteration: 2349 / 150000 | consumed_samples: 150336 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787069E+05 | tokens_per_sec_per_gpu: 9.042296E+03 | global_batch_size: 64 | lm_loss: 1.080947E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 8987669.000 [ip-26-0-154-121:0]:06/21/2023 17:48:31 [INFO|DP=0|PP=0|TP=0]: iteration: 2350 / 150000 | consumed_samples: 150400 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784567E+05 | tokens_per_sec_per_gpu: 9.038385E+03 | global_batch_size: 64 | lm_loss: 1.080903E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1046444.938 [ip-26-0-154-121:0]:06/21/2023 17:48:32 [INFO|DP=0|PP=0|TP=0]: iteration: 2351 / 150000 | consumed_samples: 150464 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785816E+05 | tokens_per_sec_per_gpu: 9.040338E+03 | global_batch_size: 64 | lm_loss: 1.081045E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 32687260.000 [ip-26-0-154-121:0]:06/21/2023 17:48:33 [INFO|DP=0|PP=0|TP=0]: iteration: 2352 / 150000 | consumed_samples: 150528 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781785E+05 | tokens_per_sec_per_gpu: 9.034039E+03 | global_batch_size: 64 | lm_loss: 1.081083E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 3844571.000 [ip-26-0-154-121:0]:06/21/2023 17:48:34 [INFO|DP=0|PP=0|TP=0]: iteration: 2353 / 150000 | consumed_samples: 150592 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786498E+05 | tokens_per_sec_per_gpu: 9.041404E+03 | global_batch_size: 64 | lm_loss: 1.081097E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2901186.000 [ip-26-0-154-121:0]:06/21/2023 17:48:35 [INFO|DP=0|PP=0|TP=0]: iteration: 2354 / 150000 | consumed_samples: 150656 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789897E+05 | tokens_per_sec_per_gpu: 9.046715E+03 | global_batch_size: 64 | lm_loss: 1.081062E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 3139495.500 [ip-26-0-154-121:0]:06/21/2023 17:48:35 [INFO|DP=0|PP=0|TP=0]: iteration: 2355 / 150000 | consumed_samples: 150720 | elapsed_time_per_iteration_ms: 905.2 | tokens_per_sec: 5.791807E+05 | tokens_per_sec_per_gpu: 9.049698E+03 | global_batch_size: 64 | lm_loss: 1.081014E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.53 | hardware_tflops_per_gpu: 104.53 | grad_norm: 6489284.500 [ip-26-0-154-121:0]:06/21/2023 17:48:36 [INFO|DP=0|PP=0|TP=0]: iteration: 2356 / 150000 | consumed_samples: 150784 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786721E+05 | tokens_per_sec_per_gpu: 9.041751E+03 | global_batch_size: 64 | lm_loss: 1.080952E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 61687628.000 [ip-26-0-154-121:0]:06/21/2023 17:48:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2357 / 150000 | consumed_samples: 150848 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789009E+05 | tokens_per_sec_per_gpu: 9.045326E+03 | global_batch_size: 64 | lm_loss: 1.080909E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 3060284.750 [ip-26-0-154-121:0]:06/21/2023 17:48:38 [INFO|DP=0|PP=0|TP=0]: iteration: 2358 / 150000 | consumed_samples: 150912 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788596E+05 | tokens_per_sec_per_gpu: 9.044681E+03 | global_batch_size: 64 | lm_loss: 1.080898E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 2616319.250 [ip-26-0-154-121:0]:06/21/2023 17:48:39 [INFO|DP=0|PP=0|TP=0]: iteration: 2359 / 150000 | consumed_samples: 150976 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786463E+05 | tokens_per_sec_per_gpu: 9.041349E+03 | global_batch_size: 64 | lm_loss: 1.080946E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2545852.500 [ip-26-0-154-121:0]:06/21/2023 17:48:40 [INFO|DP=0|PP=0|TP=0]: iteration: 2360 / 150000 | consumed_samples: 151040 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784666E+05 | tokens_per_sec_per_gpu: 9.038540E+03 | global_batch_size: 64 | lm_loss: 1.080981E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 17077872.000 [ip-26-0-154-121:0]:06/21/2023 17:48:41 [INFO|DP=0|PP=0|TP=0]: iteration: 2361 / 150000 | consumed_samples: 151104 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789329E+05 | tokens_per_sec_per_gpu: 9.045826E+03 | global_batch_size: 64 | lm_loss: 1.081018E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 18521412.000 [ip-26-0-154-121:0]:06/21/2023 17:48:42 [INFO|DP=0|PP=0|TP=0]: iteration: 2362 / 150000 | consumed_samples: 151168 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785623E+05 | tokens_per_sec_per_gpu: 9.040036E+03 | global_batch_size: 64 | lm_loss: 1.081030E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 9029454.000 [ip-26-0-154-121:0]:06/21/2023 17:48:43 [INFO|DP=0|PP=0|TP=0]: iteration: 2363 / 150000 | consumed_samples: 151232 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787286E+05 | tokens_per_sec_per_gpu: 9.042634E+03 | global_batch_size: 64 | lm_loss: 1.081028E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 9339776.000 [ip-26-0-154-121:0]:06/21/2023 17:48:44 [INFO|DP=0|PP=0|TP=0]: iteration: 2364 / 150000 | consumed_samples: 151296 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782379E+05 | tokens_per_sec_per_gpu: 9.034968E+03 | global_batch_size: 64 | lm_loss: 1.080832E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 14758901.000 [ip-26-0-154-121:0]:06/21/2023 17:48:44 [INFO|DP=0|PP=0|TP=0]: iteration: 2365 / 150000 | consumed_samples: 151360 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786121E+05 | tokens_per_sec_per_gpu: 9.040814E+03 | global_batch_size: 64 | lm_loss: 1.080978E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 21301834.000 [ip-26-0-154-121:0]:06/21/2023 17:48:45 [INFO|DP=0|PP=0|TP=0]: iteration: 2366 / 150000 | consumed_samples: 151424 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785264E+05 | tokens_per_sec_per_gpu: 9.039474E+03 | global_batch_size: 64 | lm_loss: 1.080996E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 12101997.000 [ip-26-0-154-121:0]:06/21/2023 17:48:46 [INFO|DP=0|PP=0|TP=0]: iteration: 2367 / 150000 | consumed_samples: 151488 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784548E+05 | tokens_per_sec_per_gpu: 9.038357E+03 | global_batch_size: 64 | lm_loss: 1.080950E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 5002962.500 [ip-26-0-154-121:0]:06/21/2023 17:48:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2368 / 150000 | consumed_samples: 151552 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789339E+05 | tokens_per_sec_per_gpu: 9.045843E+03 | global_batch_size: 64 | lm_loss: 1.080955E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 2332558.250 [ip-26-0-154-121:0]:06/21/2023 17:48:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2369 / 150000 | consumed_samples: 151616 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789021E+05 | tokens_per_sec_per_gpu: 9.045345E+03 | global_batch_size: 64 | lm_loss: 1.080997E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 3502621.250 [ip-26-0-154-121:0]:06/21/2023 17:48:49 [INFO|DP=0|PP=0|TP=0]: iteration: 2370 / 150000 | consumed_samples: 151680 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787852E+05 | tokens_per_sec_per_gpu: 9.043519E+03 | global_batch_size: 64 | lm_loss: 1.080991E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 29380652.000 [ip-26-0-154-121:0]:06/21/2023 17:48:50 [INFO|DP=0|PP=0|TP=0]: iteration: 2371 / 150000 | consumed_samples: 151744 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788369E+05 | tokens_per_sec_per_gpu: 9.044326E+03 | global_batch_size: 64 | lm_loss: 1.081021E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 10560414.000 [ip-26-0-154-121:0]:06/21/2023 17:48:51 [INFO|DP=0|PP=0|TP=0]: iteration: 2372 / 150000 | consumed_samples: 151808 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789307E+05 | tokens_per_sec_per_gpu: 9.045793E+03 | global_batch_size: 64 | lm_loss: 1.080981E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 10501440.000 [ip-26-0-154-121:0]:06/21/2023 17:48:52 [INFO|DP=0|PP=0|TP=0]: iteration: 2373 / 150000 | consumed_samples: 151872 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790759E+05 | tokens_per_sec_per_gpu: 9.048061E+03 | global_batch_size: 64 | lm_loss: 1.081014E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 15124921.000 [ip-26-0-154-121:0]:06/21/2023 17:48:53 [INFO|DP=0|PP=0|TP=0]: iteration: 2374 / 150000 | consumed_samples: 151936 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789289E+05 | tokens_per_sec_per_gpu: 9.045764E+03 | global_batch_size: 64 | lm_loss: 1.080974E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 10538971.000 [ip-26-0-154-121:0]:06/21/2023 17:48:54 [INFO|DP=0|PP=0|TP=0]: iteration: 2375 / 150000 | consumed_samples: 152000 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789207E+05 | tokens_per_sec_per_gpu: 9.045636E+03 | global_batch_size: 64 | lm_loss: 1.080964E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 1839243.000 [ip-26-0-154-121:0]:06/21/2023 17:48:54 [INFO|DP=0|PP=0|TP=0]: iteration: 2376 / 150000 | consumed_samples: 152064 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789568E+05 | tokens_per_sec_per_gpu: 9.046200E+03 | global_batch_size: 64 | lm_loss: 1.080962E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 1118575.250 [ip-26-0-154-121:0]:06/21/2023 17:48:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2377 / 150000 | consumed_samples: 152128 | elapsed_time_per_iteration_ms: 904.7 | tokens_per_sec: 5.794949E+05 | tokens_per_sec_per_gpu: 9.054608E+03 | global_batch_size: 64 | lm_loss: 1.080973E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.59 | hardware_tflops_per_gpu: 104.59 | grad_norm: 7431709.000 [ip-26-0-154-121:0]:06/21/2023 17:48:56 [INFO|DP=0|PP=0|TP=0]: iteration: 2378 / 150000 | consumed_samples: 152192 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788174E+05 | tokens_per_sec_per_gpu: 9.044021E+03 | global_batch_size: 64 | lm_loss: 1.080962E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 16230869.000 [ip-26-0-154-121:0]:06/21/2023 17:48:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2379 / 150000 | consumed_samples: 152256 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786859E+05 | tokens_per_sec_per_gpu: 9.041967E+03 | global_batch_size: 64 | lm_loss: 1.080986E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 16965034.000 [ip-26-0-154-121:0]:06/21/2023 17:48:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2380 / 150000 | consumed_samples: 152320 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787409E+05 | tokens_per_sec_per_gpu: 9.042826E+03 | global_batch_size: 64 | lm_loss: 1.080995E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 15907177.000 [ip-26-0-154-121:0]:06/21/2023 17:48:59 [INFO|DP=0|PP=0|TP=0]: iteration: 2381 / 150000 | consumed_samples: 152384 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786679E+05 | tokens_per_sec_per_gpu: 9.041687E+03 | global_batch_size: 64 | lm_loss: 1.081005E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 9738528.000 [ip-26-0-154-121:0]:06/21/2023 17:49:00 [INFO|DP=0|PP=0|TP=0]: iteration: 2382 / 150000 | consumed_samples: 152448 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785895E+05 | tokens_per_sec_per_gpu: 9.040461E+03 | global_batch_size: 64 | lm_loss: 1.081011E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4965230.000 [ip-26-0-154-121:0]:06/21/2023 17:49:01 [INFO|DP=0|PP=0|TP=0]: iteration: 2383 / 150000 | consumed_samples: 152512 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785530E+05 | tokens_per_sec_per_gpu: 9.039891E+03 | global_batch_size: 64 | lm_loss: 1.080923E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3900600.000 [ip-26-0-154-121:0]:06/21/2023 17:49:02 [INFO|DP=0|PP=0|TP=0]: iteration: 2384 / 150000 | consumed_samples: 152576 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785395E+05 | tokens_per_sec_per_gpu: 9.039679E+03 | global_batch_size: 64 | lm_loss: 1.080964E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3074459.250 [ip-26-0-154-121:0]:06/21/2023 17:49:03 [INFO|DP=0|PP=0|TP=0]: iteration: 2385 / 150000 | consumed_samples: 152640 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787426E+05 | tokens_per_sec_per_gpu: 9.042853E+03 | global_batch_size: 64 | lm_loss: 1.080917E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1823020.250 [ip-26-0-154-121:0]:06/21/2023 17:49:04 [INFO|DP=0|PP=0|TP=0]: iteration: 2386 / 150000 | consumed_samples: 152704 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783532E+05 | tokens_per_sec_per_gpu: 9.036769E+03 | global_batch_size: 64 | lm_loss: 1.080939E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3187711.250 [ip-26-0-154-121:0]:06/21/2023 17:49:04 [INFO|DP=0|PP=0|TP=0]: iteration: 2387 / 150000 | consumed_samples: 152768 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785465E+05 | tokens_per_sec_per_gpu: 9.039788E+03 | global_batch_size: 64 | lm_loss: 1.080926E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 4221571.000 [ip-26-0-154-121:0]:06/21/2023 17:49:05 [INFO|DP=0|PP=0|TP=0]: iteration: 2388 / 150000 | consumed_samples: 152832 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786259E+05 | tokens_per_sec_per_gpu: 9.041030E+03 | global_batch_size: 64 | lm_loss: 1.080915E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3718764.500 [ip-26-0-154-121:0]:06/21/2023 17:49:06 [INFO|DP=0|PP=0|TP=0]: iteration: 2389 / 150000 | consumed_samples: 152896 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782547E+05 | tokens_per_sec_per_gpu: 9.035229E+03 | global_batch_size: 64 | lm_loss: 1.080912E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 2633507.000 [ip-26-0-154-121:0]:06/21/2023 17:49:07 [INFO|DP=0|PP=0|TP=0]: iteration: 2390 / 150000 | consumed_samples: 152960 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785480E+05 | tokens_per_sec_per_gpu: 9.039812E+03 | global_batch_size: 64 | lm_loss: 1.080816E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1489603.500 [ip-26-0-154-121:0]:06/21/2023 17:49:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2391 / 150000 | consumed_samples: 153024 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782077E+05 | tokens_per_sec_per_gpu: 9.034495E+03 | global_batch_size: 64 | lm_loss: 1.080930E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 997892.812 [ip-26-0-154-121:0]:06/21/2023 17:49:09 [INFO|DP=0|PP=0|TP=0]: iteration: 2392 / 150000 | consumed_samples: 153088 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787118E+05 | tokens_per_sec_per_gpu: 9.042372E+03 | global_batch_size: 64 | lm_loss: 1.080939E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 972853.812 [ip-26-0-154-121:0]:06/21/2023 17:49:10 [INFO|DP=0|PP=0|TP=0]: iteration: 2393 / 150000 | consumed_samples: 153152 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787787E+05 | tokens_per_sec_per_gpu: 9.043417E+03 | global_batch_size: 64 | lm_loss: 1.080933E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1134371.375 [ip-26-0-154-121:0]:06/21/2023 17:49:11 [INFO|DP=0|PP=0|TP=0]: iteration: 2394 / 150000 | consumed_samples: 153216 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787878E+05 | tokens_per_sec_per_gpu: 9.043560E+03 | global_batch_size: 64 | lm_loss: 1.080942E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1122525.500 [ip-26-0-154-121:0]:06/21/2023 17:49:12 [INFO|DP=0|PP=0|TP=0]: iteration: 2395 / 150000 | consumed_samples: 153280 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784710E+05 | tokens_per_sec_per_gpu: 9.038609E+03 | global_batch_size: 64 | lm_loss: 1.080930E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1093580.625 [ip-26-0-154-121:0]:06/21/2023 17:49:13 [INFO|DP=0|PP=0|TP=0]: iteration: 2396 / 150000 | consumed_samples: 153344 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785690E+05 | tokens_per_sec_per_gpu: 9.040140E+03 | global_batch_size: 64 | lm_loss: 1.080897E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 985409.750 [ip-26-0-154-121:0]:06/21/2023 17:49:14 [INFO|DP=0|PP=0|TP=0]: iteration: 2397 / 150000 | consumed_samples: 153408 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786954E+05 | tokens_per_sec_per_gpu: 9.042115E+03 | global_batch_size: 64 | lm_loss: 1.080922E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 905689.125 [ip-26-0-154-121:0]:06/21/2023 17:49:14 [INFO|DP=0|PP=0|TP=0]: iteration: 2398 / 150000 | consumed_samples: 153472 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789905E+05 | tokens_per_sec_per_gpu: 9.046726E+03 | global_batch_size: 64 | lm_loss: 1.080899E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 866615.375 [ip-26-0-154-121:0]:06/21/2023 17:49:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2399 / 150000 | consumed_samples: 153536 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788494E+05 | tokens_per_sec_per_gpu: 9.044521E+03 | global_batch_size: 64 | lm_loss: 1.080894E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 862607.688 [ip-26-0-154-121:0]:06/21/2023 17:49:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2400 / 150000 | consumed_samples: 153600 | elapsed_time_per_iteration_ms: 905.2 | tokens_per_sec: 5.791718E+05 | tokens_per_sec_per_gpu: 9.049559E+03 | global_batch_size: 64 | lm_loss: 1.080893E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.53 | hardware_tflops_per_gpu: 104.53 | grad_norm: 896086.000 [ip-26-0-154-121:0]:06/21/2023 17:49:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2401 / 150000 | consumed_samples: 153664 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787605E+05 | tokens_per_sec_per_gpu: 9.043133E+03 | global_batch_size: 64 | lm_loss: 1.080893E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 924399.875 [ip-26-0-154-121:0]:06/21/2023 17:49:18 [INFO|DP=0|PP=0|TP=0]: iteration: 2402 / 150000 | consumed_samples: 153728 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786099E+05 | tokens_per_sec_per_gpu: 9.040780E+03 | global_batch_size: 64 | lm_loss: 1.080817E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 946619.938 [ip-26-0-154-121:0]:06/21/2023 17:49:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2403 / 150000 | consumed_samples: 153792 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788667E+05 | tokens_per_sec_per_gpu: 9.044793E+03 | global_batch_size: 64 | lm_loss: 1.080895E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 912289.188 [ip-26-0-154-121:0]:06/21/2023 17:49:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2404 / 150000 | consumed_samples: 153856 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787127E+05 | tokens_per_sec_per_gpu: 9.042386E+03 | global_batch_size: 64 | lm_loss: 1.080905E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 851063.188 [ip-26-0-154-121:0]:06/21/2023 17:49:21 [INFO|DP=0|PP=0|TP=0]: iteration: 2405 / 150000 | consumed_samples: 153920 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785707E+05 | tokens_per_sec_per_gpu: 9.040167E+03 | global_batch_size: 64 | lm_loss: 1.080902E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 842449.188 [ip-26-0-154-121:0]:06/21/2023 17:49:22 [INFO|DP=0|PP=0|TP=0]: iteration: 2406 / 150000 | consumed_samples: 153984 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788932E+05 | tokens_per_sec_per_gpu: 9.045207E+03 | global_batch_size: 64 | lm_loss: 1.080888E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 835330.375 [ip-26-0-154-121:0]:06/21/2023 17:49:23 [INFO|DP=0|PP=0|TP=0]: iteration: 2407 / 150000 | consumed_samples: 154048 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787685E+05 | tokens_per_sec_per_gpu: 9.043257E+03 | global_batch_size: 64 | lm_loss: 1.080888E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 838700.562 [ip-26-0-154-121:0]:06/21/2023 17:49:24 [INFO|DP=0|PP=0|TP=0]: iteration: 2408 / 150000 | consumed_samples: 154112 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786853E+05 | tokens_per_sec_per_gpu: 9.041958E+03 | global_batch_size: 64 | lm_loss: 1.080867E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 844051.250 [ip-26-0-154-121:0]:06/21/2023 17:49:24 [INFO|DP=0|PP=0|TP=0]: iteration: 2409 / 150000 | consumed_samples: 154176 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786961E+05 | tokens_per_sec_per_gpu: 9.042127E+03 | global_batch_size: 64 | lm_loss: 1.080859E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 838850.500 [ip-26-0-154-121:0]:06/21/2023 17:49:25 [INFO|DP=0|PP=0|TP=0]: iteration: 2410 / 150000 | consumed_samples: 154240 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786582E+05 | tokens_per_sec_per_gpu: 9.041534E+03 | global_batch_size: 64 | lm_loss: 1.080869E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 829979.500 [ip-26-0-154-121:0]:06/21/2023 17:49:26 [INFO|DP=0|PP=0|TP=0]: iteration: 2411 / 150000 | consumed_samples: 154304 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785315E+05 | tokens_per_sec_per_gpu: 9.039555E+03 | global_batch_size: 64 | lm_loss: 1.080875E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 819155.250 [ip-26-0-154-121:0]:06/21/2023 17:49:27 [INFO|DP=0|PP=0|TP=0]: iteration: 2412 / 150000 | consumed_samples: 154368 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784599E+05 | tokens_per_sec_per_gpu: 9.038435E+03 | global_batch_size: 64 | lm_loss: 1.080864E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 815701.875 [ip-26-0-154-121:0]:06/21/2023 17:49:28 [INFO|DP=0|PP=0|TP=0]: iteration: 2413 / 150000 | consumed_samples: 154432 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786716E+05 | tokens_per_sec_per_gpu: 9.041744E+03 | global_batch_size: 64 | lm_loss: 1.080831E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 814780.938 [ip-26-0-154-121:0]:06/21/2023 17:49:29 [INFO|DP=0|PP=0|TP=0]: iteration: 2414 / 150000 | consumed_samples: 154496 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787973E+05 | tokens_per_sec_per_gpu: 9.043707E+03 | global_batch_size: 64 | lm_loss: 1.080871E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 816672.000 [ip-26-0-154-121:0]:06/21/2023 17:49:30 [INFO|DP=0|PP=0|TP=0]: iteration: 2415 / 150000 | consumed_samples: 154560 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788934E+05 | tokens_per_sec_per_gpu: 9.045209E+03 | global_batch_size: 64 | lm_loss: 1.080769E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 815272.438 [ip-26-0-154-121:0]:06/21/2023 17:49:31 [INFO|DP=0|PP=0|TP=0]: iteration: 2416 / 150000 | consumed_samples: 154624 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787749E+05 | tokens_per_sec_per_gpu: 9.043357E+03 | global_batch_size: 64 | lm_loss: 1.080842E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 812103.188 [ip-26-0-154-121:0]:06/21/2023 17:49:32 [INFO|DP=0|PP=0|TP=0]: iteration: 2417 / 150000 | consumed_samples: 154688 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787365E+05 | tokens_per_sec_per_gpu: 9.042757E+03 | global_batch_size: 64 | lm_loss: 1.080837E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 808728.625 [ip-26-0-154-121:0]:06/21/2023 17:49:33 [INFO|DP=0|PP=0|TP=0]: iteration: 2418 / 150000 | consumed_samples: 154752 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785577E+05 | tokens_per_sec_per_gpu: 9.039964E+03 | global_batch_size: 64 | lm_loss: 1.080857E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 806789.000 [ip-26-0-154-121:0]:06/21/2023 17:49:33 [INFO|DP=0|PP=0|TP=0]: iteration: 2419 / 150000 | consumed_samples: 154816 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783917E+05 | tokens_per_sec_per_gpu: 9.037370E+03 | global_batch_size: 64 | lm_loss: 1.080857E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 802725.312 [ip-26-0-154-121:0]:06/21/2023 17:49:34 [INFO|DP=0|PP=0|TP=0]: iteration: 2420 / 150000 | consumed_samples: 154880 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785270E+05 | tokens_per_sec_per_gpu: 9.039484E+03 | global_batch_size: 64 | lm_loss: 1.080852E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 798489.812 [ip-26-0-154-121:0]:06/21/2023 17:49:35 [INFO|DP=0|PP=0|TP=0]: iteration: 2421 / 150000 | consumed_samples: 154944 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785385E+05 | tokens_per_sec_per_gpu: 9.039665E+03 | global_batch_size: 64 | lm_loss: 1.080840E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 794633.250 [ip-26-0-154-121:0]:06/21/2023 17:49:36 [INFO|DP=0|PP=0|TP=0]: iteration: 2422 / 150000 | consumed_samples: 155008 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784527E+05 | tokens_per_sec_per_gpu: 9.038324E+03 | global_batch_size: 64 | lm_loss: 1.080827E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 793781.000 [ip-26-0-154-121:0]:06/21/2023 17:49:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2423 / 150000 | consumed_samples: 155072 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785682E+05 | tokens_per_sec_per_gpu: 9.040128E+03 | global_batch_size: 64 | lm_loss: 1.080801E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 796666.938 [ip-26-0-154-121:0]:06/21/2023 17:49:38 [INFO|DP=0|PP=0|TP=0]: iteration: 2424 / 150000 | consumed_samples: 155136 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788466E+05 | tokens_per_sec_per_gpu: 9.044478E+03 | global_batch_size: 64 | lm_loss: 1.080802E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 798014.438 [ip-26-0-154-121:0]:06/21/2023 17:49:39 [INFO|DP=0|PP=0|TP=0]: iteration: 2425 / 150000 | consumed_samples: 155200 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786783E+05 | tokens_per_sec_per_gpu: 9.041848E+03 | global_batch_size: 64 | lm_loss: 1.080814E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 796053.750 [ip-26-0-154-121:0]:06/21/2023 17:49:40 [INFO|DP=0|PP=0|TP=0]: iteration: 2426 / 150000 | consumed_samples: 155264 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787924E+05 | tokens_per_sec_per_gpu: 9.043631E+03 | global_batch_size: 64 | lm_loss: 1.080825E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 792190.188 [ip-26-0-154-121:0]:06/21/2023 17:49:41 [INFO|DP=0|PP=0|TP=0]: iteration: 2427 / 150000 | consumed_samples: 155328 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787284E+05 | tokens_per_sec_per_gpu: 9.042631E+03 | global_batch_size: 64 | lm_loss: 1.080833E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 786913.875 [ip-26-0-154-121:0]:06/21/2023 17:49:42 [INFO|DP=0|PP=0|TP=0]: iteration: 2428 / 150000 | consumed_samples: 155392 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789867E+05 | tokens_per_sec_per_gpu: 9.046667E+03 | global_batch_size: 64 | lm_loss: 1.080824E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 782513.125 [ip-26-0-154-121:0]:06/21/2023 17:49:43 [INFO|DP=0|PP=0|TP=0]: iteration: 2429 / 150000 | consumed_samples: 155456 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788808E+05 | tokens_per_sec_per_gpu: 9.045012E+03 | global_batch_size: 64 | lm_loss: 1.080825E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 778469.000 [ip-26-0-154-121:0]:06/21/2023 17:49:43 [INFO|DP=0|PP=0|TP=0]: iteration: 2430 / 150000 | consumed_samples: 155520 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789382E+05 | tokens_per_sec_per_gpu: 9.045909E+03 | global_batch_size: 64 | lm_loss: 1.080819E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 774544.188 [ip-26-0-154-121:0]:06/21/2023 17:49:44 [INFO|DP=0|PP=0|TP=0]: iteration: 2431 / 150000 | consumed_samples: 155584 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790962E+05 | tokens_per_sec_per_gpu: 9.048377E+03 | global_batch_size: 64 | lm_loss: 1.080810E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 770342.438 [ip-26-0-154-121:0]:06/21/2023 17:49:45 [INFO|DP=0|PP=0|TP=0]: iteration: 2432 / 150000 | consumed_samples: 155648 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786026E+05 | tokens_per_sec_per_gpu: 9.040666E+03 | global_batch_size: 64 | lm_loss: 1.080804E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 768279.062 [ip-26-0-154-121:0]:06/21/2023 17:49:46 [INFO|DP=0|PP=0|TP=0]: iteration: 2433 / 150000 | consumed_samples: 155712 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787270E+05 | tokens_per_sec_per_gpu: 9.042610E+03 | global_batch_size: 64 | lm_loss: 1.080787E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 767615.812 [ip-26-0-154-121:0]:06/21/2023 17:49:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2434 / 150000 | consumed_samples: 155776 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786728E+05 | tokens_per_sec_per_gpu: 9.041763E+03 | global_batch_size: 64 | lm_loss: 1.080770E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 766482.312 [ip-26-0-154-121:0]:06/21/2023 17:49:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2435 / 150000 | consumed_samples: 155840 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784320E+05 | tokens_per_sec_per_gpu: 9.038000E+03 | global_batch_size: 64 | lm_loss: 1.080790E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 764368.062 [ip-26-0-154-121:0]:06/21/2023 17:49:49 [INFO|DP=0|PP=0|TP=0]: iteration: 2436 / 150000 | consumed_samples: 155904 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787860E+05 | tokens_per_sec_per_gpu: 9.043531E+03 | global_batch_size: 64 | lm_loss: 1.080795E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 760200.750 [ip-26-0-154-121:0]:06/21/2023 17:49:50 [INFO|DP=0|PP=0|TP=0]: iteration: 2437 / 150000 | consumed_samples: 155968 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784191E+05 | tokens_per_sec_per_gpu: 9.037798E+03 | global_batch_size: 64 | lm_loss: 1.080793E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 755796.750 [ip-26-0-154-121:0]:06/21/2023 17:49:51 [INFO|DP=0|PP=0|TP=0]: iteration: 2438 / 150000 | consumed_samples: 156032 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787590E+05 | tokens_per_sec_per_gpu: 9.043110E+03 | global_batch_size: 64 | lm_loss: 1.080800E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 752562.125 [ip-26-0-154-121:0]:06/21/2023 17:49:52 [INFO|DP=0|PP=0|TP=0]: iteration: 2439 / 150000 | consumed_samples: 156096 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786591E+05 | tokens_per_sec_per_gpu: 9.041549E+03 | global_batch_size: 64 | lm_loss: 1.080766E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 748629.250 [ip-26-0-154-121:0]:06/21/2023 17:49:53 [INFO|DP=0|PP=0|TP=0]: iteration: 2440 / 150000 | consumed_samples: 156160 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786775E+05 | tokens_per_sec_per_gpu: 9.041837E+03 | global_batch_size: 64 | lm_loss: 1.080795E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 743876.125 [ip-26-0-154-121:0]:06/21/2023 17:49:53 [INFO|DP=0|PP=0|TP=0]: iteration: 2441 / 150000 | consumed_samples: 156224 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788809E+05 | tokens_per_sec_per_gpu: 9.045014E+03 | global_batch_size: 64 | lm_loss: 1.080784E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 739646.812 [ip-26-0-154-121:0]:06/21/2023 17:49:54 [INFO|DP=0|PP=0|TP=0]: iteration: 2442 / 150000 | consumed_samples: 156288 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788673E+05 | tokens_per_sec_per_gpu: 9.044802E+03 | global_batch_size: 64 | lm_loss: 1.080777E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 736848.375 [ip-26-0-154-121:0]:06/21/2023 17:49:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2443 / 150000 | consumed_samples: 156352 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787650E+05 | tokens_per_sec_per_gpu: 9.043203E+03 | global_batch_size: 64 | lm_loss: 1.080750E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 735918.000 [ip-26-0-154-121:0]:06/21/2023 17:49:56 [INFO|DP=0|PP=0|TP=0]: iteration: 2444 / 150000 | consumed_samples: 156416 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789783E+05 | tokens_per_sec_per_gpu: 9.046536E+03 | global_batch_size: 64 | lm_loss: 1.080754E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 737153.062 [ip-26-0-154-121:0]:06/21/2023 17:49:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2445 / 150000 | consumed_samples: 156480 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790131E+05 | tokens_per_sec_per_gpu: 9.047079E+03 | global_batch_size: 64 | lm_loss: 1.080754E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 738091.938 [ip-26-0-154-121:0]:06/21/2023 17:49:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2446 / 150000 | consumed_samples: 156544 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791396E+05 | tokens_per_sec_per_gpu: 9.049057E+03 | global_batch_size: 64 | lm_loss: 1.080768E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.53 | hardware_tflops_per_gpu: 104.53 | grad_norm: 736718.750 [ip-26-0-154-121:0]:06/21/2023 17:49:59 [INFO|DP=0|PP=0|TP=0]: iteration: 2447 / 150000 | consumed_samples: 156608 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789867E+05 | tokens_per_sec_per_gpu: 9.046667E+03 | global_batch_size: 64 | lm_loss: 1.080748E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 734785.375 [ip-26-0-154-121:0]:06/21/2023 17:50:00 [INFO|DP=0|PP=0|TP=0]: iteration: 2448 / 150000 | consumed_samples: 156672 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788789E+05 | tokens_per_sec_per_gpu: 9.044983E+03 | global_batch_size: 64 | lm_loss: 1.080752E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 731366.312 [ip-26-0-154-121:0]:06/21/2023 17:50:01 [INFO|DP=0|PP=0|TP=0]: iteration: 2449 / 150000 | consumed_samples: 156736 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790220E+05 | tokens_per_sec_per_gpu: 9.047220E+03 | global_batch_size: 64 | lm_loss: 1.080759E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 728005.750 [ip-26-0-154-121:0]:06/21/2023 17:50:02 [INFO|DP=0|PP=0|TP=0]: iteration: 2450 / 150000 | consumed_samples: 156800 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788370E+05 | tokens_per_sec_per_gpu: 9.044328E+03 | global_batch_size: 64 | lm_loss: 1.080765E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 727778.125 [ip-26-0-154-121:0]:06/21/2023 17:50:03 [INFO|DP=0|PP=0|TP=0]: iteration: 2451 / 150000 | consumed_samples: 156864 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787206E+05 | tokens_per_sec_per_gpu: 9.042510E+03 | global_batch_size: 64 | lm_loss: 1.080757E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 726352.688 [ip-26-0-154-121:0]:06/21/2023 17:50:03 [INFO|DP=0|PP=0|TP=0]: iteration: 2452 / 150000 | consumed_samples: 156928 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784165E+05 | tokens_per_sec_per_gpu: 9.037758E+03 | global_batch_size: 64 | lm_loss: 1.080748E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 722940.000 [ip-26-0-154-121:0]:06/21/2023 17:50:04 [INFO|DP=0|PP=0|TP=0]: iteration: 2453 / 150000 | consumed_samples: 156992 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787749E+05 | tokens_per_sec_per_gpu: 9.043357E+03 | global_batch_size: 64 | lm_loss: 1.080755E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 723682.625 [ip-26-0-154-121:0]:06/21/2023 17:50:05 [INFO|DP=0|PP=0|TP=0]: iteration: 2454 / 150000 | consumed_samples: 157056 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784136E+05 | tokens_per_sec_per_gpu: 9.037713E+03 | global_batch_size: 64 | lm_loss: 1.080723E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 724172.125 [ip-26-0-154-121:0]:06/21/2023 17:50:06 [INFO|DP=0|PP=0|TP=0]: iteration: 2455 / 150000 | consumed_samples: 157120 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784554E+05 | tokens_per_sec_per_gpu: 9.038366E+03 | global_batch_size: 64 | lm_loss: 1.080734E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 721510.312 [ip-26-0-154-121:0]:06/21/2023 17:50:07 [INFO|DP=0|PP=0|TP=0]: iteration: 2456 / 150000 | consumed_samples: 157184 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781184E+05 | tokens_per_sec_per_gpu: 9.033101E+03 | global_batch_size: 64 | lm_loss: 1.080656E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 723152.438 [ip-26-0-154-121:0]:06/21/2023 17:50:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2457 / 150000 | consumed_samples: 157248 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787730E+05 | tokens_per_sec_per_gpu: 9.043329E+03 | global_batch_size: 64 | lm_loss: 1.080720E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 724700.250 [ip-26-0-154-121:0]:06/21/2023 17:50:09 [INFO|DP=0|PP=0|TP=0]: iteration: 2458 / 150000 | consumed_samples: 157312 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783409E+05 | tokens_per_sec_per_gpu: 9.036576E+03 | global_batch_size: 64 | lm_loss: 1.080734E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 727803.750 [ip-26-0-154-121:0]:06/21/2023 17:50:10 [INFO|DP=0|PP=0|TP=0]: iteration: 2459 / 150000 | consumed_samples: 157376 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787968E+05 | tokens_per_sec_per_gpu: 9.043700E+03 | global_batch_size: 64 | lm_loss: 1.080749E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 732442.812 [ip-26-0-154-121:0]:06/21/2023 17:50:11 [INFO|DP=0|PP=0|TP=0]: iteration: 2460 / 150000 | consumed_samples: 157440 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789332E+05 | tokens_per_sec_per_gpu: 9.045831E+03 | global_batch_size: 64 | lm_loss: 1.080719E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 737653.375 [ip-26-0-154-121:0]:06/21/2023 17:50:12 [INFO|DP=0|PP=0|TP=0]: iteration: 2461 / 150000 | consumed_samples: 157504 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789713E+05 | tokens_per_sec_per_gpu: 9.046426E+03 | global_batch_size: 64 | lm_loss: 1.080725E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 735911.188 [ip-26-0-154-121:0]:06/21/2023 17:50:12 [INFO|DP=0|PP=0|TP=0]: iteration: 2462 / 150000 | consumed_samples: 157568 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783611E+05 | tokens_per_sec_per_gpu: 9.036893E+03 | global_batch_size: 64 | lm_loss: 1.080726E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 747946.125 [ip-26-0-154-121:0]:06/21/2023 17:50:13 [INFO|DP=0|PP=0|TP=0]: iteration: 2463 / 150000 | consumed_samples: 157632 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786378E+05 | tokens_per_sec_per_gpu: 9.041216E+03 | global_batch_size: 64 | lm_loss: 1.080719E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 771777.938 [ip-26-0-154-121:0]:06/21/2023 17:50:14 [INFO|DP=0|PP=0|TP=0]: iteration: 2464 / 150000 | consumed_samples: 157696 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787808E+05 | tokens_per_sec_per_gpu: 9.043450E+03 | global_batch_size: 64 | lm_loss: 1.080721E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 789028.188 [ip-26-0-154-121:0]:06/21/2023 17:50:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2465 / 150000 | consumed_samples: 157760 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785871E+05 | tokens_per_sec_per_gpu: 9.040423E+03 | global_batch_size: 64 | lm_loss: 1.080497E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 803580.750 [ip-26-0-154-121:0]:06/21/2023 17:50:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2466 / 150000 | consumed_samples: 157824 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784818E+05 | tokens_per_sec_per_gpu: 9.038778E+03 | global_batch_size: 64 | lm_loss: 1.080719E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 907734.188 [ip-26-0-154-121:0]:06/21/2023 17:50:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2467 / 150000 | consumed_samples: 157888 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788014E+05 | tokens_per_sec_per_gpu: 9.043771E+03 | global_batch_size: 64 | lm_loss: 1.080552E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 935924.375 [ip-26-0-154-121:0]:06/21/2023 17:50:18 [INFO|DP=0|PP=0|TP=0]: iteration: 2468 / 150000 | consumed_samples: 157952 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783255E+05 | tokens_per_sec_per_gpu: 9.036336E+03 | global_batch_size: 64 | lm_loss: 1.080701E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 949535.688 [ip-26-0-154-121:0]:06/21/2023 17:50:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2469 / 150000 | consumed_samples: 158016 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789551E+05 | tokens_per_sec_per_gpu: 9.046174E+03 | global_batch_size: 64 | lm_loss: 1.080695E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 1190386.750 [ip-26-0-154-121:0]:06/21/2023 17:50:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2470 / 150000 | consumed_samples: 158080 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787423E+05 | tokens_per_sec_per_gpu: 9.042848E+03 | global_batch_size: 64 | lm_loss: 1.080665E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1686736.000 [ip-26-0-154-121:0]:06/21/2023 17:50:21 [INFO|DP=0|PP=0|TP=0]: iteration: 2471 / 150000 | consumed_samples: 158144 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788900E+05 | tokens_per_sec_per_gpu: 9.045157E+03 | global_batch_size: 64 | lm_loss: 1.080689E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 2068591.125 [ip-26-0-154-121:0]:06/21/2023 17:50:22 [INFO|DP=0|PP=0|TP=0]: iteration: 2472 / 150000 | consumed_samples: 158208 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787927E+05 | tokens_per_sec_per_gpu: 9.043636E+03 | global_batch_size: 64 | lm_loss: 1.080717E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1836996.625 [ip-26-0-154-121:0]:06/21/2023 17:50:22 [INFO|DP=0|PP=0|TP=0]: iteration: 2473 / 150000 | consumed_samples: 158272 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789993E+05 | tokens_per_sec_per_gpu: 9.046865E+03 | global_batch_size: 64 | lm_loss: 1.080674E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 1712608.750 [ip-26-0-154-121:0]:06/21/2023 17:50:23 [INFO|DP=0|PP=0|TP=0]: iteration: 2474 / 150000 | consumed_samples: 158336 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789992E+05 | tokens_per_sec_per_gpu: 9.046862E+03 | global_batch_size: 64 | lm_loss: 1.080711E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 2673249.250 [ip-26-0-154-121:0]:06/21/2023 17:50:24 [INFO|DP=0|PP=0|TP=0]: iteration: 2475 / 150000 | consumed_samples: 158400 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788533E+05 | tokens_per_sec_per_gpu: 9.044583E+03 | global_batch_size: 64 | lm_loss: 1.080721E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 3064807.000 [ip-26-0-154-121:0]:06/21/2023 17:50:25 [INFO|DP=0|PP=0|TP=0]: iteration: 2476 / 150000 | consumed_samples: 158464 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789440E+05 | tokens_per_sec_per_gpu: 9.046000E+03 | global_batch_size: 64 | lm_loss: 1.080729E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 4681410.500 [ip-26-0-154-121:0]:06/21/2023 17:50:26 [INFO|DP=0|PP=0|TP=0]: iteration: 2477 / 150000 | consumed_samples: 158528 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791256E+05 | tokens_per_sec_per_gpu: 9.048837E+03 | global_batch_size: 64 | lm_loss: 1.080723E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 14152011.000 [ip-26-0-154-121:0]:06/21/2023 17:50:27 [INFO|DP=0|PP=0|TP=0]: iteration: 2478 / 150000 | consumed_samples: 158592 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783852E+05 | tokens_per_sec_per_gpu: 9.037268E+03 | global_batch_size: 64 | lm_loss: 1.080704E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 250447872.000 [ip-26-0-154-121:0]:06/21/2023 17:50:28 [INFO|DP=0|PP=0|TP=0]: iteration: 2479 / 150000 | consumed_samples: 158656 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785854E+05 | tokens_per_sec_per_gpu: 9.040397E+03 | global_batch_size: 64 | lm_loss: 1.080687E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 8036599.500 [ip-26-0-154-121:0]:06/21/2023 17:50:29 [INFO|DP=0|PP=0|TP=0]: iteration: 2480 / 150000 | consumed_samples: 158720 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788680E+05 | tokens_per_sec_per_gpu: 9.044812E+03 | global_batch_size: 64 | lm_loss: 1.080656E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 5893536.500 [ip-26-0-154-121:0]:06/21/2023 17:50:30 [INFO|DP=0|PP=0|TP=0]: iteration: 2481 / 150000 | consumed_samples: 158784 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788247E+05 | tokens_per_sec_per_gpu: 9.044136E+03 | global_batch_size: 64 | lm_loss: 1.080637E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 8286018.000 [ip-26-0-154-121:0]:06/21/2023 17:50:31 [INFO|DP=0|PP=0|TP=0]: iteration: 2482 / 150000 | consumed_samples: 158848 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785013E+05 | tokens_per_sec_per_gpu: 9.039082E+03 | global_batch_size: 64 | lm_loss: 1.080653E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 2489985.750 [ip-26-0-154-121:0]:06/21/2023 17:50:32 [INFO|DP=0|PP=0|TP=0]: iteration: 2483 / 150000 | consumed_samples: 158912 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790411E+05 | tokens_per_sec_per_gpu: 9.047517E+03 | global_batch_size: 64 | lm_loss: 1.080690E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 35113916.000 [ip-26-0-154-121:0]:06/21/2023 17:50:32 [INFO|DP=0|PP=0|TP=0]: iteration: 2484 / 150000 | consumed_samples: 158976 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786880E+05 | tokens_per_sec_per_gpu: 9.042001E+03 | global_batch_size: 64 | lm_loss: 1.080700E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 8387492.500 [ip-26-0-154-121:0]:06/21/2023 17:50:33 [INFO|DP=0|PP=0|TP=0]: iteration: 2485 / 150000 | consumed_samples: 159040 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784378E+05 | tokens_per_sec_per_gpu: 9.038091E+03 | global_batch_size: 64 | lm_loss: 1.080724E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 928371.625 [ip-26-0-154-121:0]:06/21/2023 17:50:34 [INFO|DP=0|PP=0|TP=0]: iteration: 2486 / 150000 | consumed_samples: 159104 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784863E+05 | tokens_per_sec_per_gpu: 9.038849E+03 | global_batch_size: 64 | lm_loss: 1.080715E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 16968956.000 [ip-26-0-154-121:0]:06/21/2023 17:50:35 [INFO|DP=0|PP=0|TP=0]: iteration: 2487 / 150000 | consumed_samples: 159168 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787053E+05 | tokens_per_sec_per_gpu: 9.042270E+03 | global_batch_size: 64 | lm_loss: 1.080694E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1698040.750 [ip-26-0-154-121:0]:06/21/2023 17:50:36 [INFO|DP=0|PP=0|TP=0]: iteration: 2488 / 150000 | consumed_samples: 159232 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789128E+05 | tokens_per_sec_per_gpu: 9.045512E+03 | global_batch_size: 64 | lm_loss: 1.080688E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 1493329.125 [ip-26-0-154-121:0]:06/21/2023 17:50:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2489 / 150000 | consumed_samples: 159296 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786622E+05 | tokens_per_sec_per_gpu: 9.041596E+03 | global_batch_size: 64 | lm_loss: 1.080673E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1654163.875 [ip-26-0-154-121:0]:06/21/2023 17:50:38 [INFO|DP=0|PP=0|TP=0]: iteration: 2490 / 150000 | consumed_samples: 159360 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785381E+05 | tokens_per_sec_per_gpu: 9.039658E+03 | global_batch_size: 64 | lm_loss: 1.080690E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 21502974.000 [ip-26-0-154-121:0]:06/21/2023 17:50:39 [INFO|DP=0|PP=0|TP=0]: iteration: 2491 / 150000 | consumed_samples: 159424 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785871E+05 | tokens_per_sec_per_gpu: 9.040423E+03 | global_batch_size: 64 | lm_loss: 1.080673E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1299496.000 [ip-26-0-154-121:0]:06/21/2023 17:50:40 [INFO|DP=0|PP=0|TP=0]: iteration: 2492 / 150000 | consumed_samples: 159488 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786652E+05 | tokens_per_sec_per_gpu: 9.041644E+03 | global_batch_size: 64 | lm_loss: 1.080689E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 931486.812 [ip-26-0-154-121:0]:06/21/2023 17:50:41 [INFO|DP=0|PP=0|TP=0]: iteration: 2493 / 150000 | consumed_samples: 159552 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786850E+05 | tokens_per_sec_per_gpu: 9.041953E+03 | global_batch_size: 64 | lm_loss: 1.080685E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 10735852.000 [ip-26-0-154-121:0]:06/21/2023 17:50:42 [INFO|DP=0|PP=0|TP=0]: iteration: 2494 / 150000 | consumed_samples: 159616 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789105E+05 | tokens_per_sec_per_gpu: 9.045476E+03 | global_batch_size: 64 | lm_loss: 1.080688E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 7831146.500 [ip-26-0-154-121:0]:06/21/2023 17:50:42 [INFO|DP=0|PP=0|TP=0]: iteration: 2495 / 150000 | consumed_samples: 159680 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782103E+05 | tokens_per_sec_per_gpu: 9.034535E+03 | global_batch_size: 64 | lm_loss: 1.080687E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 6466560.500 [ip-26-0-154-121:0]:06/21/2023 17:50:43 [INFO|DP=0|PP=0|TP=0]: iteration: 2496 / 150000 | consumed_samples: 159744 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787499E+05 | tokens_per_sec_per_gpu: 9.042967E+03 | global_batch_size: 64 | lm_loss: 1.080683E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 5964994.500 [ip-26-0-154-121:0]:06/21/2023 17:50:44 [INFO|DP=0|PP=0|TP=0]: iteration: 2497 / 150000 | consumed_samples: 159808 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784602E+05 | tokens_per_sec_per_gpu: 9.038440E+03 | global_batch_size: 64 | lm_loss: 1.080668E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 5590981.500 [ip-26-0-154-121:0]:06/21/2023 17:50:45 [INFO|DP=0|PP=0|TP=0]: iteration: 2498 / 150000 | consumed_samples: 159872 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785771E+05 | tokens_per_sec_per_gpu: 9.040266E+03 | global_batch_size: 64 | lm_loss: 1.080696E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4093063.500 [ip-26-0-154-121:0]:06/21/2023 17:50:46 [INFO|DP=0|PP=0|TP=0]: iteration: 2499 / 150000 | consumed_samples: 159936 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788800E+05 | tokens_per_sec_per_gpu: 9.045000E+03 | global_batch_size: 64 | lm_loss: 1.080685E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 3182391.250 [ip-26-0-154-121:0]:06/21/2023 17:50:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2500 / 150000 | consumed_samples: 160000 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788868E+05 | tokens_per_sec_per_gpu: 9.045107E+03 | global_batch_size: 64 | lm_loss: 1.080680E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 47014264.000 [ip-26-0-154-121:0]:06/21/2023 17:50:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2501 / 150000 | consumed_samples: 160064 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787484E+05 | tokens_per_sec_per_gpu: 9.042943E+03 | global_batch_size: 64 | lm_loss: 1.080675E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1775903.625 [ip-26-0-154-121:0]:06/21/2023 17:50:49 [INFO|DP=0|PP=0|TP=0]: iteration: 2502 / 150000 | consumed_samples: 160128 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790428E+05 | tokens_per_sec_per_gpu: 9.047544E+03 | global_batch_size: 64 | lm_loss: 1.080674E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 1518554.000 [ip-26-0-154-121:0]:06/21/2023 17:50:50 [INFO|DP=0|PP=0|TP=0]: iteration: 2503 / 150000 | consumed_samples: 160192 | elapsed_time_per_iteration_ms: 905.1 | tokens_per_sec: 5.792859E+05 | tokens_per_sec_per_gpu: 9.051343E+03 | global_batch_size: 64 | lm_loss: 1.080671E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.55 | hardware_tflops_per_gpu: 104.55 | grad_norm: 3738902.250 [ip-26-0-154-121:0]:06/21/2023 17:50:51 [INFO|DP=0|PP=0|TP=0]: iteration: 2504 / 150000 | consumed_samples: 160256 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790576E+05 | tokens_per_sec_per_gpu: 9.047775E+03 | global_batch_size: 64 | lm_loss: 1.080621E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 6470164.500 [ip-26-0-154-121:0]:06/21/2023 17:50:51 [INFO|DP=0|PP=0|TP=0]: iteration: 2505 / 150000 | consumed_samples: 160320 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789591E+05 | tokens_per_sec_per_gpu: 9.046236E+03 | global_batch_size: 64 | lm_loss: 1.080658E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 3876494.750 [ip-26-0-154-121:0]:06/21/2023 17:50:52 [INFO|DP=0|PP=0|TP=0]: iteration: 2506 / 150000 | consumed_samples: 160384 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783547E+05 | tokens_per_sec_per_gpu: 9.036793E+03 | global_batch_size: 64 | lm_loss: 1.080655E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3324670.500 [ip-26-0-154-121:0]:06/21/2023 17:50:53 [INFO|DP=0|PP=0|TP=0]: iteration: 2507 / 150000 | consumed_samples: 160448 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789691E+05 | tokens_per_sec_per_gpu: 9.046393E+03 | global_batch_size: 64 | lm_loss: 1.080623E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 4795861.000 [ip-26-0-154-121:0]:06/21/2023 17:50:54 [INFO|DP=0|PP=0|TP=0]: iteration: 2508 / 150000 | consumed_samples: 160512 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790472E+05 | tokens_per_sec_per_gpu: 9.047613E+03 | global_batch_size: 64 | lm_loss: 1.080647E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 11675610.000 [ip-26-0-154-121:0]:06/21/2023 17:50:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2509 / 150000 | consumed_samples: 160576 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787315E+05 | tokens_per_sec_per_gpu: 9.042679E+03 | global_batch_size: 64 | lm_loss: 1.080599E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 24467686.000 [ip-26-0-154-121:0]:06/21/2023 17:50:56 [INFO|DP=0|PP=0|TP=0]: iteration: 2510 / 150000 | consumed_samples: 160640 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785860E+05 | tokens_per_sec_per_gpu: 9.040407E+03 | global_batch_size: 64 | lm_loss: 1.080570E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 6991938.500 [ip-26-0-154-121:0]:06/21/2023 17:50:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2511 / 150000 | consumed_samples: 160704 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791286E+05 | tokens_per_sec_per_gpu: 9.048885E+03 | global_batch_size: 64 | lm_loss: 1.080596E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 1341169.500 [ip-26-0-154-121:0]:06/21/2023 17:50:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2512 / 150000 | consumed_samples: 160768 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784696E+05 | tokens_per_sec_per_gpu: 9.038587E+03 | global_batch_size: 64 | lm_loss: 1.080625E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 17415874.000 [ip-26-0-154-121:0]:06/21/2023 17:50:59 [INFO|DP=0|PP=0|TP=0]: iteration: 2513 / 150000 | consumed_samples: 160832 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786689E+05 | tokens_per_sec_per_gpu: 9.041701E+03 | global_batch_size: 64 | lm_loss: 1.080664E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 20658654.000 [ip-26-0-154-121:0]:06/21/2023 17:51:00 [INFO|DP=0|PP=0|TP=0]: iteration: 2514 / 150000 | consumed_samples: 160896 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781867E+05 | tokens_per_sec_per_gpu: 9.034167E+03 | global_batch_size: 64 | lm_loss: 1.080692E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 9363021.000 [ip-26-0-154-121:0]:06/21/2023 17:51:01 [INFO|DP=0|PP=0|TP=0]: iteration: 2515 / 150000 | consumed_samples: 160960 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780394E+05 | tokens_per_sec_per_gpu: 9.031866E+03 | global_batch_size: 64 | lm_loss: 1.080685E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 9761151.000 [ip-26-0-154-121:0]:06/21/2023 17:51:01 [INFO|DP=0|PP=0|TP=0]: iteration: 2516 / 150000 | consumed_samples: 161024 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784052E+05 | tokens_per_sec_per_gpu: 9.037582E+03 | global_batch_size: 64 | lm_loss: 1.080705E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 13888440.000 [ip-26-0-154-121:0]:06/21/2023 17:51:02 [INFO|DP=0|PP=0|TP=0]: iteration: 2517 / 150000 | consumed_samples: 161088 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789687E+05 | tokens_per_sec_per_gpu: 9.046386E+03 | global_batch_size: 64 | lm_loss: 1.080681E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 12533024.000 [ip-26-0-154-121:0]:06/21/2023 17:51:03 [INFO|DP=0|PP=0|TP=0]: iteration: 2518 / 150000 | consumed_samples: 161152 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782811E+05 | tokens_per_sec_per_gpu: 9.035643E+03 | global_batch_size: 64 | lm_loss: 1.080666E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 5338948.500 [ip-26-0-154-121:0]:06/21/2023 17:51:04 [INFO|DP=0|PP=0|TP=0]: iteration: 2519 / 150000 | consumed_samples: 161216 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785950E+05 | tokens_per_sec_per_gpu: 9.040547E+03 | global_batch_size: 64 | lm_loss: 1.080674E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4210091.000 [ip-26-0-154-121:0]:06/21/2023 17:51:05 [INFO|DP=0|PP=0|TP=0]: iteration: 2520 / 150000 | consumed_samples: 161280 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780116E+05 | tokens_per_sec_per_gpu: 9.031432E+03 | global_batch_size: 64 | lm_loss: 1.080667E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 13621026.000 [ip-26-0-154-121:0]:06/21/2023 17:51:06 [INFO|DP=0|PP=0|TP=0]: iteration: 2521 / 150000 | consumed_samples: 161344 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787855E+05 | tokens_per_sec_per_gpu: 9.043524E+03 | global_batch_size: 64 | lm_loss: 1.080643E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 9288657.000 [ip-26-0-154-121:0]:06/21/2023 17:51:07 [INFO|DP=0|PP=0|TP=0]: iteration: 2522 / 150000 | consumed_samples: 161408 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782960E+05 | tokens_per_sec_per_gpu: 9.035875E+03 | global_batch_size: 64 | lm_loss: 1.080632E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 5865010.000 [ip-26-0-154-121:0]:06/21/2023 17:51:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2523 / 150000 | consumed_samples: 161472 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783917E+05 | tokens_per_sec_per_gpu: 9.037370E+03 | global_batch_size: 64 | lm_loss: 1.080636E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 5507922.500 [ip-26-0-154-121:0]:06/21/2023 17:51:09 [INFO|DP=0|PP=0|TP=0]: iteration: 2524 / 150000 | consumed_samples: 161536 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779890E+05 | tokens_per_sec_per_gpu: 9.031078E+03 | global_batch_size: 64 | lm_loss: 1.080635E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 6827937.000 [ip-26-0-154-121:0]:06/21/2023 17:51:10 [INFO|DP=0|PP=0|TP=0]: iteration: 2525 / 150000 | consumed_samples: 161600 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776805E+05 | tokens_per_sec_per_gpu: 9.026257E+03 | global_batch_size: 64 | lm_loss: 1.080642E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 9742687.000 [ip-26-0-154-121:0]:06/21/2023 17:51:11 [INFO|DP=0|PP=0|TP=0]: iteration: 2526 / 150000 | consumed_samples: 161664 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782191E+05 | tokens_per_sec_per_gpu: 9.034673E+03 | global_batch_size: 64 | lm_loss: 1.080636E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3301255.250 [ip-26-0-154-121:0]:06/21/2023 17:51:11 [INFO|DP=0|PP=0|TP=0]: iteration: 2527 / 150000 | consumed_samples: 161728 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779253E+05 | tokens_per_sec_per_gpu: 9.030083E+03 | global_batch_size: 64 | lm_loss: 1.080683E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 4273809.500 [ip-26-0-154-121:0]:06/21/2023 17:51:12 [INFO|DP=0|PP=0|TP=0]: iteration: 2528 / 150000 | consumed_samples: 161792 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784901E+05 | tokens_per_sec_per_gpu: 9.038908E+03 | global_batch_size: 64 | lm_loss: 1.080673E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 6355797.500 [ip-26-0-154-121:0]:06/21/2023 17:51:13 [INFO|DP=0|PP=0|TP=0]: iteration: 2529 / 150000 | consumed_samples: 161856 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784748E+05 | tokens_per_sec_per_gpu: 9.038668E+03 | global_batch_size: 64 | lm_loss: 1.080682E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 7550800.000 [ip-26-0-154-121:0]:06/21/2023 17:51:14 [INFO|DP=0|PP=0|TP=0]: iteration: 2530 / 150000 | consumed_samples: 161920 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779323E+05 | tokens_per_sec_per_gpu: 9.030193E+03 | global_batch_size: 64 | lm_loss: 1.080681E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 3017429.250 [ip-26-0-154-121:0]:06/21/2023 17:51:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2531 / 150000 | consumed_samples: 161984 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783801E+05 | tokens_per_sec_per_gpu: 9.037190E+03 | global_batch_size: 64 | lm_loss: 1.080655E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 7764035.500 [ip-26-0-154-121:0]:06/21/2023 17:51:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2532 / 150000 | consumed_samples: 162048 | elapsed_time_per_iteration_ms: 907.7 | tokens_per_sec: 5.776009E+05 | tokens_per_sec_per_gpu: 9.025015E+03 | global_batch_size: 64 | lm_loss: 1.080624E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.25 | hardware_tflops_per_gpu: 104.25 | grad_norm: 5539417.500 [ip-26-0-154-121:0]:06/21/2023 17:51:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2533 / 150000 | consumed_samples: 162112 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780592E+05 | tokens_per_sec_per_gpu: 9.032175E+03 | global_batch_size: 64 | lm_loss: 1.080602E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 4145771.000 [ip-26-0-154-121:0]:06/21/2023 17:51:18 [INFO|DP=0|PP=0|TP=0]: iteration: 2534 / 150000 | consumed_samples: 162176 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783249E+05 | tokens_per_sec_per_gpu: 9.036327E+03 | global_batch_size: 64 | lm_loss: 1.080588E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3166833.750 [ip-26-0-154-121:0]:06/21/2023 17:51:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2535 / 150000 | consumed_samples: 162240 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787180E+05 | tokens_per_sec_per_gpu: 9.042469E+03 | global_batch_size: 64 | lm_loss: 1.080616E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 2896450.000 [ip-26-0-154-121:0]:06/21/2023 17:51:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2536 / 150000 | consumed_samples: 162304 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779001E+05 | tokens_per_sec_per_gpu: 9.029690E+03 | global_batch_size: 64 | lm_loss: 1.080626E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 3213968.000 [ip-26-0-154-121:0]:06/21/2023 17:51:21 [INFO|DP=0|PP=0|TP=0]: iteration: 2537 / 150000 | consumed_samples: 162368 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786087E+05 | tokens_per_sec_per_gpu: 9.040761E+03 | global_batch_size: 64 | lm_loss: 1.080659E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 4151961.500 [ip-26-0-154-121:0]:06/21/2023 17:51:21 [INFO|DP=0|PP=0|TP=0]: iteration: 2538 / 150000 | consumed_samples: 162432 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780592E+05 | tokens_per_sec_per_gpu: 9.032175E+03 | global_batch_size: 64 | lm_loss: 1.080646E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 3650020.750 [ip-26-0-154-121:0]:06/21/2023 17:51:22 [INFO|DP=0|PP=0|TP=0]: iteration: 2539 / 150000 | consumed_samples: 162496 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780604E+05 | tokens_per_sec_per_gpu: 9.032194E+03 | global_batch_size: 64 | lm_loss: 1.080622E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 2365692.500 [ip-26-0-154-121:0]:06/21/2023 17:51:23 [INFO|DP=0|PP=0|TP=0]: iteration: 2540 / 150000 | consumed_samples: 162560 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778058E+05 | tokens_per_sec_per_gpu: 9.028216E+03 | global_batch_size: 64 | lm_loss: 1.080617E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 1550571.500 [ip-26-0-154-121:0]:06/21/2023 17:51:24 [INFO|DP=0|PP=0|TP=0]: iteration: 2541 / 150000 | consumed_samples: 162624 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779761E+05 | tokens_per_sec_per_gpu: 9.030876E+03 | global_batch_size: 64 | lm_loss: 1.080607E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 2192941.000 [ip-26-0-154-121:0]:06/21/2023 17:51:25 [INFO|DP=0|PP=0|TP=0]: iteration: 2542 / 150000 | consumed_samples: 162688 | elapsed_time_per_iteration_ms: 908.7 | tokens_per_sec: 5.769726E+05 | tokens_per_sec_per_gpu: 9.015197E+03 | global_batch_size: 64 | lm_loss: 1.080637E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.14 | hardware_tflops_per_gpu: 104.14 | grad_norm: 2346137.250 [ip-26-0-154-121:0]:06/21/2023 17:51:26 [INFO|DP=0|PP=0|TP=0]: iteration: 2543 / 150000 | consumed_samples: 162752 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778532E+05 | tokens_per_sec_per_gpu: 9.028956E+03 | global_batch_size: 64 | lm_loss: 1.080626E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 1748223.125 [ip-26-0-154-121:0]:06/21/2023 17:51:27 [INFO|DP=0|PP=0|TP=0]: iteration: 2544 / 150000 | consumed_samples: 162816 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786471E+05 | tokens_per_sec_per_gpu: 9.041361E+03 | global_batch_size: 64 | lm_loss: 1.080632E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 937159.812 [ip-26-0-154-121:0]:06/21/2023 17:51:28 [INFO|DP=0|PP=0|TP=0]: iteration: 2545 / 150000 | consumed_samples: 162880 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783845E+05 | tokens_per_sec_per_gpu: 9.037259E+03 | global_batch_size: 64 | lm_loss: 1.080626E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1555178.875 [ip-26-0-154-121:0]:06/21/2023 17:51:29 [INFO|DP=0|PP=0|TP=0]: iteration: 2546 / 150000 | consumed_samples: 162944 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778757E+05 | tokens_per_sec_per_gpu: 9.029307E+03 | global_batch_size: 64 | lm_loss: 1.080618E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 1302290.000 [ip-26-0-154-121:0]:06/21/2023 17:51:30 [INFO|DP=0|PP=0|TP=0]: iteration: 2547 / 150000 | consumed_samples: 163008 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785629E+05 | tokens_per_sec_per_gpu: 9.040045E+03 | global_batch_size: 64 | lm_loss: 1.080622E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 604051.750 [ip-26-0-154-121:0]:06/21/2023 17:51:31 [INFO|DP=0|PP=0|TP=0]: iteration: 2548 / 150000 | consumed_samples: 163072 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780973E+05 | tokens_per_sec_per_gpu: 9.032771E+03 | global_batch_size: 64 | lm_loss: 1.080629E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 1283337.375 [ip-26-0-154-121:0]:06/21/2023 17:51:31 [INFO|DP=0|PP=0|TP=0]: iteration: 2549 / 150000 | consumed_samples: 163136 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782808E+05 | tokens_per_sec_per_gpu: 9.035638E+03 | global_batch_size: 64 | lm_loss: 1.080630E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 1021075.000 [ip-26-0-154-121:0]:06/21/2023 17:51:32 [INFO|DP=0|PP=0|TP=0]: iteration: 2550 / 150000 | consumed_samples: 163200 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781089E+05 | tokens_per_sec_per_gpu: 9.032951E+03 | global_batch_size: 64 | lm_loss: 1.080600E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 656130.250 [ip-26-0-154-121:0]:06/21/2023 17:51:33 [INFO|DP=0|PP=0|TP=0]: iteration: 2551 / 150000 | consumed_samples: 163264 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784174E+05 | tokens_per_sec_per_gpu: 9.037772E+03 | global_batch_size: 64 | lm_loss: 1.080591E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 620501.375 [ip-26-0-154-121:0]:06/21/2023 17:51:34 [INFO|DP=0|PP=0|TP=0]: iteration: 2552 / 150000 | consumed_samples: 163328 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783275E+05 | tokens_per_sec_per_gpu: 9.036367E+03 | global_batch_size: 64 | lm_loss: 1.080601E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 620566.000 [ip-26-0-154-121:0]:06/21/2023 17:51:35 [INFO|DP=0|PP=0|TP=0]: iteration: 2553 / 150000 | consumed_samples: 163392 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786794E+05 | tokens_per_sec_per_gpu: 9.041865E+03 | global_batch_size: 64 | lm_loss: 1.080605E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 745050.625 [ip-26-0-154-121:0]:06/21/2023 17:51:36 [INFO|DP=0|PP=0|TP=0]: iteration: 2554 / 150000 | consumed_samples: 163456 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785334E+05 | tokens_per_sec_per_gpu: 9.039584E+03 | global_batch_size: 64 | lm_loss: 1.080593E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 620107.500 [ip-26-0-154-121:0]:06/21/2023 17:51:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2555 / 150000 | consumed_samples: 163520 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790439E+05 | tokens_per_sec_per_gpu: 9.047560E+03 | global_batch_size: 64 | lm_loss: 1.080597E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 580007.438 [ip-26-0-154-121:0]:06/21/2023 17:51:38 [INFO|DP=0|PP=0|TP=0]: iteration: 2556 / 150000 | consumed_samples: 163584 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787793E+05 | tokens_per_sec_per_gpu: 9.043426E+03 | global_batch_size: 64 | lm_loss: 1.080595E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 696670.750 [ip-26-0-154-121:0]:06/21/2023 17:51:39 [INFO|DP=0|PP=0|TP=0]: iteration: 2557 / 150000 | consumed_samples: 163648 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789033E+05 | tokens_per_sec_per_gpu: 9.045364E+03 | global_batch_size: 64 | lm_loss: 1.080602E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 716200.562 [ip-26-0-154-121:0]:06/21/2023 17:51:40 [INFO|DP=0|PP=0|TP=0]: iteration: 2558 / 150000 | consumed_samples: 163712 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787045E+05 | tokens_per_sec_per_gpu: 9.042258E+03 | global_batch_size: 64 | lm_loss: 1.080610E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 601449.562 [ip-26-0-154-121:0]:06/21/2023 17:51:41 [INFO|DP=0|PP=0|TP=0]: iteration: 2559 / 150000 | consumed_samples: 163776 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789923E+05 | tokens_per_sec_per_gpu: 9.046755E+03 | global_batch_size: 64 | lm_loss: 1.080554E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 605295.000 [ip-26-0-154-121:0]:06/21/2023 17:51:41 [INFO|DP=0|PP=0|TP=0]: iteration: 2560 / 150000 | consumed_samples: 163840 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786351E+05 | tokens_per_sec_per_gpu: 9.041173E+03 | global_batch_size: 64 | lm_loss: 1.080589E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 620105.125 [ip-26-0-154-121:0]:06/21/2023 17:51:42 [INFO|DP=0|PP=0|TP=0]: iteration: 2561 / 150000 | consumed_samples: 163904 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790568E+05 | tokens_per_sec_per_gpu: 9.047763E+03 | global_batch_size: 64 | lm_loss: 1.080586E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 590918.375 [ip-26-0-154-121:0]:06/21/2023 17:51:43 [INFO|DP=0|PP=0|TP=0]: iteration: 2562 / 150000 | consumed_samples: 163968 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778685E+05 | tokens_per_sec_per_gpu: 9.029196E+03 | global_batch_size: 64 | lm_loss: 1.080587E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 584491.500 [ip-26-0-154-121:0]:06/21/2023 17:51:44 [INFO|DP=0|PP=0|TP=0]: iteration: 2563 / 150000 | consumed_samples: 164032 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784276E+05 | tokens_per_sec_per_gpu: 9.037931E+03 | global_batch_size: 64 | lm_loss: 1.080577E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 581835.812 [ip-26-0-154-121:0]:06/21/2023 17:51:45 [INFO|DP=0|PP=0|TP=0]: iteration: 2564 / 150000 | consumed_samples: 164096 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779395E+05 | tokens_per_sec_per_gpu: 9.030304E+03 | global_batch_size: 64 | lm_loss: 1.080576E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 571206.250 [ip-26-0-154-121:0]:06/21/2023 17:51:46 [INFO|DP=0|PP=0|TP=0]: iteration: 2565 / 150000 | consumed_samples: 164160 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789807E+05 | tokens_per_sec_per_gpu: 9.046574E+03 | global_batch_size: 64 | lm_loss: 1.080578E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 562826.250 [ip-26-0-154-121:0]:06/21/2023 17:51:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2566 / 150000 | consumed_samples: 164224 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785605E+05 | tokens_per_sec_per_gpu: 9.040007E+03 | global_batch_size: 64 | lm_loss: 1.080564E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 553775.188 [ip-26-0-154-121:0]:06/21/2023 17:51:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2567 / 150000 | consumed_samples: 164288 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788541E+05 | tokens_per_sec_per_gpu: 9.044595E+03 | global_batch_size: 64 | lm_loss: 1.080568E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 551361.750 [ip-26-0-154-121:0]:06/21/2023 17:51:49 [INFO|DP=0|PP=0|TP=0]: iteration: 2568 / 150000 | consumed_samples: 164352 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786124E+05 | tokens_per_sec_per_gpu: 9.040818E+03 | global_batch_size: 64 | lm_loss: 1.080574E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 550541.562 [ip-26-0-154-121:0]:06/21/2023 17:51:50 [INFO|DP=0|PP=0|TP=0]: iteration: 2569 / 150000 | consumed_samples: 164416 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788454E+05 | tokens_per_sec_per_gpu: 9.044459E+03 | global_batch_size: 64 | lm_loss: 1.080569E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 551279.875 [ip-26-0-154-121:0]:06/21/2023 17:51:50 [INFO|DP=0|PP=0|TP=0]: iteration: 2570 / 150000 | consumed_samples: 164480 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786474E+05 | tokens_per_sec_per_gpu: 9.041365E+03 | global_batch_size: 64 | lm_loss: 1.080574E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 553984.375 [ip-26-0-154-121:0]:06/21/2023 17:51:51 [INFO|DP=0|PP=0|TP=0]: iteration: 2571 / 150000 | consumed_samples: 164544 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784256E+05 | tokens_per_sec_per_gpu: 9.037900E+03 | global_batch_size: 64 | lm_loss: 1.080404E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 554381.375 [ip-26-0-154-121:0]:06/21/2023 17:51:52 [INFO|DP=0|PP=0|TP=0]: iteration: 2572 / 150000 | consumed_samples: 164608 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783613E+05 | tokens_per_sec_per_gpu: 9.036895E+03 | global_batch_size: 64 | lm_loss: 1.080560E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 550982.250 [ip-26-0-154-121:0]:06/21/2023 17:51:53 [INFO|DP=0|PP=0|TP=0]: iteration: 2573 / 150000 | consumed_samples: 164672 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785302E+05 | tokens_per_sec_per_gpu: 9.039534E+03 | global_batch_size: 64 | lm_loss: 1.080560E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 544621.562 [ip-26-0-154-121:0]:06/21/2023 17:51:54 [INFO|DP=0|PP=0|TP=0]: iteration: 2574 / 150000 | consumed_samples: 164736 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782760E+05 | tokens_per_sec_per_gpu: 9.035562E+03 | global_batch_size: 64 | lm_loss: 1.080553E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 538728.875 [ip-26-0-154-121:0]:06/21/2023 17:51:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2575 / 150000 | consumed_samples: 164800 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776911E+05 | tokens_per_sec_per_gpu: 9.026423E+03 | global_batch_size: 64 | lm_loss: 1.080555E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 533489.688 [ip-26-0-154-121:0]:06/21/2023 17:51:56 [INFO|DP=0|PP=0|TP=0]: iteration: 2576 / 150000 | consumed_samples: 164864 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779846E+05 | tokens_per_sec_per_gpu: 9.031009E+03 | global_batch_size: 64 | lm_loss: 1.080560E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 528287.312 [ip-26-0-154-121:0]:06/21/2023 17:51:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2577 / 150000 | consumed_samples: 164928 | elapsed_time_per_iteration_ms: 1235.6 | tokens_per_sec: 4.243136E+05 | tokens_per_sec_per_gpu: 6.629899E+03 | global_batch_size: 64 | lm_loss: 1.080548E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 76.58 | hardware_tflops_per_gpu: 76.58 | grad_norm: 525484.875 [ip-26-0-154-121:0]:06/21/2023 17:51:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2578 / 150000 | consumed_samples: 164992 | elapsed_time_per_iteration_ms: 1234.2 | tokens_per_sec: 4.247894E+05 | tokens_per_sec_per_gpu: 6.637334E+03 | global_batch_size: 64 | lm_loss: 1.080542E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 76.67 | hardware_tflops_per_gpu: 76.67 | grad_norm: 523785.906 [ip-26-0-154-121:0]:06/21/2023 17:51:59 [INFO|DP=0|PP=0|TP=0]: iteration: 2579 / 150000 | consumed_samples: 165056 | elapsed_time_per_iteration_ms: 980.9 | tokens_per_sec: 5.345109E+05 | tokens_per_sec_per_gpu: 8.351733E+03 | global_batch_size: 64 | lm_loss: 1.080553E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 96.47 | hardware_tflops_per_gpu: 96.47 | grad_norm: 523373.406 [ip-26-0-154-121:0]:06/21/2023 17:52:00 [INFO|DP=0|PP=0|TP=0]: iteration: 2580 / 150000 | consumed_samples: 165120 | elapsed_time_per_iteration_ms: 1031.3 | tokens_per_sec: 5.083877E+05 | tokens_per_sec_per_gpu: 7.943557E+03 | global_batch_size: 64 | lm_loss: 1.080545E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 91.76 | hardware_tflops_per_gpu: 91.76 | grad_norm: 523603.125 [ip-26-0-154-121:0]:06/21/2023 17:52:01 [INFO|DP=0|PP=0|TP=0]: iteration: 2581 / 150000 | consumed_samples: 165184 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789425E+05 | tokens_per_sec_per_gpu: 9.045976E+03 | global_batch_size: 64 | lm_loss: 1.080552E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 522451.719 [ip-26-0-154-121:0]:06/21/2023 17:52:02 [INFO|DP=0|PP=0|TP=0]: iteration: 2582 / 150000 | consumed_samples: 165248 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784488E+05 | tokens_per_sec_per_gpu: 9.038262E+03 | global_batch_size: 64 | lm_loss: 1.080540E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 519615.219 [ip-26-0-154-121:0]:06/21/2023 17:52:03 [INFO|DP=0|PP=0|TP=0]: iteration: 2583 / 150000 | consumed_samples: 165312 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786396E+05 | tokens_per_sec_per_gpu: 9.041244E+03 | global_batch_size: 64 | lm_loss: 1.080536E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 516134.938 [ip-26-0-154-121:0]:06/21/2023 17:52:04 [INFO|DP=0|PP=0|TP=0]: iteration: 2584 / 150000 | consumed_samples: 165376 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788996E+05 | tokens_per_sec_per_gpu: 9.045307E+03 | global_batch_size: 64 | lm_loss: 1.080525E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 512926.312 [ip-26-0-154-121:0]:06/21/2023 17:52:05 [INFO|DP=0|PP=0|TP=0]: iteration: 2585 / 150000 | consumed_samples: 165440 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787685E+05 | tokens_per_sec_per_gpu: 9.043257E+03 | global_batch_size: 64 | lm_loss: 1.080523E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 510697.531 [ip-26-0-154-121:0]:06/21/2023 17:52:06 [INFO|DP=0|PP=0|TP=0]: iteration: 2586 / 150000 | consumed_samples: 165504 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785340E+05 | tokens_per_sec_per_gpu: 9.039593E+03 | global_batch_size: 64 | lm_loss: 1.080525E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 508113.031 [ip-26-0-154-121:0]:06/21/2023 17:52:07 [INFO|DP=0|PP=0|TP=0]: iteration: 2587 / 150000 | consumed_samples: 165568 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786626E+05 | tokens_per_sec_per_gpu: 9.041603E+03 | global_batch_size: 64 | lm_loss: 1.080530E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 504989.688 [ip-26-0-154-121:0]:06/21/2023 17:52:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2588 / 150000 | consumed_samples: 165632 | elapsed_time_per_iteration_ms: 932.8 | tokens_per_sec: 5.620386E+05 | tokens_per_sec_per_gpu: 8.781853E+03 | global_batch_size: 64 | lm_loss: 1.080523E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 101.44 | hardware_tflops_per_gpu: 101.44 | grad_norm: 501926.531 [ip-26-0-154-121:0]:06/21/2023 17:52:09 [INFO|DP=0|PP=0|TP=0]: iteration: 2589 / 150000 | consumed_samples: 165696 | elapsed_time_per_iteration_ms: 922.4 | tokens_per_sec: 5.684067E+05 | tokens_per_sec_per_gpu: 8.881354E+03 | global_batch_size: 64 | lm_loss: 1.080509E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 102.59 | hardware_tflops_per_gpu: 102.59 | grad_norm: 500606.781 [ip-26-0-154-121:0]:06/21/2023 17:52:10 [INFO|DP=0|PP=0|TP=0]: iteration: 2590 / 150000 | consumed_samples: 165760 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782787E+05 | tokens_per_sec_per_gpu: 9.035604E+03 | global_batch_size: 64 | lm_loss: 1.080524E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 499966.969 [ip-26-0-154-121:0]:06/21/2023 17:52:11 [INFO|DP=0|PP=0|TP=0]: iteration: 2591 / 150000 | consumed_samples: 165824 | elapsed_time_per_iteration_ms: 927.4 | tokens_per_sec: 5.653592E+05 | tokens_per_sec_per_gpu: 8.833737E+03 | global_batch_size: 64 | lm_loss: 1.080518E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 102.04 | hardware_tflops_per_gpu: 102.04 | grad_norm: 497094.812 [ip-26-0-154-121:0]:06/21/2023 17:52:11 [INFO|DP=0|PP=0|TP=0]: iteration: 2592 / 150000 | consumed_samples: 165888 | elapsed_time_per_iteration_ms: 907.9 | tokens_per_sec: 5.774861E+05 | tokens_per_sec_per_gpu: 9.023221E+03 | global_batch_size: 64 | lm_loss: 1.080526E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.23 | hardware_tflops_per_gpu: 104.23 | grad_norm: 495311.250 [ip-26-0-154-121:0]:06/21/2023 17:52:12 [INFO|DP=0|PP=0|TP=0]: iteration: 2593 / 150000 | consumed_samples: 165952 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787007E+05 | tokens_per_sec_per_gpu: 9.042198E+03 | global_batch_size: 64 | lm_loss: 1.080524E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 493587.656 [ip-26-0-154-121:0]:06/21/2023 17:52:13 [INFO|DP=0|PP=0|TP=0]: iteration: 2594 / 150000 | consumed_samples: 166016 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788739E+05 | tokens_per_sec_per_gpu: 9.044905E+03 | global_batch_size: 64 | lm_loss: 1.080514E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 490715.938 [ip-26-0-154-121:0]:06/21/2023 17:52:14 [INFO|DP=0|PP=0|TP=0]: iteration: 2595 / 150000 | consumed_samples: 166080 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787921E+05 | tokens_per_sec_per_gpu: 9.043626E+03 | global_batch_size: 64 | lm_loss: 1.080509E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 487502.531 [ip-26-0-154-121:0]:06/21/2023 17:52:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2596 / 150000 | consumed_samples: 166144 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788817E+05 | tokens_per_sec_per_gpu: 9.045026E+03 | global_batch_size: 64 | lm_loss: 1.080506E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 485402.812 [ip-26-0-154-121:0]:06/21/2023 17:52:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2597 / 150000 | consumed_samples: 166208 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786699E+05 | tokens_per_sec_per_gpu: 9.041718E+03 | global_batch_size: 64 | lm_loss: 1.080503E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 483485.625 [ip-26-0-154-121:0]:06/21/2023 17:52:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2598 / 150000 | consumed_samples: 166272 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788710E+05 | tokens_per_sec_per_gpu: 9.044859E+03 | global_batch_size: 64 | lm_loss: 1.080494E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 482725.312 [ip-26-0-154-121:0]:06/21/2023 17:52:18 [INFO|DP=0|PP=0|TP=0]: iteration: 2599 / 150000 | consumed_samples: 166336 | elapsed_time_per_iteration_ms: 921.0 | tokens_per_sec: 5.692482E+05 | tokens_per_sec_per_gpu: 8.894502E+03 | global_batch_size: 64 | lm_loss: 1.080505E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 102.74 | hardware_tflops_per_gpu: 102.74 | grad_norm: 482894.500 [ip-26-0-154-121:0]:06/21/2023 17:52:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2600 / 150000 | consumed_samples: 166400 | elapsed_time_per_iteration_ms: 1053.2 | tokens_per_sec: 4.978279E+05 | tokens_per_sec_per_gpu: 7.778561E+03 | global_batch_size: 64 | lm_loss: 1.080512E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 89.85 | hardware_tflops_per_gpu: 89.85 | grad_norm: 481925.656 [ip-26-0-154-121:0]:06/21/2023 17:52:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2601 / 150000 | consumed_samples: 166464 | elapsed_time_per_iteration_ms: 1046.5 | tokens_per_sec: 5.009900E+05 | tokens_per_sec_per_gpu: 7.827968E+03 | global_batch_size: 64 | lm_loss: 1.080498E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 90.42 | hardware_tflops_per_gpu: 90.42 | grad_norm: 478885.281 [ip-26-0-154-121:0]:06/21/2023 17:52:21 [INFO|DP=0|PP=0|TP=0]: iteration: 2602 / 150000 | consumed_samples: 166528 | elapsed_time_per_iteration_ms: 930.1 | tokens_per_sec: 5.636769E+05 | tokens_per_sec_per_gpu: 8.807452E+03 | global_batch_size: 64 | lm_loss: 1.080503E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 101.74 | hardware_tflops_per_gpu: 101.74 | grad_norm: 475898.938 [ip-26-0-154-121:0]:06/21/2023 17:52:22 [INFO|DP=0|PP=0|TP=0]: iteration: 2603 / 150000 | consumed_samples: 166592 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787420E+05 | tokens_per_sec_per_gpu: 9.042843E+03 | global_batch_size: 64 | lm_loss: 1.080496E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 473249.750 [ip-26-0-154-121:0]:06/21/2023 17:52:23 [INFO|DP=0|PP=0|TP=0]: iteration: 2604 / 150000 | consumed_samples: 166656 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788369E+05 | tokens_per_sec_per_gpu: 9.044326E+03 | global_batch_size: 64 | lm_loss: 1.080498E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 470701.625 [ip-26-0-154-121:0]:06/21/2023 17:52:24 [INFO|DP=0|PP=0|TP=0]: iteration: 2605 / 150000 | consumed_samples: 166720 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787238E+05 | tokens_per_sec_per_gpu: 9.042560E+03 | global_batch_size: 64 | lm_loss: 1.080500E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 469582.375 [ip-26-0-154-121:0]:06/21/2023 17:52:24 [INFO|DP=0|PP=0|TP=0]: iteration: 2606 / 150000 | consumed_samples: 166784 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785329E+05 | tokens_per_sec_per_gpu: 9.039577E+03 | global_batch_size: 64 | lm_loss: 1.080492E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 468600.219 [ip-26-0-154-121:0]:06/21/2023 17:52:25 [INFO|DP=0|PP=0|TP=0]: iteration: 2607 / 150000 | consumed_samples: 166848 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782201E+05 | tokens_per_sec_per_gpu: 9.034690E+03 | global_batch_size: 64 | lm_loss: 1.080487E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 467395.000 [ip-26-0-154-121:0]:06/21/2023 17:52:26 [INFO|DP=0|PP=0|TP=0]: iteration: 2608 / 150000 | consumed_samples: 166912 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785950E+05 | tokens_per_sec_per_gpu: 9.040547E+03 | global_batch_size: 64 | lm_loss: 1.080468E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 465302.531 [ip-26-0-154-121:0]:06/21/2023 17:52:27 [INFO|DP=0|PP=0|TP=0]: iteration: 2609 / 150000 | consumed_samples: 166976 | elapsed_time_per_iteration_ms: 914.8 | tokens_per_sec: 5.731339E+05 | tokens_per_sec_per_gpu: 8.955218E+03 | global_batch_size: 64 | lm_loss: 1.080486E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 103.44 | hardware_tflops_per_gpu: 103.44 | grad_norm: 462624.781 [ip-26-0-154-121:0]:06/21/2023 17:52:28 [INFO|DP=0|PP=0|TP=0]: iteration: 2610 / 150000 | consumed_samples: 167040 | elapsed_time_per_iteration_ms: 1135.2 | tokens_per_sec: 4.618501E+05 | tokens_per_sec_per_gpu: 7.216409E+03 | global_batch_size: 64 | lm_loss: 1.080447E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 83.36 | hardware_tflops_per_gpu: 83.36 | grad_norm: 461157.938 [ip-26-0-154-121:0]:06/21/2023 17:52:29 [INFO|DP=0|PP=0|TP=0]: iteration: 2611 / 150000 | consumed_samples: 167104 | elapsed_time_per_iteration_ms: 921.1 | tokens_per_sec: 5.692090E+05 | tokens_per_sec_per_gpu: 8.893890E+03 | global_batch_size: 64 | lm_loss: 1.080478E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 102.73 | hardware_tflops_per_gpu: 102.73 | grad_norm: 458891.250 [ip-26-0-154-121:0]:06/21/2023 17:52:30 [INFO|DP=0|PP=0|TP=0]: iteration: 2612 / 150000 | consumed_samples: 167168 | elapsed_time_per_iteration_ms: 913.1 | tokens_per_sec: 5.741917E+05 | tokens_per_sec_per_gpu: 8.971745E+03 | global_batch_size: 64 | lm_loss: 1.080472E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 103.63 | hardware_tflops_per_gpu: 103.63 | grad_norm: 456713.500 [ip-26-0-154-121:0]:06/21/2023 17:52:31 [INFO|DP=0|PP=0|TP=0]: iteration: 2613 / 150000 | consumed_samples: 167232 | elapsed_time_per_iteration_ms: 926.1 | tokens_per_sec: 5.661383E+05 | tokens_per_sec_per_gpu: 8.845911E+03 | global_batch_size: 64 | lm_loss: 1.080464E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 102.18 | hardware_tflops_per_gpu: 102.18 | grad_norm: 453249.594 [ip-26-0-154-121:0]:06/21/2023 17:52:32 [INFO|DP=0|PP=0|TP=0]: iteration: 2614 / 150000 | consumed_samples: 167296 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781216E+05 | tokens_per_sec_per_gpu: 9.033151E+03 | global_batch_size: 64 | lm_loss: 1.080471E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 449864.156 [ip-26-0-154-121:0]:06/21/2023 17:52:33 [INFO|DP=0|PP=0|TP=0]: iteration: 2615 / 150000 | consumed_samples: 167360 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784608E+05 | tokens_per_sec_per_gpu: 9.038450E+03 | global_batch_size: 64 | lm_loss: 1.080470E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 446765.500 [ip-26-0-154-121:0]:06/21/2023 17:52:34 [INFO|DP=0|PP=0|TP=0]: iteration: 2616 / 150000 | consumed_samples: 167424 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784588E+05 | tokens_per_sec_per_gpu: 9.038419E+03 | global_batch_size: 64 | lm_loss: 1.080474E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 444111.594 [ip-26-0-154-121:0]:06/21/2023 17:52:35 [INFO|DP=0|PP=0|TP=0]: iteration: 2617 / 150000 | consumed_samples: 167488 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.778919E+05 | tokens_per_sec_per_gpu: 9.029561E+03 | global_batch_size: 64 | lm_loss: 1.080478E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 441774.406 [ip-26-0-154-121:0]:06/21/2023 17:52:36 [INFO|DP=0|PP=0|TP=0]: iteration: 2618 / 150000 | consumed_samples: 167552 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782097E+05 | tokens_per_sec_per_gpu: 9.034526E+03 | global_batch_size: 64 | lm_loss: 1.080469E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 438916.656 [ip-26-0-154-121:0]:06/21/2023 17:52:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2619 / 150000 | consumed_samples: 167616 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780947E+05 | tokens_per_sec_per_gpu: 9.032730E+03 | global_batch_size: 64 | lm_loss: 1.080470E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 436979.125 [ip-26-0-154-121:0]:06/21/2023 17:52:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2620 / 150000 | consumed_samples: 167680 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779357E+05 | tokens_per_sec_per_gpu: 9.030245E+03 | global_batch_size: 64 | lm_loss: 1.080469E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 434030.875 [ip-26-0-154-121:0]:06/21/2023 17:52:38 [INFO|DP=0|PP=0|TP=0]: iteration: 2621 / 150000 | consumed_samples: 167744 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781885E+05 | tokens_per_sec_per_gpu: 9.034196E+03 | global_batch_size: 64 | lm_loss: 1.080462E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 430932.875 [ip-26-0-154-121:0]:06/21/2023 17:52:39 [INFO|DP=0|PP=0|TP=0]: iteration: 2622 / 150000 | consumed_samples: 167808 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778244E+05 | tokens_per_sec_per_gpu: 9.028506E+03 | global_batch_size: 64 | lm_loss: 1.080458E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 428792.219 [ip-26-0-154-121:0]:06/21/2023 17:52:40 [INFO|DP=0|PP=0|TP=0]: iteration: 2623 / 150000 | consumed_samples: 167872 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778186E+05 | tokens_per_sec_per_gpu: 9.028415E+03 | global_batch_size: 64 | lm_loss: 1.080470E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 427843.625 [ip-26-0-154-121:0]:06/21/2023 17:52:41 [INFO|DP=0|PP=0|TP=0]: iteration: 2624 / 150000 | consumed_samples: 167936 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784535E+05 | tokens_per_sec_per_gpu: 9.038335E+03 | global_batch_size: 64 | lm_loss: 1.080455E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 424708.281 [ip-26-0-154-121:0]:06/21/2023 17:52:42 [INFO|DP=0|PP=0|TP=0]: iteration: 2625 / 150000 | consumed_samples: 168000 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777598E+05 | tokens_per_sec_per_gpu: 9.027497E+03 | global_batch_size: 64 | lm_loss: 1.080447E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 421018.469 [ip-26-0-154-121:0]:06/21/2023 17:52:43 [INFO|DP=0|PP=0|TP=0]: iteration: 2626 / 150000 | consumed_samples: 168064 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784281E+05 | tokens_per_sec_per_gpu: 9.037938E+03 | global_batch_size: 64 | lm_loss: 1.080449E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 418586.062 [ip-26-0-154-121:0]:06/21/2023 17:52:44 [INFO|DP=0|PP=0|TP=0]: iteration: 2627 / 150000 | consumed_samples: 168128 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784796E+05 | tokens_per_sec_per_gpu: 9.038744E+03 | global_batch_size: 64 | lm_loss: 1.080404E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 415631.094 [ip-26-0-154-121:0]:06/21/2023 17:52:45 [INFO|DP=0|PP=0|TP=0]: iteration: 2628 / 150000 | consumed_samples: 168192 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785932E+05 | tokens_per_sec_per_gpu: 9.040519E+03 | global_batch_size: 64 | lm_loss: 1.080446E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 413181.375 [ip-26-0-154-121:0]:06/21/2023 17:52:46 [INFO|DP=0|PP=0|TP=0]: iteration: 2629 / 150000 | consumed_samples: 168256 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788190E+05 | tokens_per_sec_per_gpu: 9.044048E+03 | global_batch_size: 64 | lm_loss: 1.080442E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 411066.250 [ip-26-0-154-121:0]:06/21/2023 17:52:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2630 / 150000 | consumed_samples: 168320 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791631E+05 | tokens_per_sec_per_gpu: 9.049424E+03 | global_batch_size: 64 | lm_loss: 1.080438E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.53 | hardware_tflops_per_gpu: 104.53 | grad_norm: 409807.500 [ip-26-0-154-121:0]:06/21/2023 17:52:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2631 / 150000 | consumed_samples: 168384 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787334E+05 | tokens_per_sec_per_gpu: 9.042710E+03 | global_batch_size: 64 | lm_loss: 1.080445E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 408187.031 [ip-26-0-154-121:0]:06/21/2023 17:52:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2632 / 150000 | consumed_samples: 168448 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786299E+05 | tokens_per_sec_per_gpu: 9.041092E+03 | global_batch_size: 64 | lm_loss: 1.080446E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 407407.031 [ip-26-0-154-121:0]:06/21/2023 17:52:49 [INFO|DP=0|PP=0|TP=0]: iteration: 2633 / 150000 | consumed_samples: 168512 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786515E+05 | tokens_per_sec_per_gpu: 9.041430E+03 | global_batch_size: 64 | lm_loss: 1.080441E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 405468.312 [ip-26-0-154-121:0]:06/21/2023 17:52:50 [INFO|DP=0|PP=0|TP=0]: iteration: 2634 / 150000 | consumed_samples: 168576 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785369E+05 | tokens_per_sec_per_gpu: 9.039639E+03 | global_batch_size: 64 | lm_loss: 1.080439E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 403800.344 [ip-26-0-154-121:0]:06/21/2023 17:52:51 [INFO|DP=0|PP=0|TP=0]: iteration: 2635 / 150000 | consumed_samples: 168640 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780610E+05 | tokens_per_sec_per_gpu: 9.032203E+03 | global_batch_size: 64 | lm_loss: 1.080431E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 402945.312 [ip-26-0-154-121:0]:06/21/2023 17:52:52 [INFO|DP=0|PP=0|TP=0]: iteration: 2636 / 150000 | consumed_samples: 168704 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778102E+05 | tokens_per_sec_per_gpu: 9.028285E+03 | global_batch_size: 64 | lm_loss: 1.080436E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 402035.000 [ip-26-0-154-121:0]:06/21/2023 17:52:53 [INFO|DP=0|PP=0|TP=0]: iteration: 2637 / 150000 | consumed_samples: 168768 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784836E+05 | tokens_per_sec_per_gpu: 9.038806E+03 | global_batch_size: 64 | lm_loss: 1.080434E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 399135.438 [ip-26-0-154-121:0]:06/21/2023 17:52:54 [INFO|DP=0|PP=0|TP=0]: iteration: 2638 / 150000 | consumed_samples: 168832 | elapsed_time_per_iteration_ms: 914.4 | tokens_per_sec: 5.733994E+05 | tokens_per_sec_per_gpu: 8.959365E+03 | global_batch_size: 64 | lm_loss: 1.080428E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 103.49 | hardware_tflops_per_gpu: 103.49 | grad_norm: 395957.062 [ip-26-0-154-121:0]:06/21/2023 17:52:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2639 / 150000 | consumed_samples: 168896 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785370E+05 | tokens_per_sec_per_gpu: 9.039641E+03 | global_batch_size: 64 | lm_loss: 1.080405E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 393595.312 [ip-26-0-154-121:0]:06/21/2023 17:52:56 [INFO|DP=0|PP=0|TP=0]: iteration: 2640 / 150000 | consumed_samples: 168960 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785477E+05 | tokens_per_sec_per_gpu: 9.039807E+03 | global_batch_size: 64 | lm_loss: 1.080406E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 394090.750 [ip-26-0-154-121:0]:06/21/2023 17:52:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2641 / 150000 | consumed_samples: 169024 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786448E+05 | tokens_per_sec_per_gpu: 9.041325E+03 | global_batch_size: 64 | lm_loss: 1.080412E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 388702.719 [ip-26-0-154-121:0]:06/21/2023 17:52:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2642 / 150000 | consumed_samples: 169088 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778101E+05 | tokens_per_sec_per_gpu: 9.028283E+03 | global_batch_size: 64 | lm_loss: 1.080428E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 394381.594 [ip-26-0-154-121:0]:06/21/2023 17:52:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2643 / 150000 | consumed_samples: 169152 | elapsed_time_per_iteration_ms: 992.4 | tokens_per_sec: 5.282982E+05 | tokens_per_sec_per_gpu: 8.254659E+03 | global_batch_size: 64 | lm_loss: 1.080416E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 95.35 | hardware_tflops_per_gpu: 95.35 | grad_norm: 390142.188 [ip-26-0-154-121:0]:06/21/2023 17:52:59 [INFO|DP=0|PP=0|TP=0]: iteration: 2644 / 150000 | consumed_samples: 169216 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786405E+05 | tokens_per_sec_per_gpu: 9.041258E+03 | global_batch_size: 64 | lm_loss: 1.080424E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 397615.938 [ip-26-0-154-121:0]:06/21/2023 17:53:00 [INFO|DP=0|PP=0|TP=0]: iteration: 2645 / 150000 | consumed_samples: 169280 | elapsed_time_per_iteration_ms: 923.3 | tokens_per_sec: 5.678620E+05 | tokens_per_sec_per_gpu: 8.872843E+03 | global_batch_size: 64 | lm_loss: 1.080392E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 102.49 | hardware_tflops_per_gpu: 102.49 | grad_norm: 396104.062 [ip-26-0-154-121:0]:06/21/2023 17:53:01 [INFO|DP=0|PP=0|TP=0]: iteration: 2646 / 150000 | consumed_samples: 169344 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776744E+05 | tokens_per_sec_per_gpu: 9.026162E+03 | global_batch_size: 64 | lm_loss: 1.080411E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 408035.281 [ip-26-0-154-121:0]:06/21/2023 17:53:02 [INFO|DP=0|PP=0|TP=0]: iteration: 2647 / 150000 | consumed_samples: 169408 | elapsed_time_per_iteration_ms: 913.4 | tokens_per_sec: 5.739912E+05 | tokens_per_sec_per_gpu: 8.968612E+03 | global_batch_size: 64 | lm_loss: 1.080408E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 103.60 | hardware_tflops_per_gpu: 103.60 | grad_norm: 404965.688 [ip-26-0-154-121:0]:06/21/2023 17:53:03 [INFO|DP=0|PP=0|TP=0]: iteration: 2648 / 150000 | consumed_samples: 169472 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783760E+05 | tokens_per_sec_per_gpu: 9.037125E+03 | global_batch_size: 64 | lm_loss: 1.080425E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 410921.625 [ip-26-0-154-121:0]:06/21/2023 17:53:04 [INFO|DP=0|PP=0|TP=0]: iteration: 2649 / 150000 | consumed_samples: 169536 | elapsed_time_per_iteration_ms: 947.0 | tokens_per_sec: 5.536169E+05 | tokens_per_sec_per_gpu: 8.650264E+03 | global_batch_size: 64 | lm_loss: 1.080417E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 99.92 | hardware_tflops_per_gpu: 99.92 | grad_norm: 408595.656 [ip-26-0-154-121:0]:06/21/2023 17:53:05 [INFO|DP=0|PP=0|TP=0]: iteration: 2650 / 150000 | consumed_samples: 169600 | elapsed_time_per_iteration_ms: 927.0 | tokens_per_sec: 5.655694E+05 | tokens_per_sec_per_gpu: 8.837022E+03 | global_batch_size: 64 | lm_loss: 1.080412E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 102.08 | hardware_tflops_per_gpu: 102.08 | grad_norm: 404168.250 [ip-26-0-154-121:0]:06/21/2023 17:53:06 [INFO|DP=0|PP=0|TP=0]: iteration: 2651 / 150000 | consumed_samples: 169664 | elapsed_time_per_iteration_ms: 938.0 | tokens_per_sec: 5.589613E+05 | tokens_per_sec_per_gpu: 8.733771E+03 | global_batch_size: 64 | lm_loss: 1.080404E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 100.88 | hardware_tflops_per_gpu: 100.88 | grad_norm: 418936.125 [ip-26-0-154-121:0]:06/21/2023 17:53:07 [INFO|DP=0|PP=0|TP=0]: iteration: 2652 / 150000 | consumed_samples: 169728 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782618E+05 | tokens_per_sec_per_gpu: 9.035341E+03 | global_batch_size: 64 | lm_loss: 1.080414E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 412555.938 [ip-26-0-154-121:0]:06/21/2023 17:53:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2653 / 150000 | consumed_samples: 169792 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784740E+05 | tokens_per_sec_per_gpu: 9.038656E+03 | global_batch_size: 64 | lm_loss: 1.080413E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 415083.062 [ip-26-0-154-121:0]:06/21/2023 17:53:09 [INFO|DP=0|PP=0|TP=0]: iteration: 2654 / 150000 | consumed_samples: 169856 | elapsed_time_per_iteration_ms: 950.2 | tokens_per_sec: 5.517670E+05 | tokens_per_sec_per_gpu: 8.621359E+03 | global_batch_size: 64 | lm_loss: 1.080389E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 99.59 | hardware_tflops_per_gpu: 99.59 | grad_norm: 437959.375 [ip-26-0-154-121:0]:06/21/2023 17:53:10 [INFO|DP=0|PP=0|TP=0]: iteration: 2655 / 150000 | consumed_samples: 169920 | elapsed_time_per_iteration_ms: 938.8 | tokens_per_sec: 5.584844E+05 | tokens_per_sec_per_gpu: 8.726318E+03 | global_batch_size: 64 | lm_loss: 1.080401E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 100.80 | hardware_tflops_per_gpu: 100.80 | grad_norm: 437715.188 [ip-26-0-154-121:0]:06/21/2023 17:53:10 [INFO|DP=0|PP=0|TP=0]: iteration: 2656 / 150000 | consumed_samples: 169984 | elapsed_time_per_iteration_ms: 928.8 | tokens_per_sec: 5.644936E+05 | tokens_per_sec_per_gpu: 8.820213E+03 | global_batch_size: 64 | lm_loss: 1.080403E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 101.88 | hardware_tflops_per_gpu: 101.88 | grad_norm: 447484.406 [ip-26-0-154-121:0]:06/21/2023 17:53:11 [INFO|DP=0|PP=0|TP=0]: iteration: 2657 / 150000 | consumed_samples: 170048 | elapsed_time_per_iteration_ms: 934.6 | tokens_per_sec: 5.609946E+05 | tokens_per_sec_per_gpu: 8.765541E+03 | global_batch_size: 64 | lm_loss: 1.080357E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 101.25 | hardware_tflops_per_gpu: 101.25 | grad_norm: 515096.062 [ip-26-0-154-121:0]:06/21/2023 17:53:12 [INFO|DP=0|PP=0|TP=0]: iteration: 2658 / 150000 | consumed_samples: 170112 | elapsed_time_per_iteration_ms: 923.8 | tokens_per_sec: 5.675197E+05 | tokens_per_sec_per_gpu: 8.867496E+03 | global_batch_size: 64 | lm_loss: 1.080391E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 102.43 | hardware_tflops_per_gpu: 102.43 | grad_norm: 542861.375 [ip-26-0-154-121:0]:06/21/2023 17:53:13 [INFO|DP=0|PP=0|TP=0]: iteration: 2659 / 150000 | consumed_samples: 170176 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782895E+05 | tokens_per_sec_per_gpu: 9.035773E+03 | global_batch_size: 64 | lm_loss: 1.080401E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 615195.688 [ip-26-0-154-121:0]:06/21/2023 17:53:14 [INFO|DP=0|PP=0|TP=0]: iteration: 2660 / 150000 | consumed_samples: 170240 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785786E+05 | tokens_per_sec_per_gpu: 9.040290E+03 | global_batch_size: 64 | lm_loss: 1.080404E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 804500.875 [ip-26-0-154-121:0]:06/21/2023 17:53:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2661 / 150000 | consumed_samples: 170304 | elapsed_time_per_iteration_ms: 925.7 | tokens_per_sec: 5.663707E+05 | tokens_per_sec_per_gpu: 8.849543E+03 | global_batch_size: 64 | lm_loss: 1.080402E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 102.22 | hardware_tflops_per_gpu: 102.22 | grad_norm: 1026666.875 [ip-26-0-154-121:0]:06/21/2023 17:53:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2662 / 150000 | consumed_samples: 170368 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785116E+05 | tokens_per_sec_per_gpu: 9.039244E+03 | global_batch_size: 64 | lm_loss: 1.080394E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1583642.875 [ip-26-0-154-121:0]:06/21/2023 17:53:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2663 / 150000 | consumed_samples: 170432 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785883E+05 | tokens_per_sec_per_gpu: 9.040442E+03 | global_batch_size: 64 | lm_loss: 1.080380E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 5444059.500 [ip-26-0-154-121:0]:06/21/2023 17:53:18 [INFO|DP=0|PP=0|TP=0]: iteration: 2664 / 150000 | consumed_samples: 170496 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788626E+05 | tokens_per_sec_per_gpu: 9.044728E+03 | global_batch_size: 64 | lm_loss: 1.080383E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 14070412.000 [ip-26-0-154-121:0]:06/21/2023 17:53:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2665 / 150000 | consumed_samples: 170560 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782080E+05 | tokens_per_sec_per_gpu: 9.034500E+03 | global_batch_size: 64 | lm_loss: 1.080377E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 70806224.000 [ip-26-0-154-121:0]:06/21/2023 17:53:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2666 / 150000 | consumed_samples: 170624 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786972E+05 | tokens_per_sec_per_gpu: 9.042143E+03 | global_batch_size: 64 | lm_loss: 1.080384E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 32777460.000 [ip-26-0-154-121:0]:06/21/2023 17:53:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2667 / 150000 | consumed_samples: 170688 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786192E+05 | tokens_per_sec_per_gpu: 9.040925E+03 | global_batch_size: 64 | lm_loss: 1.080385E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 6300176.500 [ip-26-0-154-121:0]:06/21/2023 17:53:21 [INFO|DP=0|PP=0|TP=0]: iteration: 2668 / 150000 | consumed_samples: 170752 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.783981E+05 | tokens_per_sec_per_gpu: 9.037470E+03 | global_batch_size: 64 | lm_loss: 1.080379E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 25778866.000 [ip-26-0-154-121:0]:06/21/2023 17:53:22 [INFO|DP=0|PP=0|TP=0]: iteration: 2669 / 150000 | consumed_samples: 170816 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785577E+05 | tokens_per_sec_per_gpu: 9.039964E+03 | global_batch_size: 64 | lm_loss: 1.080383E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 27668472.000 [ip-26-0-154-121:0]:06/21/2023 17:53:23 [INFO|DP=0|PP=0|TP=0]: iteration: 2670 / 150000 | consumed_samples: 170880 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790451E+05 | tokens_per_sec_per_gpu: 9.047579E+03 | global_batch_size: 64 | lm_loss: 1.080387E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 23920944.000 [ip-26-0-154-121:0]:06/21/2023 17:53:24 [INFO|DP=0|PP=0|TP=0]: iteration: 2671 / 150000 | consumed_samples: 170944 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785936E+05 | tokens_per_sec_per_gpu: 9.040526E+03 | global_batch_size: 64 | lm_loss: 1.080378E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 16692593.000 [ip-26-0-154-121:0]:06/21/2023 17:53:25 [INFO|DP=0|PP=0|TP=0]: iteration: 2672 / 150000 | consumed_samples: 171008 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780526E+05 | tokens_per_sec_per_gpu: 9.032073E+03 | global_batch_size: 64 | lm_loss: 1.080361E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 587832.188 [ip-26-0-154-121:0]:06/21/2023 17:53:26 [INFO|DP=0|PP=0|TP=0]: iteration: 2673 / 150000 | consumed_samples: 171072 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778737E+05 | tokens_per_sec_per_gpu: 9.029277E+03 | global_batch_size: 64 | lm_loss: 1.080397E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 7331071.000 [ip-26-0-154-121:0]:06/21/2023 17:53:27 [INFO|DP=0|PP=0|TP=0]: iteration: 2674 / 150000 | consumed_samples: 171136 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784971E+05 | tokens_per_sec_per_gpu: 9.039018E+03 | global_batch_size: 64 | lm_loss: 1.080431E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 5902764.000 [ip-26-0-154-121:0]:06/21/2023 17:53:28 [INFO|DP=0|PP=0|TP=0]: iteration: 2675 / 150000 | consumed_samples: 171200 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785093E+05 | tokens_per_sec_per_gpu: 9.039208E+03 | global_batch_size: 64 | lm_loss: 1.080432E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 2062246.750 [ip-26-0-154-121:0]:06/21/2023 17:53:29 [INFO|DP=0|PP=0|TP=0]: iteration: 2676 / 150000 | consumed_samples: 171264 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785507E+05 | tokens_per_sec_per_gpu: 9.039855E+03 | global_batch_size: 64 | lm_loss: 1.080442E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3366975.000 [ip-26-0-154-121:0]:06/21/2023 17:53:30 [INFO|DP=0|PP=0|TP=0]: iteration: 2677 / 150000 | consumed_samples: 171328 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780952E+05 | tokens_per_sec_per_gpu: 9.032737E+03 | global_batch_size: 64 | lm_loss: 1.080430E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 2112297.250 [ip-26-0-154-121:0]:06/21/2023 17:53:30 [INFO|DP=0|PP=0|TP=0]: iteration: 2678 / 150000 | consumed_samples: 171392 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783327E+05 | tokens_per_sec_per_gpu: 9.036448E+03 | global_batch_size: 64 | lm_loss: 1.080396E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 6909095.000 [ip-26-0-154-121:0]:06/21/2023 17:53:31 [INFO|DP=0|PP=0|TP=0]: iteration: 2679 / 150000 | consumed_samples: 171456 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778148E+05 | tokens_per_sec_per_gpu: 9.028356E+03 | global_batch_size: 64 | lm_loss: 1.080383E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 38393648.000 [ip-26-0-154-121:0]:06/21/2023 17:53:32 [INFO|DP=0|PP=0|TP=0]: iteration: 2680 / 150000 | consumed_samples: 171520 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777201E+05 | tokens_per_sec_per_gpu: 9.026876E+03 | global_batch_size: 64 | lm_loss: 1.080368E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 2094348.250 [ip-26-0-154-121:0]:06/21/2023 17:53:33 [INFO|DP=0|PP=0|TP=0]: iteration: 2681 / 150000 | consumed_samples: 171584 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780715E+05 | tokens_per_sec_per_gpu: 9.032367E+03 | global_batch_size: 64 | lm_loss: 1.080358E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 1389444.500 [ip-26-0-154-121:0]:06/21/2023 17:53:34 [INFO|DP=0|PP=0|TP=0]: iteration: 2682 / 150000 | consumed_samples: 171648 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779384E+05 | tokens_per_sec_per_gpu: 9.030288E+03 | global_batch_size: 64 | lm_loss: 1.080373E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 1415010.625 [ip-26-0-154-121:0]:06/21/2023 17:53:35 [INFO|DP=0|PP=0|TP=0]: iteration: 2683 / 150000 | consumed_samples: 171712 | elapsed_time_per_iteration_ms: 908.8 | tokens_per_sec: 5.769130E+05 | tokens_per_sec_per_gpu: 9.014265E+03 | global_batch_size: 64 | lm_loss: 1.080401E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.12 | hardware_tflops_per_gpu: 104.12 | grad_norm: 68598240.000 [ip-26-0-154-121:0]:06/21/2023 17:53:36 [INFO|DP=0|PP=0|TP=0]: iteration: 2684 / 150000 | consumed_samples: 171776 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786760E+05 | tokens_per_sec_per_gpu: 9.041813E+03 | global_batch_size: 64 | lm_loss: 1.080478E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 6629370.500 [ip-26-0-154-121:0]:06/21/2023 17:53:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2685 / 150000 | consumed_samples: 171840 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781145E+05 | tokens_per_sec_per_gpu: 9.033039E+03 | global_batch_size: 64 | lm_loss: 1.080447E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 4244003.500 [ip-26-0-154-121:0]:06/21/2023 17:53:38 [INFO|DP=0|PP=0|TP=0]: iteration: 2686 / 150000 | consumed_samples: 171904 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780613E+05 | tokens_per_sec_per_gpu: 9.032208E+03 | global_batch_size: 64 | lm_loss: 1.080472E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 4643031.000 [ip-26-0-154-121:0]:06/21/2023 17:53:39 [INFO|DP=0|PP=0|TP=0]: iteration: 2687 / 150000 | consumed_samples: 171968 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779047E+05 | tokens_per_sec_per_gpu: 9.029761E+03 | global_batch_size: 64 | lm_loss: 1.080420E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 9921452.000 [ip-26-0-154-121:0]:06/21/2023 17:53:40 [INFO|DP=0|PP=0|TP=0]: iteration: 2688 / 150000 | consumed_samples: 172032 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781713E+05 | tokens_per_sec_per_gpu: 9.033927E+03 | global_batch_size: 64 | lm_loss: 1.080411E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 17436368.000 [ip-26-0-154-121:0]:06/21/2023 17:53:40 [INFO|DP=0|PP=0|TP=0]: iteration: 2689 / 150000 | consumed_samples: 172096 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781262E+05 | tokens_per_sec_per_gpu: 9.033222E+03 | global_batch_size: 64 | lm_loss: 1.080389E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 16848702.000 [ip-26-0-154-121:0]:06/21/2023 17:53:41 [INFO|DP=0|PP=0|TP=0]: iteration: 2690 / 150000 | consumed_samples: 172160 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784089E+05 | tokens_per_sec_per_gpu: 9.037639E+03 | global_batch_size: 64 | lm_loss: 1.080376E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 2977364.250 [ip-26-0-154-121:0]:06/21/2023 17:53:42 [INFO|DP=0|PP=0|TP=0]: iteration: 2691 / 150000 | consumed_samples: 172224 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782119E+05 | tokens_per_sec_per_gpu: 9.034562E+03 | global_batch_size: 64 | lm_loss: 1.080361E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 3842112.000 [ip-26-0-154-121:0]:06/21/2023 17:53:43 [INFO|DP=0|PP=0|TP=0]: iteration: 2692 / 150000 | consumed_samples: 172288 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785586E+05 | tokens_per_sec_per_gpu: 9.039979E+03 | global_batch_size: 64 | lm_loss: 1.080350E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 2875242.250 [ip-26-0-154-121:0]:06/21/2023 17:53:44 [INFO|DP=0|PP=0|TP=0]: iteration: 2693 / 150000 | consumed_samples: 172352 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787714E+05 | tokens_per_sec_per_gpu: 9.043302E+03 | global_batch_size: 64 | lm_loss: 1.080358E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1727995.125 [ip-26-0-154-121:0]:06/21/2023 17:53:45 [INFO|DP=0|PP=0|TP=0]: iteration: 2694 / 150000 | consumed_samples: 172416 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785901E+05 | tokens_per_sec_per_gpu: 9.040471E+03 | global_batch_size: 64 | lm_loss: 1.080373E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1066957.000 [ip-26-0-154-121:0]:06/21/2023 17:53:46 [INFO|DP=0|PP=0|TP=0]: iteration: 2695 / 150000 | consumed_samples: 172480 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784608E+05 | tokens_per_sec_per_gpu: 9.038450E+03 | global_batch_size: 64 | lm_loss: 1.080387E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 495099.750 [ip-26-0-154-121:0]:06/21/2023 17:53:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2696 / 150000 | consumed_samples: 172544 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782676E+05 | tokens_per_sec_per_gpu: 9.035431E+03 | global_batch_size: 64 | lm_loss: 1.080401E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 1745673.250 [ip-26-0-154-121:0]:06/21/2023 17:53:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2697 / 150000 | consumed_samples: 172608 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783905E+05 | tokens_per_sec_per_gpu: 9.037351E+03 | global_batch_size: 64 | lm_loss: 1.080402E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3171767.000 [ip-26-0-154-121:0]:06/21/2023 17:53:49 [INFO|DP=0|PP=0|TP=0]: iteration: 2698 / 150000 | consumed_samples: 172672 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780185E+05 | tokens_per_sec_per_gpu: 9.031538E+03 | global_batch_size: 64 | lm_loss: 1.080409E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 3953884.000 [ip-26-0-154-121:0]:06/21/2023 17:53:50 [INFO|DP=0|PP=0|TP=0]: iteration: 2699 / 150000 | consumed_samples: 172736 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780960E+05 | tokens_per_sec_per_gpu: 9.032749E+03 | global_batch_size: 64 | lm_loss: 1.080404E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 3998710.250 [ip-26-0-154-121:0]:06/21/2023 17:53:50 [INFO|DP=0|PP=0|TP=0]: iteration: 2700 / 150000 | consumed_samples: 172800 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784445E+05 | tokens_per_sec_per_gpu: 9.038195E+03 | global_batch_size: 64 | lm_loss: 1.080410E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3813077.000 [ip-26-0-154-121:0]:06/21/2023 17:53:51 [INFO|DP=0|PP=0|TP=0]: iteration: 2701 / 150000 | consumed_samples: 172864 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786177E+05 | tokens_per_sec_per_gpu: 9.040902E+03 | global_batch_size: 64 | lm_loss: 1.080365E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3794391.250 [ip-26-0-154-121:0]:06/21/2023 17:53:52 [INFO|DP=0|PP=0|TP=0]: iteration: 2702 / 150000 | consumed_samples: 172928 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785143E+05 | tokens_per_sec_per_gpu: 9.039287E+03 | global_batch_size: 64 | lm_loss: 1.080380E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 3263260.000 [ip-26-0-154-121:0]:06/21/2023 17:53:53 [INFO|DP=0|PP=0|TP=0]: iteration: 2703 / 150000 | consumed_samples: 172992 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786574E+05 | tokens_per_sec_per_gpu: 9.041522E+03 | global_batch_size: 64 | lm_loss: 1.080380E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2979584.000 [ip-26-0-154-121:0]:06/21/2023 17:53:54 [INFO|DP=0|PP=0|TP=0]: iteration: 2704 / 150000 | consumed_samples: 173056 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785200E+05 | tokens_per_sec_per_gpu: 9.039375E+03 | global_batch_size: 64 | lm_loss: 1.080372E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 2043090.625 [ip-26-0-154-121:0]:06/21/2023 17:53:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2705 / 150000 | consumed_samples: 173120 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785778E+05 | tokens_per_sec_per_gpu: 9.040278E+03 | global_batch_size: 64 | lm_loss: 1.080371E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 491951.031 [ip-26-0-154-121:0]:06/21/2023 17:53:56 [INFO|DP=0|PP=0|TP=0]: iteration: 2706 / 150000 | consumed_samples: 173184 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785746E+05 | tokens_per_sec_per_gpu: 9.040228E+03 | global_batch_size: 64 | lm_loss: 1.080357E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1538430.375 [ip-26-0-154-121:0]:06/21/2023 17:53:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2707 / 150000 | consumed_samples: 173248 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787004E+05 | tokens_per_sec_per_gpu: 9.042193E+03 | global_batch_size: 64 | lm_loss: 1.080363E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1687221.625 [ip-26-0-154-121:0]:06/21/2023 17:53:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2708 / 150000 | consumed_samples: 173312 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789009E+05 | tokens_per_sec_per_gpu: 9.045326E+03 | global_batch_size: 64 | lm_loss: 1.080353E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 456155.094 [ip-26-0-154-121:0]:06/21/2023 17:53:59 [INFO|DP=0|PP=0|TP=0]: iteration: 2709 / 150000 | consumed_samples: 173376 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788550E+05 | tokens_per_sec_per_gpu: 9.044609E+03 | global_batch_size: 64 | lm_loss: 1.080354E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 995583.750 [ip-26-0-154-121:0]:06/21/2023 17:54:00 [INFO|DP=0|PP=0|TP=0]: iteration: 2710 / 150000 | consumed_samples: 173440 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787415E+05 | tokens_per_sec_per_gpu: 9.042836E+03 | global_batch_size: 64 | lm_loss: 1.080353E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 418931.969 [ip-26-0-154-121:0]:06/21/2023 17:54:00 [INFO|DP=0|PP=0|TP=0]: iteration: 2711 / 150000 | consumed_samples: 173504 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786584E+05 | tokens_per_sec_per_gpu: 9.041537E+03 | global_batch_size: 64 | lm_loss: 1.080367E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 592653.812 [ip-26-0-154-121:0]:06/21/2023 17:54:01 [INFO|DP=0|PP=0|TP=0]: iteration: 2712 / 150000 | consumed_samples: 173568 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.777910E+05 | tokens_per_sec_per_gpu: 9.027984E+03 | global_batch_size: 64 | lm_loss: 1.080393E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 361820.000 [ip-26-0-154-121:0]:06/21/2023 17:54:02 [INFO|DP=0|PP=0|TP=0]: iteration: 2713 / 150000 | consumed_samples: 173632 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784731E+05 | tokens_per_sec_per_gpu: 9.038642E+03 | global_batch_size: 64 | lm_loss: 1.080391E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 380702.250 [ip-26-0-154-121:0]:06/21/2023 17:54:03 [INFO|DP=0|PP=0|TP=0]: iteration: 2714 / 150000 | consumed_samples: 173696 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784964E+05 | tokens_per_sec_per_gpu: 9.039006E+03 | global_batch_size: 64 | lm_loss: 1.080382E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 470825.750 [ip-26-0-154-121:0]:06/21/2023 17:54:04 [INFO|DP=0|PP=0|TP=0]: iteration: 2715 / 150000 | consumed_samples: 173760 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785900E+05 | tokens_per_sec_per_gpu: 9.040469E+03 | global_batch_size: 64 | lm_loss: 1.080372E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 475813.688 [ip-26-0-154-121:0]:06/21/2023 17:54:05 [INFO|DP=0|PP=0|TP=0]: iteration: 2716 / 150000 | consumed_samples: 173824 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789676E+05 | tokens_per_sec_per_gpu: 9.046369E+03 | global_batch_size: 64 | lm_loss: 1.080359E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 346058.406 [ip-26-0-154-121:0]:06/21/2023 17:54:06 [INFO|DP=0|PP=0|TP=0]: iteration: 2717 / 150000 | consumed_samples: 173888 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784188E+05 | tokens_per_sec_per_gpu: 9.037793E+03 | global_batch_size: 64 | lm_loss: 1.080349E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 341469.812 [ip-26-0-154-121:0]:06/21/2023 17:54:07 [INFO|DP=0|PP=0|TP=0]: iteration: 2718 / 150000 | consumed_samples: 173952 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782568E+05 | tokens_per_sec_per_gpu: 9.035262E+03 | global_batch_size: 64 | lm_loss: 1.080354E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 329427.062 [ip-26-0-154-121:0]:06/21/2023 17:54:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2719 / 150000 | consumed_samples: 174016 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785739E+05 | tokens_per_sec_per_gpu: 9.040216E+03 | global_batch_size: 64 | lm_loss: 1.080355E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 324137.688 [ip-26-0-154-121:0]:06/21/2023 17:54:09 [INFO|DP=0|PP=0|TP=0]: iteration: 2720 / 150000 | consumed_samples: 174080 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784772E+05 | tokens_per_sec_per_gpu: 9.038706E+03 | global_batch_size: 64 | lm_loss: 1.080351E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 323906.688 [ip-26-0-154-121:0]:06/21/2023 17:54:10 [INFO|DP=0|PP=0|TP=0]: iteration: 2721 / 150000 | consumed_samples: 174144 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784594E+05 | tokens_per_sec_per_gpu: 9.038428E+03 | global_batch_size: 64 | lm_loss: 1.080355E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 316093.188 [ip-26-0-154-121:0]:06/21/2023 17:54:10 [INFO|DP=0|PP=0|TP=0]: iteration: 2722 / 150000 | consumed_samples: 174208 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783138E+05 | tokens_per_sec_per_gpu: 9.036153E+03 | global_batch_size: 64 | lm_loss: 1.080357E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 309293.469 [ip-26-0-154-121:0]:06/21/2023 17:54:11 [INFO|DP=0|PP=0|TP=0]: iteration: 2723 / 150000 | consumed_samples: 174272 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780441E+05 | tokens_per_sec_per_gpu: 9.031940E+03 | global_batch_size: 64 | lm_loss: 1.080349E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 305471.844 [ip-26-0-154-121:0]:06/21/2023 17:54:12 [INFO|DP=0|PP=0|TP=0]: iteration: 2724 / 150000 | consumed_samples: 174336 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781054E+05 | tokens_per_sec_per_gpu: 9.032897E+03 | global_batch_size: 64 | lm_loss: 1.080356E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 311168.750 [ip-26-0-154-121:0]:06/21/2023 17:54:13 [INFO|DP=0|PP=0|TP=0]: iteration: 2725 / 150000 | consumed_samples: 174400 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789634E+05 | tokens_per_sec_per_gpu: 9.046302E+03 | global_batch_size: 64 | lm_loss: 1.080361E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 298957.969 [ip-26-0-154-121:0]:06/21/2023 17:54:14 [INFO|DP=0|PP=0|TP=0]: iteration: 2726 / 150000 | consumed_samples: 174464 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786836E+05 | tokens_per_sec_per_gpu: 9.041932E+03 | global_batch_size: 64 | lm_loss: 1.080357E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 289345.594 [ip-26-0-154-121:0]:06/21/2023 17:54:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2727 / 150000 | consumed_samples: 174528 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784906E+05 | tokens_per_sec_per_gpu: 9.038916E+03 | global_batch_size: 64 | lm_loss: 1.080339E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 288012.531 [ip-26-0-154-121:0]:06/21/2023 17:54:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2728 / 150000 | consumed_samples: 174592 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781447E+05 | tokens_per_sec_per_gpu: 9.033512E+03 | global_batch_size: 64 | lm_loss: 1.080352E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 285625.938 [ip-26-0-154-121:0]:06/21/2023 17:54:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2729 / 150000 | consumed_samples: 174656 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782825E+05 | tokens_per_sec_per_gpu: 9.035664E+03 | global_batch_size: 64 | lm_loss: 1.080350E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 280726.188 [ip-26-0-154-121:0]:06/21/2023 17:54:18 [INFO|DP=0|PP=0|TP=0]: iteration: 2730 / 150000 | consumed_samples: 174720 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785761E+05 | tokens_per_sec_per_gpu: 9.040252E+03 | global_batch_size: 64 | lm_loss: 1.080351E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 278652.000 [ip-26-0-154-121:0]:06/21/2023 17:54:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2731 / 150000 | consumed_samples: 174784 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784381E+05 | tokens_per_sec_per_gpu: 9.038095E+03 | global_batch_size: 64 | lm_loss: 1.080347E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 280216.281 [ip-26-0-154-121:0]:06/21/2023 17:54:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2732 / 150000 | consumed_samples: 174848 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780735E+05 | tokens_per_sec_per_gpu: 9.032398E+03 | global_batch_size: 64 | lm_loss: 1.080346E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 282182.500 [ip-26-0-154-121:0]:06/21/2023 17:54:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2733 / 150000 | consumed_samples: 174912 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780782E+05 | tokens_per_sec_per_gpu: 9.032472E+03 | global_batch_size: 64 | lm_loss: 1.080346E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 279800.281 [ip-26-0-154-121:0]:06/21/2023 17:54:21 [INFO|DP=0|PP=0|TP=0]: iteration: 2734 / 150000 | consumed_samples: 174976 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785212E+05 | tokens_per_sec_per_gpu: 9.039394E+03 | global_batch_size: 64 | lm_loss: 1.080343E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 274399.719 [ip-26-0-154-121:0]:06/21/2023 17:54:22 [INFO|DP=0|PP=0|TP=0]: iteration: 2735 / 150000 | consumed_samples: 175040 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784930E+05 | tokens_per_sec_per_gpu: 9.038954E+03 | global_batch_size: 64 | lm_loss: 1.080337E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 269880.250 [ip-26-0-154-121:0]:06/21/2023 17:54:23 [INFO|DP=0|PP=0|TP=0]: iteration: 2736 / 150000 | consumed_samples: 175104 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784040E+05 | tokens_per_sec_per_gpu: 9.037563E+03 | global_batch_size: 64 | lm_loss: 1.080334E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 266871.438 [ip-26-0-154-121:0]:06/21/2023 17:54:24 [INFO|DP=0|PP=0|TP=0]: iteration: 2737 / 150000 | consumed_samples: 175168 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784229E+05 | tokens_per_sec_per_gpu: 9.037858E+03 | global_batch_size: 64 | lm_loss: 1.080334E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 264402.906 [ip-26-0-154-121:0]:06/21/2023 17:54:25 [INFO|DP=0|PP=0|TP=0]: iteration: 2738 / 150000 | consumed_samples: 175232 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784981E+05 | tokens_per_sec_per_gpu: 9.039032E+03 | global_batch_size: 64 | lm_loss: 1.080338E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 264131.625 [ip-26-0-154-121:0]:06/21/2023 17:54:26 [INFO|DP=0|PP=0|TP=0]: iteration: 2739 / 150000 | consumed_samples: 175296 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782755E+05 | tokens_per_sec_per_gpu: 9.035555E+03 | global_batch_size: 64 | lm_loss: 1.080287E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 262534.594 [ip-26-0-154-121:0]:06/21/2023 17:54:27 [INFO|DP=0|PP=0|TP=0]: iteration: 2740 / 150000 | consumed_samples: 175360 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781124E+05 | tokens_per_sec_per_gpu: 9.033006E+03 | global_batch_size: 64 | lm_loss: 1.080338E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 259224.250 [ip-26-0-154-121:0]:06/21/2023 17:54:28 [INFO|DP=0|PP=0|TP=0]: iteration: 2741 / 150000 | consumed_samples: 175424 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787150E+05 | tokens_per_sec_per_gpu: 9.042422E+03 | global_batch_size: 64 | lm_loss: 1.080338E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 256250.625 [ip-26-0-154-121:0]:06/21/2023 17:54:29 [INFO|DP=0|PP=0|TP=0]: iteration: 2742 / 150000 | consumed_samples: 175488 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780619E+05 | tokens_per_sec_per_gpu: 9.032217E+03 | global_batch_size: 64 | lm_loss: 1.080338E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 254125.797 [ip-26-0-154-121:0]:06/21/2023 17:54:29 [INFO|DP=0|PP=0|TP=0]: iteration: 2743 / 150000 | consumed_samples: 175552 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782357E+05 | tokens_per_sec_per_gpu: 9.034932E+03 | global_batch_size: 64 | lm_loss: 1.080333E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 252234.172 [ip-26-0-154-121:0]:06/21/2023 17:54:30 [INFO|DP=0|PP=0|TP=0]: iteration: 2744 / 150000 | consumed_samples: 175616 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779650E+05 | tokens_per_sec_per_gpu: 9.030703E+03 | global_batch_size: 64 | lm_loss: 1.080330E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 250310.484 [ip-26-0-154-121:0]:06/21/2023 17:54:31 [INFO|DP=0|PP=0|TP=0]: iteration: 2745 / 150000 | consumed_samples: 175680 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780549E+05 | tokens_per_sec_per_gpu: 9.032108E+03 | global_batch_size: 64 | lm_loss: 1.080334E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 249461.203 [ip-26-0-154-121:0]:06/21/2023 17:54:32 [INFO|DP=0|PP=0|TP=0]: iteration: 2746 / 150000 | consumed_samples: 175744 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777557E+05 | tokens_per_sec_per_gpu: 9.027433E+03 | global_batch_size: 64 | lm_loss: 1.080327E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 249422.609 [ip-26-0-154-121:0]:06/21/2023 17:54:33 [INFO|DP=0|PP=0|TP=0]: iteration: 2747 / 150000 | consumed_samples: 175808 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777447E+05 | tokens_per_sec_per_gpu: 9.027260E+03 | global_batch_size: 64 | lm_loss: 1.080325E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 246912.234 [ip-26-0-154-121:0]:06/21/2023 17:54:34 [INFO|DP=0|PP=0|TP=0]: iteration: 2748 / 150000 | consumed_samples: 175872 | elapsed_time_per_iteration_ms: 907.8 | tokens_per_sec: 5.775263E+05 | tokens_per_sec_per_gpu: 9.023849E+03 | global_batch_size: 64 | lm_loss: 1.080325E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.24 | hardware_tflops_per_gpu: 104.24 | grad_norm: 242756.609 [ip-26-0-154-121:0]:06/21/2023 17:54:35 [INFO|DP=0|PP=0|TP=0]: iteration: 2749 / 150000 | consumed_samples: 175936 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779135E+05 | tokens_per_sec_per_gpu: 9.029898E+03 | global_batch_size: 64 | lm_loss: 1.080323E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 240014.750 [ip-26-0-154-121:0]:06/21/2023 17:54:36 [INFO|DP=0|PP=0|TP=0]: iteration: 2750 / 150000 | consumed_samples: 176000 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.778983E+05 | tokens_per_sec_per_gpu: 9.029661E+03 | global_batch_size: 64 | lm_loss: 1.080322E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 242919.438 [ip-26-0-154-121:0]:06/21/2023 17:54:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2751 / 150000 | consumed_samples: 176064 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779893E+05 | tokens_per_sec_per_gpu: 9.031083E+03 | global_batch_size: 64 | lm_loss: 1.080328E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 243772.953 [ip-26-0-154-121:0]:06/21/2023 17:54:38 [INFO|DP=0|PP=0|TP=0]: iteration: 2752 / 150000 | consumed_samples: 176128 | elapsed_time_per_iteration_ms: 908.7 | tokens_per_sec: 5.769398E+05 | tokens_per_sec_per_gpu: 9.014684E+03 | global_batch_size: 64 | lm_loss: 1.080325E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.13 | hardware_tflops_per_gpu: 104.13 | grad_norm: 235425.844 [ip-26-0-154-121:0]:06/21/2023 17:54:39 [INFO|DP=0|PP=0|TP=0]: iteration: 2753 / 150000 | consumed_samples: 176192 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780142E+05 | tokens_per_sec_per_gpu: 9.031472E+03 | global_batch_size: 64 | lm_loss: 1.080325E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 230184.719 [ip-26-0-154-121:0]:06/21/2023 17:54:39 [INFO|DP=0|PP=0|TP=0]: iteration: 2754 / 150000 | consumed_samples: 176256 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782416E+05 | tokens_per_sec_per_gpu: 9.035025E+03 | global_batch_size: 64 | lm_loss: 1.080323E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 230665.281 [ip-26-0-154-121:0]:06/21/2023 17:54:40 [INFO|DP=0|PP=0|TP=0]: iteration: 2755 / 150000 | consumed_samples: 176320 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.782021E+05 | tokens_per_sec_per_gpu: 9.034407E+03 | global_batch_size: 64 | lm_loss: 1.080322E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 233139.016 [ip-26-0-154-121:0]:06/21/2023 17:54:41 [INFO|DP=0|PP=0|TP=0]: iteration: 2756 / 150000 | consumed_samples: 176384 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779250E+05 | tokens_per_sec_per_gpu: 9.030079E+03 | global_batch_size: 64 | lm_loss: 1.080321E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 234182.516 [ip-26-0-154-121:0]:06/21/2023 17:54:42 [INFO|DP=0|PP=0|TP=0]: iteration: 2757 / 150000 | consumed_samples: 176448 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782414E+05 | tokens_per_sec_per_gpu: 9.035022E+03 | global_batch_size: 64 | lm_loss: 1.080317E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 231825.969 [ip-26-0-154-121:0]:06/21/2023 17:54:43 [INFO|DP=0|PP=0|TP=0]: iteration: 2758 / 150000 | consumed_samples: 176512 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779615E+05 | tokens_per_sec_per_gpu: 9.030648E+03 | global_batch_size: 64 | lm_loss: 1.080319E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 227491.250 [ip-26-0-154-121:0]:06/21/2023 17:54:44 [INFO|DP=0|PP=0|TP=0]: iteration: 2759 / 150000 | consumed_samples: 176576 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778714E+05 | tokens_per_sec_per_gpu: 9.029241E+03 | global_batch_size: 64 | lm_loss: 1.080314E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 227048.734 [ip-26-0-154-121:0]:06/21/2023 17:54:45 [INFO|DP=0|PP=0|TP=0]: iteration: 2760 / 150000 | consumed_samples: 176640 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781230E+05 | tokens_per_sec_per_gpu: 9.033172E+03 | global_batch_size: 64 | lm_loss: 1.080309E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 225602.641 [ip-26-0-154-121:0]:06/21/2023 17:54:46 [INFO|DP=0|PP=0|TP=0]: iteration: 2761 / 150000 | consumed_samples: 176704 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782849E+05 | tokens_per_sec_per_gpu: 9.035702E+03 | global_batch_size: 64 | lm_loss: 1.080315E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 227947.641 [ip-26-0-154-121:0]:06/21/2023 17:54:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2762 / 150000 | consumed_samples: 176768 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788818E+05 | tokens_per_sec_per_gpu: 9.045028E+03 | global_batch_size: 64 | lm_loss: 1.080312E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 223771.594 [ip-26-0-154-121:0]:06/21/2023 17:54:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2763 / 150000 | consumed_samples: 176832 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783607E+05 | tokens_per_sec_per_gpu: 9.036885E+03 | global_batch_size: 64 | lm_loss: 1.080311E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 219305.406 [ip-26-0-154-121:0]:06/21/2023 17:54:49 [INFO|DP=0|PP=0|TP=0]: iteration: 2764 / 150000 | consumed_samples: 176896 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779275E+05 | tokens_per_sec_per_gpu: 9.030117E+03 | global_batch_size: 64 | lm_loss: 1.080312E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 216447.531 [ip-26-0-154-121:0]:06/21/2023 17:54:49 [INFO|DP=0|PP=0|TP=0]: iteration: 2765 / 150000 | consumed_samples: 176960 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784710E+05 | tokens_per_sec_per_gpu: 9.038609E+03 | global_batch_size: 64 | lm_loss: 1.080313E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 214005.953 [ip-26-0-154-121:0]:06/21/2023 17:54:50 [INFO|DP=0|PP=0|TP=0]: iteration: 2766 / 150000 | consumed_samples: 177024 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785516E+05 | tokens_per_sec_per_gpu: 9.039869E+03 | global_batch_size: 64 | lm_loss: 1.080311E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 212784.031 [ip-26-0-154-121:0]:06/21/2023 17:54:51 [INFO|DP=0|PP=0|TP=0]: iteration: 2767 / 150000 | consumed_samples: 177088 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785311E+05 | tokens_per_sec_per_gpu: 9.039548E+03 | global_batch_size: 64 | lm_loss: 1.080308E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 213643.281 [ip-26-0-154-121:0]:06/21/2023 17:54:52 [INFO|DP=0|PP=0|TP=0]: iteration: 2768 / 150000 | consumed_samples: 177152 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783340E+05 | tokens_per_sec_per_gpu: 9.036469E+03 | global_batch_size: 64 | lm_loss: 1.080310E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 211342.578 [ip-26-0-154-121:0]:06/21/2023 17:54:53 [INFO|DP=0|PP=0|TP=0]: iteration: 2769 / 150000 | consumed_samples: 177216 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780672E+05 | tokens_per_sec_per_gpu: 9.032301E+03 | global_batch_size: 64 | lm_loss: 1.080307E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 210010.453 [ip-26-0-154-121:0]:06/21/2023 17:54:54 [INFO|DP=0|PP=0|TP=0]: iteration: 2770 / 150000 | consumed_samples: 177280 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782487E+05 | tokens_per_sec_per_gpu: 9.035136E+03 | global_batch_size: 64 | lm_loss: 1.080305E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 208495.672 [ip-26-0-154-121:0]:06/21/2023 17:54:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2771 / 150000 | consumed_samples: 177344 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778725E+05 | tokens_per_sec_per_gpu: 9.029258E+03 | global_batch_size: 64 | lm_loss: 1.080304E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 206730.375 [ip-26-0-154-121:0]:06/21/2023 17:54:56 [INFO|DP=0|PP=0|TP=0]: iteration: 2772 / 150000 | consumed_samples: 177408 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784723E+05 | tokens_per_sec_per_gpu: 9.038630E+03 | global_batch_size: 64 | lm_loss: 1.080302E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 205477.375 [ip-26-0-154-121:0]:06/21/2023 17:54:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2773 / 150000 | consumed_samples: 177472 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785576E+05 | tokens_per_sec_per_gpu: 9.039962E+03 | global_batch_size: 64 | lm_loss: 1.080301E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 208511.953 [ip-26-0-154-121:0]:06/21/2023 17:54:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2774 / 150000 | consumed_samples: 177536 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783073E+05 | tokens_per_sec_per_gpu: 9.036051E+03 | global_batch_size: 64 | lm_loss: 1.080304E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 211783.297 [ip-26-0-154-121:0]:06/21/2023 17:54:59 [INFO|DP=0|PP=0|TP=0]: iteration: 2775 / 150000 | consumed_samples: 177600 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780947E+05 | tokens_per_sec_per_gpu: 9.032730E+03 | global_batch_size: 64 | lm_loss: 1.080300E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 213699.031 [ip-26-0-154-121:0]:06/21/2023 17:54:59 [INFO|DP=0|PP=0|TP=0]: iteration: 2776 / 150000 | consumed_samples: 177664 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779978E+05 | tokens_per_sec_per_gpu: 9.031216E+03 | global_batch_size: 64 | lm_loss: 1.080299E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 209365.891 [ip-26-0-154-121:0]:06/21/2023 17:55:00 [INFO|DP=0|PP=0|TP=0]: iteration: 2777 / 150000 | consumed_samples: 177728 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785556E+05 | tokens_per_sec_per_gpu: 9.039931E+03 | global_batch_size: 64 | lm_loss: 1.080299E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 204344.453 [ip-26-0-154-121:0]:06/21/2023 17:55:01 [INFO|DP=0|PP=0|TP=0]: iteration: 2778 / 150000 | consumed_samples: 177792 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780054E+05 | tokens_per_sec_per_gpu: 9.031334E+03 | global_batch_size: 64 | lm_loss: 1.080296E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 206012.594 [ip-26-0-154-121:0]:06/21/2023 17:55:02 [INFO|DP=0|PP=0|TP=0]: iteration: 2779 / 150000 | consumed_samples: 177856 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780447E+05 | tokens_per_sec_per_gpu: 9.031949E+03 | global_batch_size: 64 | lm_loss: 1.080295E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 203225.750 [ip-26-0-154-121:0]:06/21/2023 17:55:03 [INFO|DP=0|PP=0|TP=0]: iteration: 2780 / 150000 | consumed_samples: 177920 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.780820E+05 | tokens_per_sec_per_gpu: 9.032531E+03 | global_batch_size: 64 | lm_loss: 1.080297E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 199785.797 [ip-26-0-154-121:0]:06/21/2023 17:55:04 [INFO|DP=0|PP=0|TP=0]: iteration: 2781 / 150000 | consumed_samples: 177984 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781440E+05 | tokens_per_sec_per_gpu: 9.033500E+03 | global_batch_size: 64 | lm_loss: 1.080298E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 201701.812 [ip-26-0-154-121:0]:06/21/2023 17:55:05 [INFO|DP=0|PP=0|TP=0]: iteration: 2782 / 150000 | consumed_samples: 178048 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776903E+05 | tokens_per_sec_per_gpu: 9.026411E+03 | global_batch_size: 64 | lm_loss: 1.080296E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 202139.547 [ip-26-0-154-121:0]:06/21/2023 17:55:06 [INFO|DP=0|PP=0|TP=0]: iteration: 2783 / 150000 | consumed_samples: 178112 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782366E+05 | tokens_per_sec_per_gpu: 9.034946E+03 | global_batch_size: 64 | lm_loss: 1.080294E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 196309.984 [ip-26-0-154-121:0]:06/21/2023 17:55:07 [INFO|DP=0|PP=0|TP=0]: iteration: 2784 / 150000 | consumed_samples: 178176 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.780107E+05 | tokens_per_sec_per_gpu: 9.031417E+03 | global_batch_size: 64 | lm_loss: 1.080293E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 199198.469 [ip-26-0-154-121:0]:06/21/2023 17:55:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2785 / 150000 | consumed_samples: 178240 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780308E+05 | tokens_per_sec_per_gpu: 9.031731E+03 | global_batch_size: 64 | lm_loss: 1.080290E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 199007.438 [ip-26-0-154-121:0]:06/21/2023 17:55:09 [INFO|DP=0|PP=0|TP=0]: iteration: 2786 / 150000 | consumed_samples: 178304 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784594E+05 | tokens_per_sec_per_gpu: 9.038428E+03 | global_batch_size: 64 | lm_loss: 1.080289E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 204146.156 [ip-26-0-154-121:0]:06/21/2023 17:55:09 [INFO|DP=0|PP=0|TP=0]: iteration: 2787 / 150000 | consumed_samples: 178368 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785550E+05 | tokens_per_sec_per_gpu: 9.039922E+03 | global_batch_size: 64 | lm_loss: 1.080294E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 204467.453 [ip-26-0-154-121:0]:06/21/2023 17:55:10 [INFO|DP=0|PP=0|TP=0]: iteration: 2788 / 150000 | consumed_samples: 178432 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783672E+05 | tokens_per_sec_per_gpu: 9.036988E+03 | global_batch_size: 64 | lm_loss: 1.080293E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 218087.812 [ip-26-0-154-121:0]:06/21/2023 17:55:11 [INFO|DP=0|PP=0|TP=0]: iteration: 2789 / 150000 | consumed_samples: 178496 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784917E+05 | tokens_per_sec_per_gpu: 9.038932E+03 | global_batch_size: 64 | lm_loss: 1.080294E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 218105.656 [ip-26-0-154-121:0]:06/21/2023 17:55:12 [INFO|DP=0|PP=0|TP=0]: iteration: 2790 / 150000 | consumed_samples: 178560 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780292E+05 | tokens_per_sec_per_gpu: 9.031707E+03 | global_batch_size: 64 | lm_loss: 1.080294E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 242494.625 [ip-26-0-154-121:0]:06/21/2023 17:55:13 [INFO|DP=0|PP=0|TP=0]: iteration: 2791 / 150000 | consumed_samples: 178624 | elapsed_time_per_iteration_ms: 907.6 | tokens_per_sec: 5.776478E+05 | tokens_per_sec_per_gpu: 9.025747E+03 | global_batch_size: 64 | lm_loss: 1.080289E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.26 | hardware_tflops_per_gpu: 104.26 | grad_norm: 225901.328 [ip-26-0-154-121:0]:06/21/2023 17:55:14 [INFO|DP=0|PP=0|TP=0]: iteration: 2792 / 150000 | consumed_samples: 178688 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782503E+05 | tokens_per_sec_per_gpu: 9.035160E+03 | global_batch_size: 64 | lm_loss: 1.080290E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 231499.516 [ip-26-0-154-121:0]:06/21/2023 17:55:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2793 / 150000 | consumed_samples: 178752 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785282E+05 | tokens_per_sec_per_gpu: 9.039503E+03 | global_batch_size: 64 | lm_loss: 1.080284E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 236514.469 [ip-26-0-154-121:0]:06/21/2023 17:55:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2794 / 150000 | consumed_samples: 178816 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778708E+05 | tokens_per_sec_per_gpu: 9.029232E+03 | global_batch_size: 64 | lm_loss: 1.080285E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 234399.672 [ip-26-0-154-121:0]:06/21/2023 17:55:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2795 / 150000 | consumed_samples: 178880 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.779083E+05 | tokens_per_sec_per_gpu: 9.029818E+03 | global_batch_size: 64 | lm_loss: 1.080285E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 222821.641 [ip-26-0-154-121:0]:06/21/2023 17:55:18 [INFO|DP=0|PP=0|TP=0]: iteration: 2796 / 150000 | consumed_samples: 178944 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785019E+05 | tokens_per_sec_per_gpu: 9.039092E+03 | global_batch_size: 64 | lm_loss: 1.080283E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 226974.453 [ip-26-0-154-121:0]:06/21/2023 17:55:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2797 / 150000 | consumed_samples: 179008 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787799E+05 | tokens_per_sec_per_gpu: 9.043436E+03 | global_batch_size: 64 | lm_loss: 1.080282E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 230435.938 [ip-26-0-154-121:0]:06/21/2023 17:55:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2798 / 150000 | consumed_samples: 179072 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782160E+05 | tokens_per_sec_per_gpu: 9.034626E+03 | global_batch_size: 64 | lm_loss: 1.080282E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 219286.188 [ip-26-0-154-121:0]:06/21/2023 17:55:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2799 / 150000 | consumed_samples: 179136 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783593E+05 | tokens_per_sec_per_gpu: 9.036864E+03 | global_batch_size: 64 | lm_loss: 1.080280E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 207085.359 [ip-26-0-154-121:0]:06/21/2023 17:55:21 [INFO|DP=0|PP=0|TP=0]: iteration: 2800 / 150000 | consumed_samples: 179200 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786207E+05 | tokens_per_sec_per_gpu: 9.040949E+03 | global_batch_size: 64 | lm_loss: 1.080281E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 199699.188 [ip-26-0-154-121:0]:06/21/2023 17:55:22 [INFO|DP=0|PP=0|TP=0]: iteration: 2801 / 150000 | consumed_samples: 179264 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780713E+05 | tokens_per_sec_per_gpu: 9.032365E+03 | global_batch_size: 64 | lm_loss: 1.080278E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 195693.500 [ip-26-0-154-121:0]:06/21/2023 17:55:23 [INFO|DP=0|PP=0|TP=0]: iteration: 2802 / 150000 | consumed_samples: 179328 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783004E+05 | tokens_per_sec_per_gpu: 9.035944E+03 | global_batch_size: 64 | lm_loss: 1.080280E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 192121.500 [ip-26-0-154-121:0]:06/21/2023 17:55:24 [INFO|DP=0|PP=0|TP=0]: iteration: 2803 / 150000 | consumed_samples: 179392 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787685E+05 | tokens_per_sec_per_gpu: 9.043257E+03 | global_batch_size: 64 | lm_loss: 1.080279E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 185020.984 [ip-26-0-154-121:0]:06/21/2023 17:55:25 [INFO|DP=0|PP=0|TP=0]: iteration: 2804 / 150000 | consumed_samples: 179456 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787053E+05 | tokens_per_sec_per_gpu: 9.042270E+03 | global_batch_size: 64 | lm_loss: 1.080278E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 187229.062 [ip-26-0-154-121:0]:06/21/2023 17:55:26 [INFO|DP=0|PP=0|TP=0]: iteration: 2805 / 150000 | consumed_samples: 179520 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786192E+05 | tokens_per_sec_per_gpu: 9.040925E+03 | global_batch_size: 64 | lm_loss: 1.080277E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 182793.234 [ip-26-0-154-121:0]:06/21/2023 17:55:27 [INFO|DP=0|PP=0|TP=0]: iteration: 2806 / 150000 | consumed_samples: 179584 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785445E+05 | tokens_per_sec_per_gpu: 9.039757E+03 | global_batch_size: 64 | lm_loss: 1.080275E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 177316.141 [ip-26-0-154-121:0]:06/21/2023 17:55:28 [INFO|DP=0|PP=0|TP=0]: iteration: 2807 / 150000 | consumed_samples: 179648 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788590E+05 | tokens_per_sec_per_gpu: 9.044671E+03 | global_batch_size: 64 | lm_loss: 1.080274E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 174887.531 [ip-26-0-154-121:0]:06/21/2023 17:55:28 [INFO|DP=0|PP=0|TP=0]: iteration: 2808 / 150000 | consumed_samples: 179712 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787826E+05 | tokens_per_sec_per_gpu: 9.043479E+03 | global_batch_size: 64 | lm_loss: 1.080277E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 171721.500 [ip-26-0-154-121:0]:06/21/2023 17:55:29 [INFO|DP=0|PP=0|TP=0]: iteration: 2809 / 150000 | consumed_samples: 179776 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788742E+05 | tokens_per_sec_per_gpu: 9.044909E+03 | global_batch_size: 64 | lm_loss: 1.080274E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 172431.359 [ip-26-0-154-121:0]:06/21/2023 17:55:30 [INFO|DP=0|PP=0|TP=0]: iteration: 2810 / 150000 | consumed_samples: 179840 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786937E+05 | tokens_per_sec_per_gpu: 9.042089E+03 | global_batch_size: 64 | lm_loss: 1.080277E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 169094.000 [ip-26-0-154-121:0]:06/21/2023 17:55:31 [INFO|DP=0|PP=0|TP=0]: iteration: 2811 / 150000 | consumed_samples: 179904 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783398E+05 | tokens_per_sec_per_gpu: 9.036560E+03 | global_batch_size: 64 | lm_loss: 1.080278E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 162836.812 [ip-26-0-154-121:0]:06/21/2023 17:55:32 [INFO|DP=0|PP=0|TP=0]: iteration: 2812 / 150000 | consumed_samples: 179968 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785819E+05 | tokens_per_sec_per_gpu: 9.040343E+03 | global_batch_size: 64 | lm_loss: 1.080276E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 177284.109 [ip-26-0-154-121:0]:06/21/2023 17:55:33 [INFO|DP=0|PP=0|TP=0]: iteration: 2813 / 150000 | consumed_samples: 180032 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789010E+05 | tokens_per_sec_per_gpu: 9.045328E+03 | global_batch_size: 64 | lm_loss: 1.080272E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 177132.125 [ip-26-0-154-121:0]:06/21/2023 17:55:34 [INFO|DP=0|PP=0|TP=0]: iteration: 2814 / 150000 | consumed_samples: 180096 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790304E+05 | tokens_per_sec_per_gpu: 9.047351E+03 | global_batch_size: 64 | lm_loss: 1.081162E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 182782.812 [ip-26-0-154-121:0]:06/21/2023 17:55:35 [INFO|DP=0|PP=0|TP=0]: iteration: 2815 / 150000 | consumed_samples: 180160 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788407E+05 | tokens_per_sec_per_gpu: 9.044386E+03 | global_batch_size: 64 | lm_loss: 1.080264E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 180545.656 [ip-26-0-154-121:0]:06/21/2023 17:55:36 [INFO|DP=0|PP=0|TP=0]: iteration: 2816 / 150000 | consumed_samples: 180224 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789155E+05 | tokens_per_sec_per_gpu: 9.045555E+03 | global_batch_size: 64 | lm_loss: 1.080264E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 172525.000 [ip-26-0-154-121:0]:06/21/2023 17:55:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2817 / 150000 | consumed_samples: 180288 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784338E+05 | tokens_per_sec_per_gpu: 9.038029E+03 | global_batch_size: 64 | lm_loss: 1.080268E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 504551.938 [ip-26-0-154-121:0]:06/21/2023 17:55:38 [INFO|DP=0|PP=0|TP=0]: iteration: 2818 / 150000 | consumed_samples: 180352 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785612E+05 | tokens_per_sec_per_gpu: 9.040019E+03 | global_batch_size: 64 | lm_loss: 1.080267E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 223886.438 [ip-26-0-154-121:0]:06/21/2023 17:55:38 [INFO|DP=0|PP=0|TP=0]: iteration: 2819 / 150000 | consumed_samples: 180416 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783716E+05 | tokens_per_sec_per_gpu: 9.037057E+03 | global_batch_size: 64 | lm_loss: 1.080274E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 193371.469 [ip-26-0-154-121:0]:06/21/2023 17:55:39 [INFO|DP=0|PP=0|TP=0]: iteration: 2820 / 150000 | consumed_samples: 180480 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782950E+05 | tokens_per_sec_per_gpu: 9.035859E+03 | global_batch_size: 64 | lm_loss: 1.080262E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 212051.016 [ip-26-0-154-121:0]:06/21/2023 17:55:40 [INFO|DP=0|PP=0|TP=0]: iteration: 2821 / 150000 | consumed_samples: 180544 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786785E+05 | tokens_per_sec_per_gpu: 9.041851E+03 | global_batch_size: 64 | lm_loss: 1.080278E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 205424.172 [ip-26-0-154-121:0]:06/21/2023 17:55:41 [INFO|DP=0|PP=0|TP=0]: iteration: 2822 / 150000 | consumed_samples: 180608 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782191E+05 | tokens_per_sec_per_gpu: 9.034673E+03 | global_batch_size: 64 | lm_loss: 1.080276E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 207868.047 [ip-26-0-154-121:0]:06/21/2023 17:55:42 [INFO|DP=0|PP=0|TP=0]: iteration: 2823 / 150000 | consumed_samples: 180672 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779744E+05 | tokens_per_sec_per_gpu: 9.030850E+03 | global_batch_size: 64 | lm_loss: 1.080271E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 225036.500 [ip-26-0-154-121:0]:06/21/2023 17:55:43 [INFO|DP=0|PP=0|TP=0]: iteration: 2824 / 150000 | consumed_samples: 180736 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783331E+05 | tokens_per_sec_per_gpu: 9.036455E+03 | global_batch_size: 64 | lm_loss: 1.080270E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 225509.344 [ip-26-0-154-121:0]:06/21/2023 17:55:44 [INFO|DP=0|PP=0|TP=0]: iteration: 2825 / 150000 | consumed_samples: 180800 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781791E+05 | tokens_per_sec_per_gpu: 9.034048E+03 | global_batch_size: 64 | lm_loss: 1.080264E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 223958.109 [ip-26-0-154-121:0]:06/21/2023 17:55:45 [INFO|DP=0|PP=0|TP=0]: iteration: 2826 / 150000 | consumed_samples: 180864 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779891E+05 | tokens_per_sec_per_gpu: 9.031080E+03 | global_batch_size: 64 | lm_loss: 1.080266E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 249692.328 [ip-26-0-154-121:0]:06/21/2023 17:55:46 [INFO|DP=0|PP=0|TP=0]: iteration: 2827 / 150000 | consumed_samples: 180928 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782109E+05 | tokens_per_sec_per_gpu: 9.034545E+03 | global_batch_size: 64 | lm_loss: 1.080269E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 241769.141 [ip-26-0-154-121:0]:06/21/2023 17:55:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2828 / 150000 | consumed_samples: 180992 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781840E+05 | tokens_per_sec_per_gpu: 9.034124E+03 | global_batch_size: 64 | lm_loss: 1.080400E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 302103.125 [ip-26-0-154-121:0]:06/21/2023 17:55:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2829 / 150000 | consumed_samples: 181056 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784049E+05 | tokens_per_sec_per_gpu: 9.037577E+03 | global_batch_size: 64 | lm_loss: 1.080267E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 592036.312 [ip-26-0-154-121:0]:06/21/2023 17:55:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2830 / 150000 | consumed_samples: 181120 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781607E+05 | tokens_per_sec_per_gpu: 9.033761E+03 | global_batch_size: 64 | lm_loss: 1.080261E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 301675.125 [ip-26-0-154-121:0]:06/21/2023 17:55:49 [INFO|DP=0|PP=0|TP=0]: iteration: 2831 / 150000 | consumed_samples: 181184 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.783998E+05 | tokens_per_sec_per_gpu: 9.037496E+03 | global_batch_size: 64 | lm_loss: 1.080264E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 312424.219 [ip-26-0-154-121:0]:06/21/2023 17:55:50 [INFO|DP=0|PP=0|TP=0]: iteration: 2832 / 150000 | consumed_samples: 181248 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783902E+05 | tokens_per_sec_per_gpu: 9.037347E+03 | global_batch_size: 64 | lm_loss: 1.080381E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 2267669.500 [ip-26-0-154-121:0]:06/21/2023 17:55:51 [INFO|DP=0|PP=0|TP=0]: iteration: 2833 / 150000 | consumed_samples: 181312 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780473E+05 | tokens_per_sec_per_gpu: 9.031990E+03 | global_batch_size: 64 | lm_loss: 1.080370E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 2593817.500 [ip-26-0-154-121:0]:06/21/2023 17:55:52 [INFO|DP=0|PP=0|TP=0]: iteration: 2834 / 150000 | consumed_samples: 181376 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783023E+05 | tokens_per_sec_per_gpu: 9.035973E+03 | global_batch_size: 64 | lm_loss: 1.080432E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 2355771.250 [ip-26-0-154-121:0]:06/21/2023 17:55:53 [INFO|DP=0|PP=0|TP=0]: iteration: 2835 / 150000 | consumed_samples: 181440 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787563E+05 | tokens_per_sec_per_gpu: 9.043067E+03 | global_batch_size: 64 | lm_loss: 1.080483E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 2390558.750 [ip-26-0-154-121:0]:06/21/2023 17:55:54 [INFO|DP=0|PP=0|TP=0]: iteration: 2836 / 150000 | consumed_samples: 181504 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783327E+05 | tokens_per_sec_per_gpu: 9.036448E+03 | global_batch_size: 64 | lm_loss: 1.080366E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 2800537.500 [ip-26-0-154-121:0]:06/21/2023 17:55:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2837 / 150000 | consumed_samples: 181568 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783610E+05 | tokens_per_sec_per_gpu: 9.036890E+03 | global_batch_size: 64 | lm_loss: 1.080408E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 2727349.000 [ip-26-0-154-121:0]:06/21/2023 17:55:56 [INFO|DP=0|PP=0|TP=0]: iteration: 2838 / 150000 | consumed_samples: 181632 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783800E+05 | tokens_per_sec_per_gpu: 9.037187E+03 | global_batch_size: 64 | lm_loss: 1.080386E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 2737284.000 [ip-26-0-154-121:0]:06/21/2023 17:55:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2839 / 150000 | consumed_samples: 181696 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786337E+05 | tokens_per_sec_per_gpu: 9.041151E+03 | global_batch_size: 64 | lm_loss: 1.080372E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 4193737.500 [ip-26-0-154-121:0]:06/21/2023 17:55:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2840 / 150000 | consumed_samples: 181760 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788681E+05 | tokens_per_sec_per_gpu: 9.044814E+03 | global_batch_size: 64 | lm_loss: 1.080390E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 3023525.500 [ip-26-0-154-121:0]:06/21/2023 17:55:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2841 / 150000 | consumed_samples: 181824 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782887E+05 | tokens_per_sec_per_gpu: 9.035761E+03 | global_batch_size: 64 | lm_loss: 1.080351E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 2991454.750 [ip-26-0-154-121:0]:06/21/2023 17:55:59 [INFO|DP=0|PP=0|TP=0]: iteration: 2842 / 150000 | consumed_samples: 181888 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788291E+05 | tokens_per_sec_per_gpu: 9.044205E+03 | global_batch_size: 64 | lm_loss: 1.080345E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 3035328.000 [ip-26-0-154-121:0]:06/21/2023 17:56:00 [INFO|DP=0|PP=0|TP=0]: iteration: 2843 / 150000 | consumed_samples: 181952 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787051E+05 | tokens_per_sec_per_gpu: 9.042267E+03 | global_batch_size: 64 | lm_loss: 1.080364E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3140248.000 [ip-26-0-154-121:0]:06/21/2023 17:56:01 [INFO|DP=0|PP=0|TP=0]: iteration: 2844 / 150000 | consumed_samples: 182016 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786865E+05 | tokens_per_sec_per_gpu: 9.041977E+03 | global_batch_size: 64 | lm_loss: 1.080331E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 3124574.500 [ip-26-0-154-121:0]:06/21/2023 17:56:02 [INFO|DP=0|PP=0|TP=0]: iteration: 2845 / 150000 | consumed_samples: 182080 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785662E+05 | tokens_per_sec_per_gpu: 9.040098E+03 | global_batch_size: 64 | lm_loss: 1.080364E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 3268868.750 [ip-26-0-154-121:0]:06/21/2023 17:56:03 [INFO|DP=0|PP=0|TP=0]: iteration: 2846 / 150000 | consumed_samples: 182144 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786136E+05 | tokens_per_sec_per_gpu: 9.040837E+03 | global_batch_size: 64 | lm_loss: 1.080291E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 3187516.250 [ip-26-0-154-121:0]:06/21/2023 17:56:04 [INFO|DP=0|PP=0|TP=0]: iteration: 2847 / 150000 | consumed_samples: 182208 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780707E+05 | tokens_per_sec_per_gpu: 9.032355E+03 | global_batch_size: 64 | lm_loss: 1.080354E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 2849231.750 [ip-26-0-154-121:0]:06/21/2023 17:56:05 [INFO|DP=0|PP=0|TP=0]: iteration: 2848 / 150000 | consumed_samples: 182272 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785020E+05 | tokens_per_sec_per_gpu: 9.039094E+03 | global_batch_size: 64 | lm_loss: 1.080346E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 2793999.750 [ip-26-0-154-121:0]:06/21/2023 17:56:06 [INFO|DP=0|PP=0|TP=0]: iteration: 2849 / 150000 | consumed_samples: 182336 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783194E+05 | tokens_per_sec_per_gpu: 9.036241E+03 | global_batch_size: 64 | lm_loss: 1.080383E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 2828367.500 [ip-26-0-154-121:0]:06/21/2023 17:56:07 [INFO|DP=0|PP=0|TP=0]: iteration: 2850 / 150000 | consumed_samples: 182400 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785465E+05 | tokens_per_sec_per_gpu: 9.039788E+03 | global_batch_size: 64 | lm_loss: 1.080480E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 2729894.250 [ip-26-0-154-121:0]:06/21/2023 17:56:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2851 / 150000 | consumed_samples: 182464 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783050E+05 | tokens_per_sec_per_gpu: 9.036016E+03 | global_batch_size: 64 | lm_loss: 1.080390E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 3085385.000 [ip-26-0-154-121:0]:06/21/2023 17:56:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2852 / 150000 | consumed_samples: 182528 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784448E+05 | tokens_per_sec_per_gpu: 9.038200E+03 | global_batch_size: 64 | lm_loss: 1.080347E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3490530.000 [ip-26-0-154-121:0]:06/21/2023 17:56:09 [INFO|DP=0|PP=0|TP=0]: iteration: 2853 / 150000 | consumed_samples: 182592 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783937E+05 | tokens_per_sec_per_gpu: 9.037401E+03 | global_batch_size: 64 | lm_loss: 1.080486E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 2957086.500 [ip-26-0-154-121:0]:06/21/2023 17:56:10 [INFO|DP=0|PP=0|TP=0]: iteration: 2854 / 150000 | consumed_samples: 182656 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783722E+05 | tokens_per_sec_per_gpu: 9.037066E+03 | global_batch_size: 64 | lm_loss: 1.080360E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 3186607.000 [ip-26-0-154-121:0]:06/21/2023 17:56:11 [INFO|DP=0|PP=0|TP=0]: iteration: 2855 / 150000 | consumed_samples: 182720 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787101E+05 | tokens_per_sec_per_gpu: 9.042346E+03 | global_batch_size: 64 | lm_loss: 1.080354E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 3324992.750 [ip-26-0-154-121:0]:06/21/2023 17:56:12 [INFO|DP=0|PP=0|TP=0]: iteration: 2856 / 150000 | consumed_samples: 182784 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784533E+05 | tokens_per_sec_per_gpu: 9.038333E+03 | global_batch_size: 64 | lm_loss: 1.080363E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 3249222.750 [ip-26-0-154-121:0]:06/21/2023 17:56:13 [INFO|DP=0|PP=0|TP=0]: iteration: 2857 / 150000 | consumed_samples: 182848 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786357E+05 | tokens_per_sec_per_gpu: 9.041182E+03 | global_batch_size: 64 | lm_loss: 1.080360E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2873766.750 [ip-26-0-154-121:0]:06/21/2023 17:56:14 [INFO|DP=0|PP=0|TP=0]: iteration: 2858 / 150000 | consumed_samples: 182912 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788910E+05 | tokens_per_sec_per_gpu: 9.045171E+03 | global_batch_size: 64 | lm_loss: 1.080435E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 2750545.000 [ip-26-0-154-121:0]:06/21/2023 17:56:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2859 / 150000 | consumed_samples: 182976 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781598E+05 | tokens_per_sec_per_gpu: 9.033747E+03 | global_batch_size: 64 | lm_loss: 1.080398E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 2729823.250 [ip-26-0-154-121:0]:06/21/2023 17:56:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2860 / 150000 | consumed_samples: 183040 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785665E+05 | tokens_per_sec_per_gpu: 9.040102E+03 | global_batch_size: 64 | lm_loss: 1.080453E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 2550194.750 [ip-26-0-154-121:0]:06/21/2023 17:56:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2861 / 150000 | consumed_samples: 183104 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783601E+05 | tokens_per_sec_per_gpu: 9.036876E+03 | global_batch_size: 64 | lm_loss: 1.080413E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 2641627.500 [ip-26-0-154-121:0]:06/21/2023 17:56:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2862 / 150000 | consumed_samples: 183168 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785737E+05 | tokens_per_sec_per_gpu: 9.040214E+03 | global_batch_size: 64 | lm_loss: 1.080419E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 2602200.000 [ip-26-0-154-121:0]:06/21/2023 17:56:18 [INFO|DP=0|PP=0|TP=0]: iteration: 2863 / 150000 | consumed_samples: 183232 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785276E+05 | tokens_per_sec_per_gpu: 9.039493E+03 | global_batch_size: 64 | lm_loss: 1.080457E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 2483250.500 [ip-26-0-154-121:0]:06/21/2023 17:56:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2864 / 150000 | consumed_samples: 183296 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784086E+05 | tokens_per_sec_per_gpu: 9.037634E+03 | global_batch_size: 64 | lm_loss: 1.080415E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 2546539.000 [ip-26-0-154-121:0]:06/21/2023 17:56:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2865 / 150000 | consumed_samples: 183360 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784392E+05 | tokens_per_sec_per_gpu: 9.038112E+03 | global_batch_size: 64 | lm_loss: 1.080465E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 2204801.000 [ip-26-0-154-121:0]:06/21/2023 17:56:21 [INFO|DP=0|PP=0|TP=0]: iteration: 2866 / 150000 | consumed_samples: 183424 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783435E+05 | tokens_per_sec_per_gpu: 9.036617E+03 | global_batch_size: 64 | lm_loss: 1.080420E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 1905998.750 [ip-26-0-154-121:0]:06/21/2023 17:56:22 [INFO|DP=0|PP=0|TP=0]: iteration: 2867 / 150000 | consumed_samples: 183488 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788680E+05 | tokens_per_sec_per_gpu: 9.044812E+03 | global_batch_size: 64 | lm_loss: 1.080437E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 2255166.750 [ip-26-0-154-121:0]:06/21/2023 17:56:23 [INFO|DP=0|PP=0|TP=0]: iteration: 2868 / 150000 | consumed_samples: 183552 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785119E+05 | tokens_per_sec_per_gpu: 9.039249E+03 | global_batch_size: 64 | lm_loss: 1.080478E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1930547.125 [ip-26-0-154-121:0]:06/21/2023 17:56:24 [INFO|DP=0|PP=0|TP=0]: iteration: 2869 / 150000 | consumed_samples: 183616 | elapsed_time_per_iteration_ms: 905.1 | tokens_per_sec: 5.792508E+05 | tokens_per_sec_per_gpu: 9.050794E+03 | global_batch_size: 64 | lm_loss: 1.080736E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.55 | hardware_tflops_per_gpu: 104.55 | grad_norm: 653615.875 [ip-26-0-154-121:0]:06/21/2023 17:56:25 [INFO|DP=0|PP=0|TP=0]: iteration: 2870 / 150000 | consumed_samples: 183680 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785670E+05 | tokens_per_sec_per_gpu: 9.040109E+03 | global_batch_size: 64 | lm_loss: 1.080519E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 480021.375 [ip-26-0-154-121:0]:06/21/2023 17:56:26 [INFO|DP=0|PP=0|TP=0]: iteration: 2871 / 150000 | consumed_samples: 183744 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784220E+05 | tokens_per_sec_per_gpu: 9.037843E+03 | global_batch_size: 64 | lm_loss: 1.080414E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 435906.969 [ip-26-0-154-121:0]:06/21/2023 17:56:27 [INFO|DP=0|PP=0|TP=0]: iteration: 2872 / 150000 | consumed_samples: 183808 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785041E+05 | tokens_per_sec_per_gpu: 9.039127E+03 | global_batch_size: 64 | lm_loss: 1.080482E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 891779.875 [ip-26-0-154-121:0]:06/21/2023 17:56:27 [INFO|DP=0|PP=0|TP=0]: iteration: 2873 / 150000 | consumed_samples: 183872 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783363E+05 | tokens_per_sec_per_gpu: 9.036505E+03 | global_batch_size: 64 | lm_loss: 1.080429E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 440521.625 [ip-26-0-154-121:0]:06/21/2023 17:56:28 [INFO|DP=0|PP=0|TP=0]: iteration: 2874 / 150000 | consumed_samples: 183936 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784034E+05 | tokens_per_sec_per_gpu: 9.037553E+03 | global_batch_size: 64 | lm_loss: 1.080416E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 390494.625 [ip-26-0-154-121:0]:06/21/2023 17:56:29 [INFO|DP=0|PP=0|TP=0]: iteration: 2875 / 150000 | consumed_samples: 184000 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787080E+05 | tokens_per_sec_per_gpu: 9.042312E+03 | global_batch_size: 64 | lm_loss: 1.080368E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 364506.781 [ip-26-0-154-121:0]:06/21/2023 17:56:30 [INFO|DP=0|PP=0|TP=0]: iteration: 2876 / 150000 | consumed_samples: 184064 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786445E+05 | tokens_per_sec_per_gpu: 9.041320E+03 | global_batch_size: 64 | lm_loss: 1.080290E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 370545.812 [ip-26-0-154-121:0]:06/21/2023 17:56:31 [INFO|DP=0|PP=0|TP=0]: iteration: 2877 / 150000 | consumed_samples: 184128 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789262E+05 | tokens_per_sec_per_gpu: 9.045721E+03 | global_batch_size: 64 | lm_loss: 1.080331E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 356049.344 [ip-26-0-154-121:0]:06/21/2023 17:56:32 [INFO|DP=0|PP=0|TP=0]: iteration: 2878 / 150000 | consumed_samples: 184192 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782972E+05 | tokens_per_sec_per_gpu: 9.035894E+03 | global_batch_size: 64 | lm_loss: 1.080397E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 407244.344 [ip-26-0-154-121:0]:06/21/2023 17:56:33 [INFO|DP=0|PP=0|TP=0]: iteration: 2879 / 150000 | consumed_samples: 184256 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785475E+05 | tokens_per_sec_per_gpu: 9.039805E+03 | global_batch_size: 64 | lm_loss: 1.080489E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 547287.062 [ip-26-0-154-121:0]:06/21/2023 17:56:34 [INFO|DP=0|PP=0|TP=0]: iteration: 2880 / 150000 | consumed_samples: 184320 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783903E+05 | tokens_per_sec_per_gpu: 9.037349E+03 | global_batch_size: 64 | lm_loss: 1.080296E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1398347.250 [ip-26-0-154-121:0]:06/21/2023 17:56:35 [INFO|DP=0|PP=0|TP=0]: iteration: 2881 / 150000 | consumed_samples: 184384 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788524E+05 | tokens_per_sec_per_gpu: 9.044569E+03 | global_batch_size: 64 | lm_loss: 1.080278E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1559591.125 [ip-26-0-154-121:0]:06/21/2023 17:56:36 [INFO|DP=0|PP=0|TP=0]: iteration: 2882 / 150000 | consumed_samples: 184448 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788942E+05 | tokens_per_sec_per_gpu: 9.045221E+03 | global_batch_size: 64 | lm_loss: 1.080287E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1722361.125 [ip-26-0-154-121:0]:06/21/2023 17:56:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2883 / 150000 | consumed_samples: 184512 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784778E+05 | tokens_per_sec_per_gpu: 9.038716E+03 | global_batch_size: 64 | lm_loss: 1.080333E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1710127.250 [ip-26-0-154-121:0]:06/21/2023 17:56:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2884 / 150000 | consumed_samples: 184576 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783219E+05 | tokens_per_sec_per_gpu: 9.036279E+03 | global_batch_size: 64 | lm_loss: 1.080396E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 1822966.000 [ip-26-0-154-121:0]:06/21/2023 17:56:38 [INFO|DP=0|PP=0|TP=0]: iteration: 2885 / 150000 | consumed_samples: 184640 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784956E+05 | tokens_per_sec_per_gpu: 9.038994E+03 | global_batch_size: 64 | lm_loss: 1.080445E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1927821.250 [ip-26-0-154-121:0]:06/21/2023 17:56:39 [INFO|DP=0|PP=0|TP=0]: iteration: 2886 / 150000 | consumed_samples: 184704 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784904E+05 | tokens_per_sec_per_gpu: 9.038913E+03 | global_batch_size: 64 | lm_loss: 1.080495E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1941886.125 [ip-26-0-154-121:0]:06/21/2023 17:56:40 [INFO|DP=0|PP=0|TP=0]: iteration: 2887 / 150000 | consumed_samples: 184768 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784821E+05 | tokens_per_sec_per_gpu: 9.038782E+03 | global_batch_size: 64 | lm_loss: 1.080465E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 2127439.000 [ip-26-0-154-121:0]:06/21/2023 17:56:41 [INFO|DP=0|PP=0|TP=0]: iteration: 2888 / 150000 | consumed_samples: 184832 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787334E+05 | tokens_per_sec_per_gpu: 9.042710E+03 | global_batch_size: 64 | lm_loss: 1.080478E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 4852013.500 [ip-26-0-154-121:0]:06/21/2023 17:56:42 [INFO|DP=0|PP=0|TP=0]: iteration: 2889 / 150000 | consumed_samples: 184896 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786326E+05 | tokens_per_sec_per_gpu: 9.041135E+03 | global_batch_size: 64 | lm_loss: 1.080497E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2744439.000 [ip-26-0-154-121:0]:06/21/2023 17:56:43 [INFO|DP=0|PP=0|TP=0]: iteration: 2890 / 150000 | consumed_samples: 184960 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783861E+05 | tokens_per_sec_per_gpu: 9.037282E+03 | global_batch_size: 64 | lm_loss: 1.080595E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 2075826.875 [ip-26-0-154-121:0]:06/21/2023 17:56:44 [INFO|DP=0|PP=0|TP=0]: iteration: 2891 / 150000 | consumed_samples: 185024 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783830E+05 | tokens_per_sec_per_gpu: 9.037235E+03 | global_batch_size: 64 | lm_loss: 1.080529E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 2054890.000 [ip-26-0-154-121:0]:06/21/2023 17:56:45 [INFO|DP=0|PP=0|TP=0]: iteration: 2892 / 150000 | consumed_samples: 185088 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785889E+05 | tokens_per_sec_per_gpu: 9.040452E+03 | global_batch_size: 64 | lm_loss: 1.080490E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 2099051.500 [ip-26-0-154-121:0]:06/21/2023 17:56:46 [INFO|DP=0|PP=0|TP=0]: iteration: 2893 / 150000 | consumed_samples: 185152 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789827E+05 | tokens_per_sec_per_gpu: 9.046605E+03 | global_batch_size: 64 | lm_loss: 1.080474E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 2172748.500 [ip-26-0-154-121:0]:06/21/2023 17:56:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2894 / 150000 | consumed_samples: 185216 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789221E+05 | tokens_per_sec_per_gpu: 9.045657E+03 | global_batch_size: 64 | lm_loss: 1.080502E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 2136056.500 [ip-26-0-154-121:0]:06/21/2023 17:56:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2895 / 150000 | consumed_samples: 185280 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786378E+05 | tokens_per_sec_per_gpu: 9.041216E+03 | global_batch_size: 64 | lm_loss: 1.080513E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2179638.750 [ip-26-0-154-121:0]:06/21/2023 17:56:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2896 / 150000 | consumed_samples: 185344 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785939E+05 | tokens_per_sec_per_gpu: 9.040530E+03 | global_batch_size: 64 | lm_loss: 1.080523E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 2220445.500 [ip-26-0-154-121:0]:06/21/2023 17:56:49 [INFO|DP=0|PP=0|TP=0]: iteration: 2897 / 150000 | consumed_samples: 185408 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785474E+05 | tokens_per_sec_per_gpu: 9.039803E+03 | global_batch_size: 64 | lm_loss: 1.080546E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 2625405.750 [ip-26-0-154-121:0]:06/21/2023 17:56:50 [INFO|DP=0|PP=0|TP=0]: iteration: 2898 / 150000 | consumed_samples: 185472 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787741E+05 | tokens_per_sec_per_gpu: 9.043345E+03 | global_batch_size: 64 | lm_loss: 1.080547E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 2276550.000 [ip-26-0-154-121:0]:06/21/2023 17:56:51 [INFO|DP=0|PP=0|TP=0]: iteration: 2899 / 150000 | consumed_samples: 185536 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788530E+05 | tokens_per_sec_per_gpu: 9.044578E+03 | global_batch_size: 64 | lm_loss: 1.080508E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 2614937.500 [ip-26-0-154-121:0]:06/21/2023 17:56:52 [INFO|DP=0|PP=0|TP=0]: iteration: 2900 / 150000 | consumed_samples: 185600 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786973E+05 | tokens_per_sec_per_gpu: 9.042146E+03 | global_batch_size: 64 | lm_loss: 1.080464E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 2268287.250 [ip-26-0-154-121:0]:06/21/2023 17:56:53 [INFO|DP=0|PP=0|TP=0]: iteration: 2901 / 150000 | consumed_samples: 185664 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786072E+05 | tokens_per_sec_per_gpu: 9.040737E+03 | global_batch_size: 64 | lm_loss: 1.080493E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 2156618.500 [ip-26-0-154-121:0]:06/21/2023 17:56:54 [INFO|DP=0|PP=0|TP=0]: iteration: 2902 / 150000 | consumed_samples: 185728 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791360E+05 | tokens_per_sec_per_gpu: 9.048999E+03 | global_batch_size: 64 | lm_loss: 1.080503E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.53 | hardware_tflops_per_gpu: 104.53 | grad_norm: 2182044.250 [ip-26-0-154-121:0]:06/21/2023 17:56:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2903 / 150000 | consumed_samples: 185792 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786267E+05 | tokens_per_sec_per_gpu: 9.041042E+03 | global_batch_size: 64 | lm_loss: 1.080524E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 2136328.750 [ip-26-0-154-121:0]:06/21/2023 17:56:56 [INFO|DP=0|PP=0|TP=0]: iteration: 2904 / 150000 | consumed_samples: 185856 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786600E+05 | tokens_per_sec_per_gpu: 9.041563E+03 | global_batch_size: 64 | lm_loss: 1.080517E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2217186.250 [ip-26-0-154-121:0]:06/21/2023 17:56:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2905 / 150000 | consumed_samples: 185920 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789958E+05 | tokens_per_sec_per_gpu: 9.046810E+03 | global_batch_size: 64 | lm_loss: 1.080523E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 2172627.250 [ip-26-0-154-121:0]:06/21/2023 17:56:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2906 / 150000 | consumed_samples: 185984 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788041E+05 | tokens_per_sec_per_gpu: 9.043814E+03 | global_batch_size: 64 | lm_loss: 1.080532E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 2086780.875 [ip-26-0-154-121:0]:06/21/2023 17:56:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2907 / 150000 | consumed_samples: 186048 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786110E+05 | tokens_per_sec_per_gpu: 9.040797E+03 | global_batch_size: 64 | lm_loss: 1.080551E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 2246557.750 [ip-26-0-154-121:0]:06/21/2023 17:56:59 [INFO|DP=0|PP=0|TP=0]: iteration: 2908 / 150000 | consumed_samples: 186112 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789980E+05 | tokens_per_sec_per_gpu: 9.046843E+03 | global_batch_size: 64 | lm_loss: 1.080570E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 2216984.000 [ip-26-0-154-121:0]:06/21/2023 17:57:00 [INFO|DP=0|PP=0|TP=0]: iteration: 2909 / 150000 | consumed_samples: 186176 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785245E+05 | tokens_per_sec_per_gpu: 9.039446E+03 | global_batch_size: 64 | lm_loss: 1.080535E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 2175690.000 [ip-26-0-154-121:0]:06/21/2023 17:57:01 [INFO|DP=0|PP=0|TP=0]: iteration: 2910 / 150000 | consumed_samples: 186240 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787120E+05 | tokens_per_sec_per_gpu: 9.042374E+03 | global_batch_size: 64 | lm_loss: 1.080522E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 2188670.750 [ip-26-0-154-121:0]:06/21/2023 17:57:02 [INFO|DP=0|PP=0|TP=0]: iteration: 2911 / 150000 | consumed_samples: 186304 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788236E+05 | tokens_per_sec_per_gpu: 9.044119E+03 | global_batch_size: 64 | lm_loss: 1.080524E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 2116221.000 [ip-26-0-154-121:0]:06/21/2023 17:57:03 [INFO|DP=0|PP=0|TP=0]: iteration: 2912 / 150000 | consumed_samples: 186368 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786521E+05 | tokens_per_sec_per_gpu: 9.041439E+03 | global_batch_size: 64 | lm_loss: 1.080509E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2127103.000 [ip-26-0-154-121:0]:06/21/2023 17:57:04 [INFO|DP=0|PP=0|TP=0]: iteration: 2913 / 150000 | consumed_samples: 186432 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785997E+05 | tokens_per_sec_per_gpu: 9.040621E+03 | global_batch_size: 64 | lm_loss: 1.080520E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 2073831.625 [ip-26-0-154-121:0]:06/21/2023 17:57:05 [INFO|DP=0|PP=0|TP=0]: iteration: 2914 / 150000 | consumed_samples: 186496 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791514E+05 | tokens_per_sec_per_gpu: 9.049240E+03 | global_batch_size: 64 | lm_loss: 1.080505E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.53 | hardware_tflops_per_gpu: 104.53 | grad_norm: 2063152.125 [ip-26-0-154-121:0]:06/21/2023 17:57:06 [INFO|DP=0|PP=0|TP=0]: iteration: 2915 / 150000 | consumed_samples: 186560 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787580E+05 | tokens_per_sec_per_gpu: 9.043093E+03 | global_batch_size: 64 | lm_loss: 1.080514E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 2000652.750 [ip-26-0-154-121:0]:06/21/2023 17:57:06 [INFO|DP=0|PP=0|TP=0]: iteration: 2916 / 150000 | consumed_samples: 186624 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788340E+05 | tokens_per_sec_per_gpu: 9.044281E+03 | global_batch_size: 64 | lm_loss: 1.080540E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 2011866.750 [ip-26-0-154-121:0]:06/21/2023 17:57:07 [INFO|DP=0|PP=0|TP=0]: iteration: 2917 / 150000 | consumed_samples: 186688 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789012E+05 | tokens_per_sec_per_gpu: 9.045331E+03 | global_batch_size: 64 | lm_loss: 1.080536E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 2065590.500 [ip-26-0-154-121:0]:06/21/2023 17:57:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2918 / 150000 | consumed_samples: 186752 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786820E+05 | tokens_per_sec_per_gpu: 9.041906E+03 | global_batch_size: 64 | lm_loss: 1.080538E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1995975.125 [ip-26-0-154-121:0]:06/21/2023 17:57:09 [INFO|DP=0|PP=0|TP=0]: iteration: 2919 / 150000 | consumed_samples: 186816 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786404E+05 | tokens_per_sec_per_gpu: 9.041256E+03 | global_batch_size: 64 | lm_loss: 1.080508E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 2026823.000 [ip-26-0-154-121:0]:06/21/2023 17:57:10 [INFO|DP=0|PP=0|TP=0]: iteration: 2920 / 150000 | consumed_samples: 186880 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786142E+05 | tokens_per_sec_per_gpu: 9.040847E+03 | global_batch_size: 64 | lm_loss: 1.080530E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 2074825.250 [ip-26-0-154-121:0]:06/21/2023 17:57:11 [INFO|DP=0|PP=0|TP=0]: iteration: 2921 / 150000 | consumed_samples: 186944 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788625E+05 | tokens_per_sec_per_gpu: 9.044726E+03 | global_batch_size: 64 | lm_loss: 1.080499E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1973054.250 [ip-26-0-154-121:0]:06/21/2023 17:57:12 [INFO|DP=0|PP=0|TP=0]: iteration: 2922 / 150000 | consumed_samples: 187008 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787944E+05 | tokens_per_sec_per_gpu: 9.043662E+03 | global_batch_size: 64 | lm_loss: 1.080510E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1953625.625 [ip-26-0-154-121:0]:06/21/2023 17:57:13 [INFO|DP=0|PP=0|TP=0]: iteration: 2923 / 150000 | consumed_samples: 187072 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789524E+05 | tokens_per_sec_per_gpu: 9.046131E+03 | global_batch_size: 64 | lm_loss: 1.080440E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 1948680.250 [ip-26-0-154-121:0]:06/21/2023 17:57:14 [INFO|DP=0|PP=0|TP=0]: iteration: 2924 / 150000 | consumed_samples: 187136 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786434E+05 | tokens_per_sec_per_gpu: 9.041304E+03 | global_batch_size: 64 | lm_loss: 1.080552E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1934449.750 [ip-26-0-154-121:0]:06/21/2023 17:57:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2925 / 150000 | consumed_samples: 187200 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789225E+05 | tokens_per_sec_per_gpu: 9.045664E+03 | global_batch_size: 64 | lm_loss: 1.080550E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 1944709.000 [ip-26-0-154-121:0]:06/21/2023 17:57:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2926 / 150000 | consumed_samples: 187264 | elapsed_time_per_iteration_ms: 904.7 | tokens_per_sec: 5.794890E+05 | tokens_per_sec_per_gpu: 9.054515E+03 | global_batch_size: 64 | lm_loss: 1.080554E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.59 | hardware_tflops_per_gpu: 104.59 | grad_norm: 1930281.125 [ip-26-0-154-121:0]:06/21/2023 17:57:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2927 / 150000 | consumed_samples: 187328 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788664E+05 | tokens_per_sec_per_gpu: 9.044788E+03 | global_batch_size: 64 | lm_loss: 1.080551E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1905627.875 [ip-26-0-154-121:0]:06/21/2023 17:57:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2928 / 150000 | consumed_samples: 187392 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788579E+05 | tokens_per_sec_per_gpu: 9.044655E+03 | global_batch_size: 64 | lm_loss: 1.080529E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1879993.000 [ip-26-0-154-121:0]:06/21/2023 17:57:18 [INFO|DP=0|PP=0|TP=0]: iteration: 2929 / 150000 | consumed_samples: 187456 | elapsed_time_per_iteration_ms: 905.2 | tokens_per_sec: 5.792269E+05 | tokens_per_sec_per_gpu: 9.050420E+03 | global_batch_size: 64 | lm_loss: 1.080536E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.54 | hardware_tflops_per_gpu: 104.54 | grad_norm: 1863142.625 [ip-26-0-154-121:0]:06/21/2023 17:57:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2930 / 150000 | consumed_samples: 187520 | elapsed_time_per_iteration_ms: 905.1 | tokens_per_sec: 5.792502E+05 | tokens_per_sec_per_gpu: 9.050785E+03 | global_batch_size: 64 | lm_loss: 1.080536E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.55 | hardware_tflops_per_gpu: 104.55 | grad_norm: 1846614.125 [ip-26-0-154-121:0]:06/21/2023 17:57:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2931 / 150000 | consumed_samples: 187584 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788989E+05 | tokens_per_sec_per_gpu: 9.045295E+03 | global_batch_size: 64 | lm_loss: 1.080512E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1833812.125 [ip-26-0-154-121:0]:06/21/2023 17:57:21 [INFO|DP=0|PP=0|TP=0]: iteration: 2932 / 150000 | consumed_samples: 187648 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787829E+05 | tokens_per_sec_per_gpu: 9.043483E+03 | global_batch_size: 64 | lm_loss: 1.080569E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1823664.625 [ip-26-0-154-121:0]:06/21/2023 17:57:22 [INFO|DP=0|PP=0|TP=0]: iteration: 2933 / 150000 | consumed_samples: 187712 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782340E+05 | tokens_per_sec_per_gpu: 9.034906E+03 | global_batch_size: 64 | lm_loss: 1.080532E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 1814958.625 [ip-26-0-154-121:0]:06/21/2023 17:57:23 [INFO|DP=0|PP=0|TP=0]: iteration: 2934 / 150000 | consumed_samples: 187776 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784707E+05 | tokens_per_sec_per_gpu: 9.038604E+03 | global_batch_size: 64 | lm_loss: 1.080525E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1812269.250 [ip-26-0-154-121:0]:06/21/2023 17:57:24 [INFO|DP=0|PP=0|TP=0]: iteration: 2935 / 150000 | consumed_samples: 187840 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789463E+05 | tokens_per_sec_per_gpu: 9.046036E+03 | global_batch_size: 64 | lm_loss: 1.080527E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 1820554.750 [ip-26-0-154-121:0]:06/21/2023 17:57:25 [INFO|DP=0|PP=0|TP=0]: iteration: 2936 / 150000 | consumed_samples: 187904 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786829E+05 | tokens_per_sec_per_gpu: 9.041920E+03 | global_batch_size: 64 | lm_loss: 1.080296E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1779394.125 [ip-26-0-154-121:0]:06/21/2023 17:57:26 [INFO|DP=0|PP=0|TP=0]: iteration: 2937 / 150000 | consumed_samples: 187968 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786151E+05 | tokens_per_sec_per_gpu: 9.040861E+03 | global_batch_size: 64 | lm_loss: 1.080545E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1775060.750 [ip-26-0-154-121:0]:06/21/2023 17:57:26 [INFO|DP=0|PP=0|TP=0]: iteration: 2938 / 150000 | consumed_samples: 188032 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790170E+05 | tokens_per_sec_per_gpu: 9.047141E+03 | global_batch_size: 64 | lm_loss: 1.080541E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 1774320.500 [ip-26-0-154-121:0]:06/21/2023 17:57:27 [INFO|DP=0|PP=0|TP=0]: iteration: 2939 / 150000 | consumed_samples: 188096 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787909E+05 | tokens_per_sec_per_gpu: 9.043607E+03 | global_batch_size: 64 | lm_loss: 1.080552E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1758871.000 [ip-26-0-154-121:0]:06/21/2023 17:57:28 [INFO|DP=0|PP=0|TP=0]: iteration: 2940 / 150000 | consumed_samples: 188160 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785224E+05 | tokens_per_sec_per_gpu: 9.039413E+03 | global_batch_size: 64 | lm_loss: 1.080539E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1754765.250 [ip-26-0-154-121:0]:06/21/2023 17:57:29 [INFO|DP=0|PP=0|TP=0]: iteration: 2941 / 150000 | consumed_samples: 188224 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784418E+05 | tokens_per_sec_per_gpu: 9.038152E+03 | global_batch_size: 64 | lm_loss: 1.080559E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1733466.750 [ip-26-0-154-121:0]:06/21/2023 17:57:30 [INFO|DP=0|PP=0|TP=0]: iteration: 2942 / 150000 | consumed_samples: 188288 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789047E+05 | tokens_per_sec_per_gpu: 9.045386E+03 | global_batch_size: 64 | lm_loss: 1.080542E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1732274.250 [ip-26-0-154-121:0]:06/21/2023 17:57:31 [INFO|DP=0|PP=0|TP=0]: iteration: 2943 / 150000 | consumed_samples: 188352 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785696E+05 | tokens_per_sec_per_gpu: 9.040150E+03 | global_batch_size: 64 | lm_loss: 1.080530E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1798570.500 [ip-26-0-154-121:0]:06/21/2023 17:57:32 [INFO|DP=0|PP=0|TP=0]: iteration: 2944 / 150000 | consumed_samples: 188416 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787231E+05 | tokens_per_sec_per_gpu: 9.042548E+03 | global_batch_size: 64 | lm_loss: 1.080547E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1700582.125 [ip-26-0-154-121:0]:06/21/2023 17:57:33 [INFO|DP=0|PP=0|TP=0]: iteration: 2945 / 150000 | consumed_samples: 188480 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787441E+05 | tokens_per_sec_per_gpu: 9.042876E+03 | global_batch_size: 64 | lm_loss: 1.080422E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1683967.125 [ip-26-0-154-121:0]:06/21/2023 17:57:34 [INFO|DP=0|PP=0|TP=0]: iteration: 2946 / 150000 | consumed_samples: 188544 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786873E+05 | tokens_per_sec_per_gpu: 9.041989E+03 | global_batch_size: 64 | lm_loss: 1.080538E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1686693.375 [ip-26-0-154-121:0]:06/21/2023 17:57:35 [INFO|DP=0|PP=0|TP=0]: iteration: 2947 / 150000 | consumed_samples: 188608 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784784E+05 | tokens_per_sec_per_gpu: 9.038725E+03 | global_batch_size: 64 | lm_loss: 1.080530E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 1688671.125 [ip-26-0-154-121:0]:06/21/2023 17:57:36 [INFO|DP=0|PP=0|TP=0]: iteration: 2948 / 150000 | consumed_samples: 188672 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783739E+05 | tokens_per_sec_per_gpu: 9.037092E+03 | global_batch_size: 64 | lm_loss: 1.080518E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1672215.250 [ip-26-0-154-121:0]:06/21/2023 17:57:36 [INFO|DP=0|PP=0|TP=0]: iteration: 2949 / 150000 | consumed_samples: 188736 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783885E+05 | tokens_per_sec_per_gpu: 9.037320E+03 | global_batch_size: 64 | lm_loss: 1.080539E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 1653683.250 [ip-26-0-154-121:0]:06/21/2023 17:57:37 [INFO|DP=0|PP=0|TP=0]: iteration: 2950 / 150000 | consumed_samples: 188800 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785827E+05 | tokens_per_sec_per_gpu: 9.040354E+03 | global_batch_size: 64 | lm_loss: 1.080524E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1642597.500 [ip-26-0-154-121:0]:06/21/2023 17:57:38 [INFO|DP=0|PP=0|TP=0]: iteration: 2951 / 150000 | consumed_samples: 188864 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786900E+05 | tokens_per_sec_per_gpu: 9.042032E+03 | global_batch_size: 64 | lm_loss: 1.080530E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1636625.125 [ip-26-0-154-121:0]:06/21/2023 17:57:39 [INFO|DP=0|PP=0|TP=0]: iteration: 2952 / 150000 | consumed_samples: 188928 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787811E+05 | tokens_per_sec_per_gpu: 9.043455E+03 | global_batch_size: 64 | lm_loss: 1.080524E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1623740.750 [ip-26-0-154-121:0]:06/21/2023 17:57:40 [INFO|DP=0|PP=0|TP=0]: iteration: 2953 / 150000 | consumed_samples: 188992 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790621E+05 | tokens_per_sec_per_gpu: 9.047846E+03 | global_batch_size: 64 | lm_loss: 1.080532E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 1606992.250 [ip-26-0-154-121:0]:06/21/2023 17:57:41 [INFO|DP=0|PP=0|TP=0]: iteration: 2954 / 150000 | consumed_samples: 189056 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787566E+05 | tokens_per_sec_per_gpu: 9.043072E+03 | global_batch_size: 64 | lm_loss: 1.080501E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1592139.625 [ip-26-0-154-121:0]:06/21/2023 17:57:42 [INFO|DP=0|PP=0|TP=0]: iteration: 2955 / 150000 | consumed_samples: 189120 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782449E+05 | tokens_per_sec_per_gpu: 9.035077E+03 | global_batch_size: 64 | lm_loss: 1.080521E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 1561639.250 [ip-26-0-154-121:0]:06/21/2023 17:57:43 [INFO|DP=0|PP=0|TP=0]: iteration: 2956 / 150000 | consumed_samples: 189184 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783351E+05 | tokens_per_sec_per_gpu: 9.036486E+03 | global_batch_size: 64 | lm_loss: 1.080542E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 1725995.125 [ip-26-0-154-121:0]:06/21/2023 17:57:44 [INFO|DP=0|PP=0|TP=0]: iteration: 2957 / 150000 | consumed_samples: 189248 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788961E+05 | tokens_per_sec_per_gpu: 9.045252E+03 | global_batch_size: 64 | lm_loss: 1.080515E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1588357.875 [ip-26-0-154-121:0]:06/21/2023 17:57:45 [INFO|DP=0|PP=0|TP=0]: iteration: 2958 / 150000 | consumed_samples: 189312 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786987E+05 | tokens_per_sec_per_gpu: 9.042167E+03 | global_batch_size: 64 | lm_loss: 1.080517E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1556790.750 [ip-26-0-154-121:0]:06/21/2023 17:57:46 [INFO|DP=0|PP=0|TP=0]: iteration: 2959 / 150000 | consumed_samples: 189376 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785414E+05 | tokens_per_sec_per_gpu: 9.039710E+03 | global_batch_size: 64 | lm_loss: 1.080511E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1679397.875 [ip-26-0-154-121:0]:06/21/2023 17:57:46 [INFO|DP=0|PP=0|TP=0]: iteration: 2960 / 150000 | consumed_samples: 189440 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789977E+05 | tokens_per_sec_per_gpu: 9.046838E+03 | global_batch_size: 64 | lm_loss: 1.080496E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 1543225.875 [ip-26-0-154-121:0]:06/21/2023 17:57:47 [INFO|DP=0|PP=0|TP=0]: iteration: 2961 / 150000 | consumed_samples: 189504 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790373E+05 | tokens_per_sec_per_gpu: 9.047458E+03 | global_batch_size: 64 | lm_loss: 1.080499E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 1535653.000 [ip-26-0-154-121:0]:06/21/2023 17:57:48 [INFO|DP=0|PP=0|TP=0]: iteration: 2962 / 150000 | consumed_samples: 189568 | elapsed_time_per_iteration_ms: 905.2 | tokens_per_sec: 5.791779E+05 | tokens_per_sec_per_gpu: 9.049655E+03 | global_batch_size: 64 | lm_loss: 1.080490E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.53 | hardware_tflops_per_gpu: 104.53 | grad_norm: 1530944.500 [ip-26-0-154-121:0]:06/21/2023 17:57:49 [INFO|DP=0|PP=0|TP=0]: iteration: 2963 / 150000 | consumed_samples: 189632 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791059E+05 | tokens_per_sec_per_gpu: 9.048530E+03 | global_batch_size: 64 | lm_loss: 1.080500E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 1516551.625 [ip-26-0-154-121:0]:06/21/2023 17:57:50 [INFO|DP=0|PP=0|TP=0]: iteration: 2964 / 150000 | consumed_samples: 189696 | elapsed_time_per_iteration_ms: 907.0 | tokens_per_sec: 5.780343E+05 | tokens_per_sec_per_gpu: 9.031785E+03 | global_batch_size: 64 | lm_loss: 1.080524E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.33 | hardware_tflops_per_gpu: 104.33 | grad_norm: 1508208.000 [ip-26-0-154-121:0]:06/21/2023 17:57:51 [INFO|DP=0|PP=0|TP=0]: iteration: 2965 / 150000 | consumed_samples: 189760 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786619E+05 | tokens_per_sec_per_gpu: 9.041591E+03 | global_batch_size: 64 | lm_loss: 1.080478E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1561855.625 [ip-26-0-154-121:0]:06/21/2023 17:57:52 [INFO|DP=0|PP=0|TP=0]: iteration: 2966 / 150000 | consumed_samples: 189824 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786108E+05 | tokens_per_sec_per_gpu: 9.040794E+03 | global_batch_size: 64 | lm_loss: 1.080531E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1498414.750 [ip-26-0-154-121:0]:06/21/2023 17:57:53 [INFO|DP=0|PP=0|TP=0]: iteration: 2967 / 150000 | consumed_samples: 189888 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786501E+05 | tokens_per_sec_per_gpu: 9.041408E+03 | global_batch_size: 64 | lm_loss: 1.080647E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1464265.875 [ip-26-0-154-121:0]:06/21/2023 17:57:54 [INFO|DP=0|PP=0|TP=0]: iteration: 2968 / 150000 | consumed_samples: 189952 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785685E+05 | tokens_per_sec_per_gpu: 9.040133E+03 | global_batch_size: 64 | lm_loss: 1.080488E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1445434.000 [ip-26-0-154-121:0]:06/21/2023 17:57:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2969 / 150000 | consumed_samples: 190016 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785769E+05 | tokens_per_sec_per_gpu: 9.040264E+03 | global_batch_size: 64 | lm_loss: 1.080547E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1463848.500 [ip-26-0-154-121:0]:06/21/2023 17:57:55 [INFO|DP=0|PP=0|TP=0]: iteration: 2970 / 150000 | consumed_samples: 190080 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787231E+05 | tokens_per_sec_per_gpu: 9.042548E+03 | global_batch_size: 64 | lm_loss: 1.080538E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1417736.250 [ip-26-0-154-121:0]:06/21/2023 17:57:56 [INFO|DP=0|PP=0|TP=0]: iteration: 2971 / 150000 | consumed_samples: 190144 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788282E+05 | tokens_per_sec_per_gpu: 9.044190E+03 | global_batch_size: 64 | lm_loss: 1.080481E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1415828.125 [ip-26-0-154-121:0]:06/21/2023 17:57:57 [INFO|DP=0|PP=0|TP=0]: iteration: 2972 / 150000 | consumed_samples: 190208 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786189E+05 | tokens_per_sec_per_gpu: 9.040921E+03 | global_batch_size: 64 | lm_loss: 1.080462E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1406927.625 [ip-26-0-154-121:0]:06/21/2023 17:57:58 [INFO|DP=0|PP=0|TP=0]: iteration: 2973 / 150000 | consumed_samples: 190272 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786312E+05 | tokens_per_sec_per_gpu: 9.041113E+03 | global_batch_size: 64 | lm_loss: 1.080559E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1390621.750 [ip-26-0-154-121:0]:06/21/2023 17:57:59 [INFO|DP=0|PP=0|TP=0]: iteration: 2974 / 150000 | consumed_samples: 190336 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786521E+05 | tokens_per_sec_per_gpu: 9.041439E+03 | global_batch_size: 64 | lm_loss: 1.080527E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1379029.250 [ip-26-0-154-121:0]:06/21/2023 17:58:00 [INFO|DP=0|PP=0|TP=0]: iteration: 2975 / 150000 | consumed_samples: 190400 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790535E+05 | tokens_per_sec_per_gpu: 9.047710E+03 | global_batch_size: 64 | lm_loss: 1.080511E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 1356265.000 [ip-26-0-154-121:0]:06/21/2023 17:58:01 [INFO|DP=0|PP=0|TP=0]: iteration: 2976 / 150000 | consumed_samples: 190464 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785994E+05 | tokens_per_sec_per_gpu: 9.040616E+03 | global_batch_size: 64 | lm_loss: 1.080511E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1351459.375 [ip-26-0-154-121:0]:06/21/2023 17:58:02 [INFO|DP=0|PP=0|TP=0]: iteration: 2977 / 150000 | consumed_samples: 190528 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791032E+05 | tokens_per_sec_per_gpu: 9.048487E+03 | global_batch_size: 64 | lm_loss: 1.080243E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 1390419.125 [ip-26-0-154-121:0]:06/21/2023 17:58:03 [INFO|DP=0|PP=0|TP=0]: iteration: 2978 / 150000 | consumed_samples: 190592 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787511E+05 | tokens_per_sec_per_gpu: 9.042986E+03 | global_batch_size: 64 | lm_loss: 1.080620E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 4375720.000 [ip-26-0-154-121:0]:06/21/2023 17:58:04 [INFO|DP=0|PP=0|TP=0]: iteration: 2979 / 150000 | consumed_samples: 190656 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786935E+05 | tokens_per_sec_per_gpu: 9.042086E+03 | global_batch_size: 64 | lm_loss: 1.080523E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1699159.500 [ip-26-0-154-121:0]:06/21/2023 17:58:05 [INFO|DP=0|PP=0|TP=0]: iteration: 2980 / 150000 | consumed_samples: 190720 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787843E+05 | tokens_per_sec_per_gpu: 9.043505E+03 | global_batch_size: 64 | lm_loss: 1.080507E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1352082.000 [ip-26-0-154-121:0]:06/21/2023 17:58:05 [INFO|DP=0|PP=0|TP=0]: iteration: 2981 / 150000 | consumed_samples: 190784 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790458E+05 | tokens_per_sec_per_gpu: 9.047591E+03 | global_batch_size: 64 | lm_loss: 1.080536E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 1301018.125 [ip-26-0-154-121:0]:06/21/2023 17:58:06 [INFO|DP=0|PP=0|TP=0]: iteration: 2982 / 150000 | consumed_samples: 190848 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788605E+05 | tokens_per_sec_per_gpu: 9.044695E+03 | global_batch_size: 64 | lm_loss: 1.080473E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1293008.375 [ip-26-0-154-121:0]:06/21/2023 17:58:07 [INFO|DP=0|PP=0|TP=0]: iteration: 2983 / 150000 | consumed_samples: 190912 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790725E+05 | tokens_per_sec_per_gpu: 9.048008E+03 | global_batch_size: 64 | lm_loss: 1.080566E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 1276707.750 [ip-26-0-154-121:0]:06/21/2023 17:58:08 [INFO|DP=0|PP=0|TP=0]: iteration: 2984 / 150000 | consumed_samples: 190976 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789059E+05 | tokens_per_sec_per_gpu: 9.045405E+03 | global_batch_size: 64 | lm_loss: 1.080559E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 1257429.000 [ip-26-0-154-121:0]:06/21/2023 17:58:09 [INFO|DP=0|PP=0|TP=0]: iteration: 2985 / 150000 | consumed_samples: 191040 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785288E+05 | tokens_per_sec_per_gpu: 9.039512E+03 | global_batch_size: 64 | lm_loss: 1.080534E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1238899.750 [ip-26-0-154-121:0]:06/21/2023 17:58:10 [INFO|DP=0|PP=0|TP=0]: iteration: 2986 / 150000 | consumed_samples: 191104 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787557E+05 | tokens_per_sec_per_gpu: 9.043057E+03 | global_batch_size: 64 | lm_loss: 1.080558E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1235471.375 [ip-26-0-154-121:0]:06/21/2023 17:58:11 [INFO|DP=0|PP=0|TP=0]: iteration: 2987 / 150000 | consumed_samples: 191168 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785841E+05 | tokens_per_sec_per_gpu: 9.040376E+03 | global_batch_size: 64 | lm_loss: 1.080550E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1254388.500 [ip-26-0-154-121:0]:06/21/2023 17:58:12 [INFO|DP=0|PP=0|TP=0]: iteration: 2988 / 150000 | consumed_samples: 191232 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786442E+05 | tokens_per_sec_per_gpu: 9.041315E+03 | global_batch_size: 64 | lm_loss: 1.080548E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 1200538.625 [ip-26-0-154-121:0]:06/21/2023 17:58:13 [INFO|DP=0|PP=0|TP=0]: iteration: 2989 / 150000 | consumed_samples: 191296 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788328E+05 | tokens_per_sec_per_gpu: 9.044262E+03 | global_batch_size: 64 | lm_loss: 1.080549E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 1235412.625 [ip-26-0-154-121:0]:06/21/2023 17:58:14 [INFO|DP=0|PP=0|TP=0]: iteration: 2990 / 150000 | consumed_samples: 191360 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791273E+05 | tokens_per_sec_per_gpu: 9.048864E+03 | global_batch_size: 64 | lm_loss: 1.080536E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 1185737.750 [ip-26-0-154-121:0]:06/21/2023 17:58:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2991 / 150000 | consumed_samples: 191424 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787037E+05 | tokens_per_sec_per_gpu: 9.042246E+03 | global_batch_size: 64 | lm_loss: 1.080521E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 1164033.625 [ip-26-0-154-121:0]:06/21/2023 17:58:15 [INFO|DP=0|PP=0|TP=0]: iteration: 2992 / 150000 | consumed_samples: 191488 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786296E+05 | tokens_per_sec_per_gpu: 9.041087E+03 | global_batch_size: 64 | lm_loss: 1.080541E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 1199676.125 [ip-26-0-154-121:0]:06/21/2023 17:58:16 [INFO|DP=0|PP=0|TP=0]: iteration: 2993 / 150000 | consumed_samples: 191552 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784538E+05 | tokens_per_sec_per_gpu: 9.038340E+03 | global_batch_size: 64 | lm_loss: 1.080572E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 1125323.000 [ip-26-0-154-121:0]:06/21/2023 17:58:17 [INFO|DP=0|PP=0|TP=0]: iteration: 2994 / 150000 | consumed_samples: 191616 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782703E+05 | tokens_per_sec_per_gpu: 9.035474E+03 | global_batch_size: 64 | lm_loss: 1.080590E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 1070832.750 [ip-26-0-154-121:0]:06/21/2023 17:58:18 [INFO|DP=0|PP=0|TP=0]: iteration: 2995 / 150000 | consumed_samples: 191680 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785408E+05 | tokens_per_sec_per_gpu: 9.039700E+03 | global_batch_size: 64 | lm_loss: 1.080588E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 1051258.750 [ip-26-0-154-121:0]:06/21/2023 17:58:19 [INFO|DP=0|PP=0|TP=0]: iteration: 2996 / 150000 | consumed_samples: 191744 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789673E+05 | tokens_per_sec_per_gpu: 9.046364E+03 | global_batch_size: 64 | lm_loss: 1.080592E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 1037240.188 [ip-26-0-154-121:0]:06/21/2023 17:58:20 [INFO|DP=0|PP=0|TP=0]: iteration: 2997 / 150000 | consumed_samples: 191808 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791491E+05 | tokens_per_sec_per_gpu: 9.049204E+03 | global_batch_size: 64 | lm_loss: 1.080595E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.53 | hardware_tflops_per_gpu: 104.53 | grad_norm: 1016172.125 [ip-26-0-154-121:0]:06/21/2023 17:58:21 [INFO|DP=0|PP=0|TP=0]: iteration: 2998 / 150000 | consumed_samples: 191872 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787650E+05 | tokens_per_sec_per_gpu: 9.043203E+03 | global_batch_size: 64 | lm_loss: 1.080610E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 997645.000 [ip-26-0-154-121:0]:06/21/2023 17:58:22 [INFO|DP=0|PP=0|TP=0]: iteration: 2999 / 150000 | consumed_samples: 191936 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787162E+05 | tokens_per_sec_per_gpu: 9.042441E+03 | global_batch_size: 64 | lm_loss: 1.080580E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 976598.062 [ip-26-0-154-121:0]:06/21/2023 17:58:23 [INFO|DP=0|PP=0|TP=0]: iteration: 3000 / 150000 | consumed_samples: 192000 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786110E+05 | tokens_per_sec_per_gpu: 9.040797E+03 | global_batch_size: 64 | lm_loss: 1.080617E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 943027.125 [ip-26-0-154-121:0]:06/21/2023 17:58:24 [INFO|DP=0|PP=0|TP=0]: iteration: 3001 / 150000 | consumed_samples: 192064 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785314E+05 | tokens_per_sec_per_gpu: 9.039553E+03 | global_batch_size: 64 | lm_loss: 1.080610E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 936862.750 [ip-26-0-154-121:0]:06/21/2023 17:58:25 [INFO|DP=0|PP=0|TP=0]: iteration: 3002 / 150000 | consumed_samples: 192128 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787711E+05 | tokens_per_sec_per_gpu: 9.043298E+03 | global_batch_size: 64 | lm_loss: 1.080622E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 924799.688 [ip-26-0-154-121:0]:06/21/2023 17:58:25 [INFO|DP=0|PP=0|TP=0]: iteration: 3003 / 150000 | consumed_samples: 192192 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788166E+05 | tokens_per_sec_per_gpu: 9.044009E+03 | global_batch_size: 64 | lm_loss: 1.080602E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 899571.375 [ip-26-0-154-121:0]:06/21/2023 17:58:26 [INFO|DP=0|PP=0|TP=0]: iteration: 3004 / 150000 | consumed_samples: 192256 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786309E+05 | tokens_per_sec_per_gpu: 9.041109E+03 | global_batch_size: 64 | lm_loss: 1.080619E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 869886.062 [ip-26-0-154-121:0]:06/21/2023 17:58:27 [INFO|DP=0|PP=0|TP=0]: iteration: 3005 / 150000 | consumed_samples: 192320 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789916E+05 | tokens_per_sec_per_gpu: 9.046743E+03 | global_batch_size: 64 | lm_loss: 1.080595E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 868872.312 [ip-26-0-154-121:0]:06/21/2023 17:58:28 [INFO|DP=0|PP=0|TP=0]: iteration: 3006 / 150000 | consumed_samples: 192384 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786506E+05 | tokens_per_sec_per_gpu: 9.041415E+03 | global_batch_size: 64 | lm_loss: 1.080606E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 848813.312 [ip-26-0-154-121:0]:06/21/2023 17:58:29 [INFO|DP=0|PP=0|TP=0]: iteration: 3007 / 150000 | consumed_samples: 192448 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.782046E+05 | tokens_per_sec_per_gpu: 9.034447E+03 | global_batch_size: 64 | lm_loss: 1.080761E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 832512.938 [ip-26-0-154-121:0]:06/21/2023 17:58:30 [INFO|DP=0|PP=0|TP=0]: iteration: 3008 / 150000 | consumed_samples: 192512 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787759E+05 | tokens_per_sec_per_gpu: 9.043374E+03 | global_batch_size: 64 | lm_loss: 1.080624E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 804547.438 [ip-26-0-154-121:0]:06/21/2023 17:58:31 [INFO|DP=0|PP=0|TP=0]: iteration: 3009 / 150000 | consumed_samples: 192576 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783027E+05 | tokens_per_sec_per_gpu: 9.035980E+03 | global_batch_size: 64 | lm_loss: 1.080632E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 1489724.125 [ip-26-0-154-121:0]:06/21/2023 17:58:32 [INFO|DP=0|PP=0|TP=0]: iteration: 3010 / 150000 | consumed_samples: 192640 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790103E+05 | tokens_per_sec_per_gpu: 9.047036E+03 | global_batch_size: 64 | lm_loss: 1.080657E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 786034.750 [ip-26-0-154-121:0]:06/21/2023 17:58:33 [INFO|DP=0|PP=0|TP=0]: iteration: 3011 / 150000 | consumed_samples: 192704 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785623E+05 | tokens_per_sec_per_gpu: 9.040036E+03 | global_batch_size: 64 | lm_loss: 1.080636E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 748452.062 [ip-26-0-154-121:0]:06/21/2023 17:58:34 [INFO|DP=0|PP=0|TP=0]: iteration: 3012 / 150000 | consumed_samples: 192768 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785029E+05 | tokens_per_sec_per_gpu: 9.039108E+03 | global_batch_size: 64 | lm_loss: 1.080622E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 715007.062 [ip-26-0-154-121:0]:06/21/2023 17:58:34 [INFO|DP=0|PP=0|TP=0]: iteration: 3013 / 150000 | consumed_samples: 192832 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784885E+05 | tokens_per_sec_per_gpu: 9.038882E+03 | global_batch_size: 64 | lm_loss: 1.080635E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 715953.125 [ip-26-0-154-121:0]:06/21/2023 17:58:35 [INFO|DP=0|PP=0|TP=0]: iteration: 3014 / 150000 | consumed_samples: 192896 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787718E+05 | tokens_per_sec_per_gpu: 9.043310E+03 | global_batch_size: 64 | lm_loss: 1.080638E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 678807.625 [ip-26-0-154-121:0]:06/21/2023 17:58:36 [INFO|DP=0|PP=0|TP=0]: iteration: 3015 / 150000 | consumed_samples: 192960 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784675E+05 | tokens_per_sec_per_gpu: 9.038554E+03 | global_batch_size: 64 | lm_loss: 1.080626E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 685421.562 [ip-26-0-154-121:0]:06/21/2023 17:58:37 [INFO|DP=0|PP=0|TP=0]: iteration: 3016 / 150000 | consumed_samples: 193024 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787351E+05 | tokens_per_sec_per_gpu: 9.042736E+03 | global_batch_size: 64 | lm_loss: 1.080653E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 664479.000 [ip-26-0-154-121:0]:06/21/2023 17:58:38 [INFO|DP=0|PP=0|TP=0]: iteration: 3017 / 150000 | consumed_samples: 193088 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787724E+05 | tokens_per_sec_per_gpu: 9.043319E+03 | global_batch_size: 64 | lm_loss: 1.080645E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 659441.125 [ip-26-0-154-121:0]:06/21/2023 17:58:39 [INFO|DP=0|PP=0|TP=0]: iteration: 3018 / 150000 | consumed_samples: 193152 | elapsed_time_per_iteration_ms: 904.9 | tokens_per_sec: 5.794053E+05 | tokens_per_sec_per_gpu: 9.053208E+03 | global_batch_size: 64 | lm_loss: 1.080653E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.57 | hardware_tflops_per_gpu: 104.57 | grad_norm: 616729.250 [ip-26-0-154-121:0]:06/21/2023 17:58:40 [INFO|DP=0|PP=0|TP=0]: iteration: 3019 / 150000 | consumed_samples: 193216 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789393E+05 | tokens_per_sec_per_gpu: 9.045926E+03 | global_batch_size: 64 | lm_loss: 1.080645E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 618951.250 [ip-26-0-154-121:0]:06/21/2023 17:58:41 [INFO|DP=0|PP=0|TP=0]: iteration: 3020 / 150000 | consumed_samples: 193280 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789189E+05 | tokens_per_sec_per_gpu: 9.045607E+03 | global_batch_size: 64 | lm_loss: 1.080642E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 625075.375 [ip-26-0-154-121:0]:06/21/2023 17:58:42 [INFO|DP=0|PP=0|TP=0]: iteration: 3021 / 150000 | consumed_samples: 193344 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785142E+05 | tokens_per_sec_per_gpu: 9.039284E+03 | global_batch_size: 64 | lm_loss: 1.080641E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 587386.625 [ip-26-0-154-121:0]:06/21/2023 17:58:43 [INFO|DP=0|PP=0|TP=0]: iteration: 3022 / 150000 | consumed_samples: 193408 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788637E+05 | tokens_per_sec_per_gpu: 9.044745E+03 | global_batch_size: 64 | lm_loss: 1.080658E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 560293.625 [ip-26-0-154-121:0]:06/21/2023 17:58:44 [INFO|DP=0|PP=0|TP=0]: iteration: 3023 / 150000 | consumed_samples: 193472 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787395E+05 | tokens_per_sec_per_gpu: 9.042805E+03 | global_batch_size: 64 | lm_loss: 1.080636E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 588867.812 [ip-26-0-154-121:0]:06/21/2023 17:58:44 [INFO|DP=0|PP=0|TP=0]: iteration: 3024 / 150000 | consumed_samples: 193536 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785690E+05 | tokens_per_sec_per_gpu: 9.040140E+03 | global_batch_size: 64 | lm_loss: 1.080638E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 578664.938 [ip-26-0-154-121:0]:06/21/2023 17:58:45 [INFO|DP=0|PP=0|TP=0]: iteration: 3025 / 150000 | consumed_samples: 193600 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787739E+05 | tokens_per_sec_per_gpu: 9.043343E+03 | global_batch_size: 64 | lm_loss: 1.080646E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 538414.188 [ip-26-0-154-121:0]:06/21/2023 17:58:46 [INFO|DP=0|PP=0|TP=0]: iteration: 3026 / 150000 | consumed_samples: 193664 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786545E+05 | tokens_per_sec_per_gpu: 9.041477E+03 | global_batch_size: 64 | lm_loss: 1.080654E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 537844.375 [ip-26-0-154-121:0]:06/21/2023 17:58:47 [INFO|DP=0|PP=0|TP=0]: iteration: 3027 / 150000 | consumed_samples: 193728 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788443E+05 | tokens_per_sec_per_gpu: 9.044443E+03 | global_batch_size: 64 | lm_loss: 1.080639E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 534278.062 [ip-26-0-154-121:0]:06/21/2023 17:58:48 [INFO|DP=0|PP=0|TP=0]: iteration: 3028 / 150000 | consumed_samples: 193792 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785764E+05 | tokens_per_sec_per_gpu: 9.040257E+03 | global_batch_size: 64 | lm_loss: 1.080635E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 520370.000 [ip-26-0-154-121:0]:06/21/2023 17:58:49 [INFO|DP=0|PP=0|TP=0]: iteration: 3029 / 150000 | consumed_samples: 193856 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786239E+05 | tokens_per_sec_per_gpu: 9.040999E+03 | global_batch_size: 64 | lm_loss: 1.080622E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 510850.250 [ip-26-0-154-121:0]:06/21/2023 17:58:50 [INFO|DP=0|PP=0|TP=0]: iteration: 3030 / 150000 | consumed_samples: 193920 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787318E+05 | tokens_per_sec_per_gpu: 9.042684E+03 | global_batch_size: 64 | lm_loss: 1.080633E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 482001.312 [ip-26-0-154-121:0]:06/21/2023 17:58:51 [INFO|DP=0|PP=0|TP=0]: iteration: 3031 / 150000 | consumed_samples: 193984 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788748E+05 | tokens_per_sec_per_gpu: 9.044919E+03 | global_batch_size: 64 | lm_loss: 1.080634E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 481539.188 [ip-26-0-154-121:0]:06/21/2023 17:58:52 [INFO|DP=0|PP=0|TP=0]: iteration: 3032 / 150000 | consumed_samples: 194048 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784629E+05 | tokens_per_sec_per_gpu: 9.038483E+03 | global_batch_size: 64 | lm_loss: 1.080626E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 477309.156 [ip-26-0-154-121:0]:06/21/2023 17:58:53 [INFO|DP=0|PP=0|TP=0]: iteration: 3033 / 150000 | consumed_samples: 194112 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787247E+05 | tokens_per_sec_per_gpu: 9.042574E+03 | global_batch_size: 64 | lm_loss: 1.080619E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 470756.969 [ip-26-0-154-121:0]:06/21/2023 17:58:54 [INFO|DP=0|PP=0|TP=0]: iteration: 3034 / 150000 | consumed_samples: 194176 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784052E+05 | tokens_per_sec_per_gpu: 9.037582E+03 | global_batch_size: 64 | lm_loss: 1.080615E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 467210.906 [ip-26-0-154-121:0]:06/21/2023 17:58:54 [INFO|DP=0|PP=0|TP=0]: iteration: 3035 / 150000 | consumed_samples: 194240 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787481E+05 | tokens_per_sec_per_gpu: 9.042938E+03 | global_batch_size: 64 | lm_loss: 1.080628E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 438022.250 [ip-26-0-154-121:0]:06/21/2023 17:58:55 [INFO|DP=0|PP=0|TP=0]: iteration: 3036 / 150000 | consumed_samples: 194304 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786742E+05 | tokens_per_sec_per_gpu: 9.041784E+03 | global_batch_size: 64 | lm_loss: 1.080594E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 439463.062 [ip-26-0-154-121:0]:06/21/2023 17:58:56 [INFO|DP=0|PP=0|TP=0]: iteration: 3037 / 150000 | consumed_samples: 194368 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784311E+05 | tokens_per_sec_per_gpu: 9.037986E+03 | global_batch_size: 64 | lm_loss: 1.080608E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 431125.125 [ip-26-0-154-121:0]:06/21/2023 17:58:57 [INFO|DP=0|PP=0|TP=0]: iteration: 3038 / 150000 | consumed_samples: 194432 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785676E+05 | tokens_per_sec_per_gpu: 9.040119E+03 | global_batch_size: 64 | lm_loss: 1.080613E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 444936.406 [ip-26-0-154-121:0]:06/21/2023 17:58:58 [INFO|DP=0|PP=0|TP=0]: iteration: 3039 / 150000 | consumed_samples: 194496 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784194E+05 | tokens_per_sec_per_gpu: 9.037803E+03 | global_batch_size: 64 | lm_loss: 1.080616E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 607313.688 [ip-26-0-154-121:0]:06/21/2023 17:58:59 [INFO|DP=0|PP=0|TP=0]: iteration: 3040 / 150000 | consumed_samples: 194560 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785623E+05 | tokens_per_sec_per_gpu: 9.040036E+03 | global_batch_size: 64 | lm_loss: 1.080605E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 437762.562 [ip-26-0-154-121:0]:06/21/2023 17:59:00 [INFO|DP=0|PP=0|TP=0]: iteration: 3041 / 150000 | consumed_samples: 194624 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781941E+05 | tokens_per_sec_per_gpu: 9.034284E+03 | global_batch_size: 64 | lm_loss: 1.080591E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 422837.375 [ip-26-0-154-121:0]:06/21/2023 17:59:01 [INFO|DP=0|PP=0|TP=0]: iteration: 3042 / 150000 | consumed_samples: 194688 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787438E+05 | tokens_per_sec_per_gpu: 9.042872E+03 | global_batch_size: 64 | lm_loss: 1.080619E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 419854.219 [ip-26-0-154-121:0]:06/21/2023 17:59:02 [INFO|DP=0|PP=0|TP=0]: iteration: 3043 / 150000 | consumed_samples: 194752 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785108E+05 | tokens_per_sec_per_gpu: 9.039232E+03 | global_batch_size: 64 | lm_loss: 1.080580E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 413048.719 [ip-26-0-154-121:0]:06/21/2023 17:59:03 [INFO|DP=0|PP=0|TP=0]: iteration: 3044 / 150000 | consumed_samples: 194816 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786118E+05 | tokens_per_sec_per_gpu: 9.040809E+03 | global_batch_size: 64 | lm_loss: 1.080588E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 400595.250 [ip-26-0-154-121:0]:06/21/2023 17:59:04 [INFO|DP=0|PP=0|TP=0]: iteration: 3045 / 150000 | consumed_samples: 194880 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787991E+05 | tokens_per_sec_per_gpu: 9.043736E+03 | global_batch_size: 64 | lm_loss: 1.080581E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 396420.375 [ip-26-0-154-121:0]:06/21/2023 17:59:04 [INFO|DP=0|PP=0|TP=0]: iteration: 3046 / 150000 | consumed_samples: 194944 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787319E+05 | tokens_per_sec_per_gpu: 9.042686E+03 | global_batch_size: 64 | lm_loss: 1.080589E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 383284.031 [ip-26-0-154-121:0]:06/21/2023 17:59:05 [INFO|DP=0|PP=0|TP=0]: iteration: 3047 / 150000 | consumed_samples: 195008 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788370E+05 | tokens_per_sec_per_gpu: 9.044328E+03 | global_batch_size: 64 | lm_loss: 1.080575E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 384377.188 [ip-26-0-154-121:0]:06/21/2023 17:59:06 [INFO|DP=0|PP=0|TP=0]: iteration: 3048 / 150000 | consumed_samples: 195072 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788111E+05 | tokens_per_sec_per_gpu: 9.043924E+03 | global_batch_size: 64 | lm_loss: 1.080564E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 390249.844 [ip-26-0-154-121:0]:06/21/2023 17:59:07 [INFO|DP=0|PP=0|TP=0]: iteration: 3049 / 150000 | consumed_samples: 195136 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787566E+05 | tokens_per_sec_per_gpu: 9.043072E+03 | global_batch_size: 64 | lm_loss: 1.080609E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 385288.906 [ip-26-0-154-121:0]:06/21/2023 17:59:08 [INFO|DP=0|PP=0|TP=0]: iteration: 3050 / 150000 | consumed_samples: 195200 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788606E+05 | tokens_per_sec_per_gpu: 9.044697E+03 | global_batch_size: 64 | lm_loss: 1.080573E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 376508.938 [ip-26-0-154-121:0]:06/21/2023 17:59:09 [INFO|DP=0|PP=0|TP=0]: iteration: 3051 / 150000 | consumed_samples: 195264 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788836E+05 | tokens_per_sec_per_gpu: 9.045057E+03 | global_batch_size: 64 | lm_loss: 1.080563E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 406686.469 [ip-26-0-154-121:0]:06/21/2023 17:59:10 [INFO|DP=0|PP=0|TP=0]: iteration: 3052 / 150000 | consumed_samples: 195328 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788260E+05 | tokens_per_sec_per_gpu: 9.044157E+03 | global_batch_size: 64 | lm_loss: 1.080561E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 466576.531 [ip-26-0-154-121:0]:06/21/2023 17:59:11 [INFO|DP=0|PP=0|TP=0]: iteration: 3053 / 150000 | consumed_samples: 195392 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788247E+05 | tokens_per_sec_per_gpu: 9.044136E+03 | global_batch_size: 64 | lm_loss: 1.080568E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 555463.438 [ip-26-0-154-121:0]:06/21/2023 17:59:12 [INFO|DP=0|PP=0|TP=0]: iteration: 3054 / 150000 | consumed_samples: 195456 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787752E+05 | tokens_per_sec_per_gpu: 9.043362E+03 | global_batch_size: 64 | lm_loss: 1.080580E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 443833.938 [ip-26-0-154-121:0]:06/21/2023 17:59:13 [INFO|DP=0|PP=0|TP=0]: iteration: 3055 / 150000 | consumed_samples: 195520 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789769E+05 | tokens_per_sec_per_gpu: 9.046514E+03 | global_batch_size: 64 | lm_loss: 1.080547E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 459948.156 [ip-26-0-154-121:0]:06/21/2023 17:59:14 [INFO|DP=0|PP=0|TP=0]: iteration: 3056 / 150000 | consumed_samples: 195584 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785422E+05 | tokens_per_sec_per_gpu: 9.039722E+03 | global_batch_size: 64 | lm_loss: 1.080553E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 429703.406 [ip-26-0-154-121:0]:06/21/2023 17:59:14 [INFO|DP=0|PP=0|TP=0]: iteration: 3057 / 150000 | consumed_samples: 195648 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787787E+05 | tokens_per_sec_per_gpu: 9.043417E+03 | global_batch_size: 64 | lm_loss: 1.080564E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 451604.469 [ip-26-0-154-121:0]:06/21/2023 17:59:15 [INFO|DP=0|PP=0|TP=0]: iteration: 3058 / 150000 | consumed_samples: 195712 | elapsed_time_per_iteration_ms: 905.2 | tokens_per_sec: 5.792225E+05 | tokens_per_sec_per_gpu: 9.050351E+03 | global_batch_size: 64 | lm_loss: 1.080568E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.54 | hardware_tflops_per_gpu: 104.54 | grad_norm: 394694.188 [ip-26-0-154-121:0]:06/21/2023 17:59:16 [INFO|DP=0|PP=0|TP=0]: iteration: 3059 / 150000 | consumed_samples: 195776 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784859E+05 | tokens_per_sec_per_gpu: 9.038842E+03 | global_batch_size: 64 | lm_loss: 1.080560E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 383299.875 [ip-26-0-154-121:0]:06/21/2023 17:59:17 [INFO|DP=0|PP=0|TP=0]: iteration: 3060 / 150000 | consumed_samples: 195840 | elapsed_time_per_iteration_ms: 905.0 | tokens_per_sec: 5.793036E+05 | tokens_per_sec_per_gpu: 9.051619E+03 | global_batch_size: 64 | lm_loss: 1.080544E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.56 | hardware_tflops_per_gpu: 104.56 | grad_norm: 373718.656 [ip-26-0-154-121:0]:06/21/2023 17:59:18 [INFO|DP=0|PP=0|TP=0]: iteration: 3061 / 150000 | consumed_samples: 195904 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787904E+05 | tokens_per_sec_per_gpu: 9.043600E+03 | global_batch_size: 64 | lm_loss: 1.080542E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 363000.562 [ip-26-0-154-121:0]:06/21/2023 17:59:19 [INFO|DP=0|PP=0|TP=0]: iteration: 3062 / 150000 | consumed_samples: 195968 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787877E+05 | tokens_per_sec_per_gpu: 9.043557E+03 | global_batch_size: 64 | lm_loss: 1.080534E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 353669.688 [ip-26-0-154-121:0]:06/21/2023 17:59:20 [INFO|DP=0|PP=0|TP=0]: iteration: 3063 / 150000 | consumed_samples: 196032 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788427E+05 | tokens_per_sec_per_gpu: 9.044417E+03 | global_batch_size: 64 | lm_loss: 1.080489E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 350291.469 [ip-26-0-154-121:0]:06/21/2023 17:59:21 [INFO|DP=0|PP=0|TP=0]: iteration: 3064 / 150000 | consumed_samples: 196096 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788229E+05 | tokens_per_sec_per_gpu: 9.044107E+03 | global_batch_size: 64 | lm_loss: 1.080545E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 375316.250 [ip-26-0-154-121:0]:06/21/2023 17:59:22 [INFO|DP=0|PP=0|TP=0]: iteration: 3065 / 150000 | consumed_samples: 196160 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785023E+05 | tokens_per_sec_per_gpu: 9.039099E+03 | global_batch_size: 64 | lm_loss: 1.080535E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 360325.625 [ip-26-0-154-121:0]:06/21/2023 17:59:23 [INFO|DP=0|PP=0|TP=0]: iteration: 3066 / 150000 | consumed_samples: 196224 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783922E+05 | tokens_per_sec_per_gpu: 9.037377E+03 | global_batch_size: 64 | lm_loss: 1.080529E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 350773.719 [ip-26-0-154-121:0]:06/21/2023 17:59:23 [INFO|DP=0|PP=0|TP=0]: iteration: 3067 / 150000 | consumed_samples: 196288 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785188E+05 | tokens_per_sec_per_gpu: 9.039356E+03 | global_batch_size: 64 | lm_loss: 1.080534E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 375488.656 [ip-26-0-154-121:0]:06/21/2023 17:59:24 [INFO|DP=0|PP=0|TP=0]: iteration: 3068 / 150000 | consumed_samples: 196352 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784871E+05 | tokens_per_sec_per_gpu: 9.038861E+03 | global_batch_size: 64 | lm_loss: 1.080527E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 337663.781 [ip-26-0-154-121:0]:06/21/2023 17:59:25 [INFO|DP=0|PP=0|TP=0]: iteration: 3069 / 150000 | consumed_samples: 196416 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781793E+05 | tokens_per_sec_per_gpu: 9.034051E+03 | global_batch_size: 64 | lm_loss: 1.080524E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 345625.688 [ip-26-0-154-121:0]:06/21/2023 17:59:26 [INFO|DP=0|PP=0|TP=0]: iteration: 3070 / 150000 | consumed_samples: 196480 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783149E+05 | tokens_per_sec_per_gpu: 9.036170E+03 | global_batch_size: 64 | lm_loss: 1.080559E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 360121.375 [ip-26-0-154-121:0]:06/21/2023 17:59:27 [INFO|DP=0|PP=0|TP=0]: iteration: 3071 / 150000 | consumed_samples: 196544 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785433E+05 | tokens_per_sec_per_gpu: 9.039738E+03 | global_batch_size: 64 | lm_loss: 1.080526E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 345489.344 [ip-26-0-154-121:0]:06/21/2023 17:59:28 [INFO|DP=0|PP=0|TP=0]: iteration: 3072 / 150000 | consumed_samples: 196608 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784407E+05 | tokens_per_sec_per_gpu: 9.038136E+03 | global_batch_size: 64 | lm_loss: 1.080519E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 346932.906 [ip-26-0-154-121:0]:06/21/2023 17:59:29 [INFO|DP=0|PP=0|TP=0]: iteration: 3073 / 150000 | consumed_samples: 196672 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787811E+05 | tokens_per_sec_per_gpu: 9.043455E+03 | global_batch_size: 64 | lm_loss: 1.080516E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 343703.562 [ip-26-0-154-121:0]:06/21/2023 17:59:30 [INFO|DP=0|PP=0|TP=0]: iteration: 3074 / 150000 | consumed_samples: 196736 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787846E+05 | tokens_per_sec_per_gpu: 9.043510E+03 | global_batch_size: 64 | lm_loss: 1.080481E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 362919.469 [ip-26-0-154-121:0]:06/21/2023 17:59:31 [INFO|DP=0|PP=0|TP=0]: iteration: 3075 / 150000 | consumed_samples: 196800 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782303E+05 | tokens_per_sec_per_gpu: 9.034849E+03 | global_batch_size: 64 | lm_loss: 1.080517E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 342172.156 [ip-26-0-154-121:0]:06/21/2023 17:59:32 [INFO|DP=0|PP=0|TP=0]: iteration: 3076 / 150000 | consumed_samples: 196864 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787814E+05 | tokens_per_sec_per_gpu: 9.043460E+03 | global_batch_size: 64 | lm_loss: 1.080506E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 329397.188 [ip-26-0-154-121:0]:06/21/2023 17:59:33 [INFO|DP=0|PP=0|TP=0]: iteration: 3077 / 150000 | consumed_samples: 196928 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790922E+05 | tokens_per_sec_per_gpu: 9.048315E+03 | global_batch_size: 64 | lm_loss: 1.080499E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 326684.031 [ip-26-0-154-121:0]:06/21/2023 17:59:33 [INFO|DP=0|PP=0|TP=0]: iteration: 3078 / 150000 | consumed_samples: 196992 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.783975E+05 | tokens_per_sec_per_gpu: 9.037461E+03 | global_batch_size: 64 | lm_loss: 1.080510E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 341284.812 [ip-26-0-154-121:0]:06/21/2023 17:59:34 [INFO|DP=0|PP=0|TP=0]: iteration: 3079 / 150000 | consumed_samples: 197056 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784179E+05 | tokens_per_sec_per_gpu: 9.037779E+03 | global_batch_size: 64 | lm_loss: 1.080496E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 326366.688 [ip-26-0-154-121:0]:06/21/2023 17:59:35 [INFO|DP=0|PP=0|TP=0]: iteration: 3080 / 150000 | consumed_samples: 197120 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785017E+05 | tokens_per_sec_per_gpu: 9.039089E+03 | global_batch_size: 64 | lm_loss: 1.080513E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 318548.031 [ip-26-0-154-121:0]:06/21/2023 17:59:36 [INFO|DP=0|PP=0|TP=0]: iteration: 3081 / 150000 | consumed_samples: 197184 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789670E+05 | tokens_per_sec_per_gpu: 9.046360E+03 | global_batch_size: 64 | lm_loss: 1.080502E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 397768.312 [ip-26-0-154-121:0]:06/21/2023 17:59:37 [INFO|DP=0|PP=0|TP=0]: iteration: 3082 / 150000 | consumed_samples: 197248 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781827E+05 | tokens_per_sec_per_gpu: 9.034105E+03 | global_batch_size: 64 | lm_loss: 1.080488E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 355706.500 [ip-26-0-154-121:0]:06/21/2023 17:59:38 [INFO|DP=0|PP=0|TP=0]: iteration: 3083 / 150000 | consumed_samples: 197312 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786276E+05 | tokens_per_sec_per_gpu: 9.041056E+03 | global_batch_size: 64 | lm_loss: 1.080264E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 306661.781 [ip-26-0-154-121:0]:06/21/2023 17:59:39 [INFO|DP=0|PP=0|TP=0]: iteration: 3084 / 150000 | consumed_samples: 197376 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779747E+05 | tokens_per_sec_per_gpu: 9.030855E+03 | global_batch_size: 64 | lm_loss: 1.080494E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.32 | hardware_tflops_per_gpu: 104.32 | grad_norm: 304513.031 [ip-26-0-154-121:0]:06/21/2023 17:59:40 [INFO|DP=0|PP=0|TP=0]: iteration: 3085 / 150000 | consumed_samples: 197440 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778001E+05 | tokens_per_sec_per_gpu: 9.028126E+03 | global_batch_size: 64 | lm_loss: 1.080484E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.28 | hardware_tflops_per_gpu: 104.28 | grad_norm: 312028.406 [ip-26-0-154-121:0]:06/21/2023 17:59:41 [INFO|DP=0|PP=0|TP=0]: iteration: 3086 / 150000 | consumed_samples: 197504 | elapsed_time_per_iteration_ms: 907.4 | tokens_per_sec: 5.778046E+05 | tokens_per_sec_per_gpu: 9.028197E+03 | global_batch_size: 64 | lm_loss: 1.080481E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 304295.812 [ip-26-0-154-121:0]:06/21/2023 17:59:42 [INFO|DP=0|PP=0|TP=0]: iteration: 3087 / 150000 | consumed_samples: 197568 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781385E+05 | tokens_per_sec_per_gpu: 9.033414E+03 | global_batch_size: 64 | lm_loss: 1.080490E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 314258.562 [ip-26-0-154-121:0]:06/21/2023 17:59:43 [INFO|DP=0|PP=0|TP=0]: iteration: 3088 / 150000 | consumed_samples: 197632 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787945E+05 | tokens_per_sec_per_gpu: 9.043664E+03 | global_batch_size: 64 | lm_loss: 1.080487E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 299788.156 [ip-26-0-154-121:0]:06/21/2023 17:59:43 [INFO|DP=0|PP=0|TP=0]: iteration: 3089 / 150000 | consumed_samples: 197696 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786588E+05 | tokens_per_sec_per_gpu: 9.041544E+03 | global_batch_size: 64 | lm_loss: 1.080474E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 293202.562 [ip-26-0-154-121:0]:06/21/2023 17:59:44 [INFO|DP=0|PP=0|TP=0]: iteration: 3090 / 150000 | consumed_samples: 197760 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781008E+05 | tokens_per_sec_per_gpu: 9.032825E+03 | global_batch_size: 64 | lm_loss: 1.080481E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 302539.812 [ip-26-0-154-121:0]:06/21/2023 17:59:45 [INFO|DP=0|PP=0|TP=0]: iteration: 3091 / 150000 | consumed_samples: 197824 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781499E+05 | tokens_per_sec_per_gpu: 9.033592E+03 | global_batch_size: 64 | lm_loss: 1.080476E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 317580.500 [ip-26-0-154-121:0]:06/21/2023 17:59:46 [INFO|DP=0|PP=0|TP=0]: iteration: 3092 / 150000 | consumed_samples: 197888 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785678E+05 | tokens_per_sec_per_gpu: 9.040121E+03 | global_batch_size: 64 | lm_loss: 1.080477E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 341988.906 [ip-26-0-154-121:0]:06/21/2023 17:59:47 [INFO|DP=0|PP=0|TP=0]: iteration: 3093 / 150000 | consumed_samples: 197952 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782163E+05 | tokens_per_sec_per_gpu: 9.034630E+03 | global_batch_size: 64 | lm_loss: 1.080470E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 316593.812 [ip-26-0-154-121:0]:06/21/2023 17:59:48 [INFO|DP=0|PP=0|TP=0]: iteration: 3094 / 150000 | consumed_samples: 198016 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789271E+05 | tokens_per_sec_per_gpu: 9.045736E+03 | global_batch_size: 64 | lm_loss: 1.080472E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 301961.406 [ip-26-0-154-121:0]:06/21/2023 17:59:49 [INFO|DP=0|PP=0|TP=0]: iteration: 3095 / 150000 | consumed_samples: 198080 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786641E+05 | tokens_per_sec_per_gpu: 9.041627E+03 | global_batch_size: 64 | lm_loss: 1.080472E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 347403.719 [ip-26-0-154-121:0]:06/21/2023 17:59:50 [INFO|DP=0|PP=0|TP=0]: iteration: 3096 / 150000 | consumed_samples: 198144 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789534E+05 | tokens_per_sec_per_gpu: 9.046148E+03 | global_batch_size: 64 | lm_loss: 1.080474E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 319312.250 [ip-26-0-154-121:0]:06/21/2023 17:59:51 [INFO|DP=0|PP=0|TP=0]: iteration: 3097 / 150000 | consumed_samples: 198208 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786864E+05 | tokens_per_sec_per_gpu: 9.041975E+03 | global_batch_size: 64 | lm_loss: 1.080459E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 309173.688 [ip-26-0-154-121:0]:06/21/2023 17:59:52 [INFO|DP=0|PP=0|TP=0]: iteration: 3098 / 150000 | consumed_samples: 198272 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788833E+05 | tokens_per_sec_per_gpu: 9.045052E+03 | global_batch_size: 64 | lm_loss: 1.080438E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 300077.125 [ip-26-0-154-121:0]:06/21/2023 17:59:53 [INFO|DP=0|PP=0|TP=0]: iteration: 3099 / 150000 | consumed_samples: 198336 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787005E+05 | tokens_per_sec_per_gpu: 9.042196E+03 | global_batch_size: 64 | lm_loss: 1.080456E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 295917.531 [ip-26-0-154-121:0]:06/21/2023 17:59:53 [INFO|DP=0|PP=0|TP=0]: iteration: 3100 / 150000 | consumed_samples: 198400 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790545E+05 | tokens_per_sec_per_gpu: 9.047727E+03 | global_batch_size: 64 | lm_loss: 1.080454E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 297807.562 [ip-26-0-154-121:0]:06/21/2023 17:59:54 [INFO|DP=0|PP=0|TP=0]: iteration: 3101 / 150000 | consumed_samples: 198464 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790140E+05 | tokens_per_sec_per_gpu: 9.047093E+03 | global_batch_size: 64 | lm_loss: 1.080468E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 302529.375 [ip-26-0-154-121:0]:06/21/2023 17:59:55 [INFO|DP=0|PP=0|TP=0]: iteration: 3102 / 150000 | consumed_samples: 198528 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787053E+05 | tokens_per_sec_per_gpu: 9.042270E+03 | global_batch_size: 64 | lm_loss: 1.080445E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 298800.812 [ip-26-0-154-121:0]:06/21/2023 17:59:56 [INFO|DP=0|PP=0|TP=0]: iteration: 3103 / 150000 | consumed_samples: 198592 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787676E+05 | tokens_per_sec_per_gpu: 9.043243E+03 | global_batch_size: 64 | lm_loss: 1.080442E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 289990.656 [ip-26-0-154-121:0]:06/21/2023 17:59:57 [INFO|DP=0|PP=0|TP=0]: iteration: 3104 / 150000 | consumed_samples: 198656 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783953E+05 | tokens_per_sec_per_gpu: 9.037427E+03 | global_batch_size: 64 | lm_loss: 1.080444E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 279465.562 [ip-26-0-154-121:0]:06/21/2023 17:59:58 [INFO|DP=0|PP=0|TP=0]: iteration: 3105 / 150000 | consumed_samples: 198720 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785850E+05 | tokens_per_sec_per_gpu: 9.040390E+03 | global_batch_size: 64 | lm_loss: 1.080449E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 282311.375 [ip-26-0-154-121:0]:06/21/2023 17:59:59 [INFO|DP=0|PP=0|TP=0]: iteration: 3106 / 150000 | consumed_samples: 198784 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786016E+05 | tokens_per_sec_per_gpu: 9.040649E+03 | global_batch_size: 64 | lm_loss: 1.080441E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 283520.500 [ip-26-0-154-121:0]:06/21/2023 18:00:00 [INFO|DP=0|PP=0|TP=0]: iteration: 3107 / 150000 | consumed_samples: 198848 | elapsed_time_per_iteration_ms: 905.2 | tokens_per_sec: 5.792192E+05 | tokens_per_sec_per_gpu: 9.050301E+03 | global_batch_size: 64 | lm_loss: 1.080447E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.54 | hardware_tflops_per_gpu: 104.54 | grad_norm: 286149.000 [ip-26-0-154-121:0]:06/21/2023 18:00:01 [INFO|DP=0|PP=0|TP=0]: iteration: 3108 / 150000 | consumed_samples: 198912 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783185E+05 | tokens_per_sec_per_gpu: 9.036227E+03 | global_batch_size: 64 | lm_loss: 1.080445E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 304536.406 [ip-26-0-154-121:0]:06/21/2023 18:00:02 [INFO|DP=0|PP=0|TP=0]: iteration: 3109 / 150000 | consumed_samples: 198976 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786006E+05 | tokens_per_sec_per_gpu: 9.040635E+03 | global_batch_size: 64 | lm_loss: 1.080435E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 289390.656 [ip-26-0-154-121:0]:06/21/2023 18:00:03 [INFO|DP=0|PP=0|TP=0]: iteration: 3110 / 150000 | consumed_samples: 199040 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787858E+05 | tokens_per_sec_per_gpu: 9.043529E+03 | global_batch_size: 64 | lm_loss: 1.080437E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 294684.125 [ip-26-0-154-121:0]:06/21/2023 18:00:03 [INFO|DP=0|PP=0|TP=0]: iteration: 3111 / 150000 | consumed_samples: 199104 | elapsed_time_per_iteration_ms: 907.5 | tokens_per_sec: 5.777322E+05 | tokens_per_sec_per_gpu: 9.027066E+03 | global_batch_size: 64 | lm_loss: 1.080427E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.27 | hardware_tflops_per_gpu: 104.27 | grad_norm: 301400.031 [ip-26-0-154-121:0]:06/21/2023 18:00:04 [INFO|DP=0|PP=0|TP=0]: iteration: 3112 / 150000 | consumed_samples: 199168 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788384E+05 | tokens_per_sec_per_gpu: 9.044350E+03 | global_batch_size: 64 | lm_loss: 1.080432E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 276694.375 [ip-26-0-154-121:0]:06/21/2023 18:00:05 [INFO|DP=0|PP=0|TP=0]: iteration: 3113 / 150000 | consumed_samples: 199232 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788655E+05 | tokens_per_sec_per_gpu: 9.044774E+03 | global_batch_size: 64 | lm_loss: 1.080421E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 259079.250 [ip-26-0-154-121:0]:06/21/2023 18:00:06 [INFO|DP=0|PP=0|TP=0]: iteration: 3114 / 150000 | consumed_samples: 199296 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788355E+05 | tokens_per_sec_per_gpu: 9.044305E+03 | global_batch_size: 64 | lm_loss: 1.080417E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 256244.141 [ip-26-0-154-121:0]:06/21/2023 18:00:07 [INFO|DP=0|PP=0|TP=0]: iteration: 3115 / 150000 | consumed_samples: 199360 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789073E+05 | tokens_per_sec_per_gpu: 9.045426E+03 | global_batch_size: 64 | lm_loss: 1.080422E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 264421.438 [ip-26-0-154-121:0]:06/21/2023 18:00:08 [INFO|DP=0|PP=0|TP=0]: iteration: 3116 / 150000 | consumed_samples: 199424 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787644E+05 | tokens_per_sec_per_gpu: 9.043193E+03 | global_batch_size: 64 | lm_loss: 1.080422E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 270582.562 [ip-26-0-154-121:0]:06/21/2023 18:00:09 [INFO|DP=0|PP=0|TP=0]: iteration: 3117 / 150000 | consumed_samples: 199488 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787124E+05 | tokens_per_sec_per_gpu: 9.042381E+03 | global_batch_size: 64 | lm_loss: 1.080421E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 244366.516 [ip-26-0-154-121:0]:06/21/2023 18:00:10 [INFO|DP=0|PP=0|TP=0]: iteration: 3118 / 150000 | consumed_samples: 199552 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787225E+05 | tokens_per_sec_per_gpu: 9.042539E+03 | global_batch_size: 64 | lm_loss: 1.080427E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 275895.812 [ip-26-0-154-121:0]:06/21/2023 18:00:11 [INFO|DP=0|PP=0|TP=0]: iteration: 3119 / 150000 | consumed_samples: 199616 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788334E+05 | tokens_per_sec_per_gpu: 9.044271E+03 | global_batch_size: 64 | lm_loss: 1.080427E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 337925.312 [ip-26-0-154-121:0]:06/21/2023 18:00:12 [INFO|DP=0|PP=0|TP=0]: iteration: 3120 / 150000 | consumed_samples: 199680 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787007E+05 | tokens_per_sec_per_gpu: 9.042198E+03 | global_batch_size: 64 | lm_loss: 1.080421E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 305170.000 [ip-26-0-154-121:0]:06/21/2023 18:00:12 [INFO|DP=0|PP=0|TP=0]: iteration: 3121 / 150000 | consumed_samples: 199744 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787027E+05 | tokens_per_sec_per_gpu: 9.042229E+03 | global_batch_size: 64 | lm_loss: 1.080419E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 415290.562 [ip-26-0-154-121:0]:06/21/2023 18:00:13 [INFO|DP=0|PP=0|TP=0]: iteration: 3122 / 150000 | consumed_samples: 199808 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786643E+05 | tokens_per_sec_per_gpu: 9.041630E+03 | global_batch_size: 64 | lm_loss: 1.080423E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 330440.344 [ip-26-0-154-121:0]:06/21/2023 18:00:14 [INFO|DP=0|PP=0|TP=0]: iteration: 3123 / 150000 | consumed_samples: 199872 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787759E+05 | tokens_per_sec_per_gpu: 9.043374E+03 | global_batch_size: 64 | lm_loss: 1.080403E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 284263.406 [ip-26-0-154-121:0]:06/21/2023 18:00:15 [INFO|DP=0|PP=0|TP=0]: iteration: 3124 / 150000 | consumed_samples: 199936 | elapsed_time_per_iteration_ms: 905.1 | tokens_per_sec: 5.792881E+05 | tokens_per_sec_per_gpu: 9.051376E+03 | global_batch_size: 64 | lm_loss: 1.080412E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.55 | hardware_tflops_per_gpu: 104.55 | grad_norm: 263742.844 [ip-26-0-154-121:0]:06/21/2023 18:00:16 [INFO|DP=0|PP=0|TP=0]: iteration: 3125 / 150000 | consumed_samples: 200000 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788680E+05 | tokens_per_sec_per_gpu: 9.044812E+03 | global_batch_size: 64 | lm_loss: 1.080383E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 280972.656 [ip-26-0-154-121:0]:06/21/2023 18:00:17 [INFO|DP=0|PP=0|TP=0]: iteration: 3126 / 150000 | consumed_samples: 200064 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.790236E+05 | tokens_per_sec_per_gpu: 9.047243E+03 | global_batch_size: 64 | lm_loss: 1.080407E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 241049.406 [ip-26-0-154-121:0]:06/21/2023 18:00:18 [INFO|DP=0|PP=0|TP=0]: iteration: 3127 / 150000 | consumed_samples: 200128 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789899E+05 | tokens_per_sec_per_gpu: 9.046717E+03 | global_batch_size: 64 | lm_loss: 1.080410E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 264550.469 [ip-26-0-154-121:0]:06/21/2023 18:00:19 [INFO|DP=0|PP=0|TP=0]: iteration: 3128 / 150000 | consumed_samples: 200192 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788038E+05 | tokens_per_sec_per_gpu: 9.043809E+03 | global_batch_size: 64 | lm_loss: 1.080408E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 270734.594 [ip-26-0-154-121:0]:06/21/2023 18:00:20 [INFO|DP=0|PP=0|TP=0]: iteration: 3129 / 150000 | consumed_samples: 200256 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786949E+05 | tokens_per_sec_per_gpu: 9.042108E+03 | global_batch_size: 64 | lm_loss: 1.080406E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 245077.422 [ip-26-0-154-121:0]:06/21/2023 18:00:21 [INFO|DP=0|PP=0|TP=0]: iteration: 3130 / 150000 | consumed_samples: 200320 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785962E+05 | tokens_per_sec_per_gpu: 9.040566E+03 | global_batch_size: 64 | lm_loss: 1.080407E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 232093.641 [ip-26-0-154-121:0]:06/21/2023 18:00:22 [INFO|DP=0|PP=0|TP=0]: iteration: 3131 / 150000 | consumed_samples: 200384 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786045E+05 | tokens_per_sec_per_gpu: 9.040695E+03 | global_batch_size: 64 | lm_loss: 1.080401E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 242936.797 [ip-26-0-154-121:0]:06/21/2023 18:00:22 [INFO|DP=0|PP=0|TP=0]: iteration: 3132 / 150000 | consumed_samples: 200448 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786579E+05 | tokens_per_sec_per_gpu: 9.041530E+03 | global_batch_size: 64 | lm_loss: 1.080404E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 240683.062 [ip-26-0-154-121:0]:06/21/2023 18:00:23 [INFO|DP=0|PP=0|TP=0]: iteration: 3133 / 150000 | consumed_samples: 200512 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783871E+05 | tokens_per_sec_per_gpu: 9.037299E+03 | global_batch_size: 64 | lm_loss: 1.080400E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 250392.688 [ip-26-0-154-121:0]:06/21/2023 18:00:24 [INFO|DP=0|PP=0|TP=0]: iteration: 3134 / 150000 | consumed_samples: 200576 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789717E+05 | tokens_per_sec_per_gpu: 9.046433E+03 | global_batch_size: 64 | lm_loss: 1.080393E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 246395.250 [ip-26-0-154-121:0]:06/21/2023 18:00:25 [INFO|DP=0|PP=0|TP=0]: iteration: 3135 / 150000 | consumed_samples: 200640 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788710E+05 | tokens_per_sec_per_gpu: 9.044859E+03 | global_batch_size: 64 | lm_loss: 1.080391E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 235717.938 [ip-26-0-154-121:0]:06/21/2023 18:00:26 [INFO|DP=0|PP=0|TP=0]: iteration: 3136 / 150000 | consumed_samples: 200704 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785649E+05 | tokens_per_sec_per_gpu: 9.040076E+03 | global_batch_size: 64 | lm_loss: 1.080383E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 239457.047 [ip-26-0-154-121:0]:06/21/2023 18:00:27 [INFO|DP=0|PP=0|TP=0]: iteration: 3137 / 150000 | consumed_samples: 200768 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783932E+05 | tokens_per_sec_per_gpu: 9.037394E+03 | global_batch_size: 64 | lm_loss: 1.080277E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 265166.406 [ip-26-0-154-121:0]:06/21/2023 18:00:28 [INFO|DP=0|PP=0|TP=0]: iteration: 3138 / 150000 | consumed_samples: 200832 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786235E+05 | tokens_per_sec_per_gpu: 9.040992E+03 | global_batch_size: 64 | lm_loss: 1.080381E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 245920.172 [ip-26-0-154-121:0]:06/21/2023 18:00:29 [INFO|DP=0|PP=0|TP=0]: iteration: 3139 / 150000 | consumed_samples: 200896 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785286E+05 | tokens_per_sec_per_gpu: 9.039510E+03 | global_batch_size: 64 | lm_loss: 1.080383E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 256529.578 [ip-26-0-154-121:0]:06/21/2023 18:00:30 [INFO|DP=0|PP=0|TP=0]: iteration: 3140 / 150000 | consumed_samples: 200960 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785806E+05 | tokens_per_sec_per_gpu: 9.040321E+03 | global_batch_size: 64 | lm_loss: 1.080371E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 297918.938 [ip-26-0-154-121:0]:06/21/2023 18:00:31 [INFO|DP=0|PP=0|TP=0]: iteration: 3141 / 150000 | consumed_samples: 201024 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787403E+05 | tokens_per_sec_per_gpu: 9.042817E+03 | global_batch_size: 64 | lm_loss: 1.080381E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 254251.562 [ip-26-0-154-121:0]:06/21/2023 18:00:32 [INFO|DP=0|PP=0|TP=0]: iteration: 3142 / 150000 | consumed_samples: 201088 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787739E+05 | tokens_per_sec_per_gpu: 9.043343E+03 | global_batch_size: 64 | lm_loss: 1.080376E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 246139.219 [ip-26-0-154-121:0]:06/21/2023 18:00:32 [INFO|DP=0|PP=0|TP=0]: iteration: 3143 / 150000 | consumed_samples: 201152 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788486E+05 | tokens_per_sec_per_gpu: 9.044509E+03 | global_batch_size: 64 | lm_loss: 1.080371E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 259674.219 [ip-26-0-154-121:0]:06/21/2023 18:00:33 [INFO|DP=0|PP=0|TP=0]: iteration: 3144 / 150000 | consumed_samples: 201216 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783658E+05 | tokens_per_sec_per_gpu: 9.036966E+03 | global_batch_size: 64 | lm_loss: 1.080369E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 262641.844 [ip-26-0-154-121:0]:06/21/2023 18:00:34 [INFO|DP=0|PP=0|TP=0]: iteration: 3145 / 150000 | consumed_samples: 201280 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782874E+05 | tokens_per_sec_per_gpu: 9.035740E+03 | global_batch_size: 64 | lm_loss: 1.080365E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 285244.906 [ip-26-0-154-121:0]:06/21/2023 18:00:35 [INFO|DP=0|PP=0|TP=0]: iteration: 3146 / 150000 | consumed_samples: 201344 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789042E+05 | tokens_per_sec_per_gpu: 9.045378E+03 | global_batch_size: 64 | lm_loss: 1.080366E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 265114.281 [ip-26-0-154-121:0]:06/21/2023 18:00:36 [INFO|DP=0|PP=0|TP=0]: iteration: 3147 / 150000 | consumed_samples: 201408 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789160E+05 | tokens_per_sec_per_gpu: 9.045562E+03 | global_batch_size: 64 | lm_loss: 1.080372E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 256011.750 [ip-26-0-154-121:0]:06/21/2023 18:00:37 [INFO|DP=0|PP=0|TP=0]: iteration: 3148 / 150000 | consumed_samples: 201472 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786236E+05 | tokens_per_sec_per_gpu: 9.040994E+03 | global_batch_size: 64 | lm_loss: 1.080368E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 273595.688 [ip-26-0-154-121:0]:06/21/2023 18:00:38 [INFO|DP=0|PP=0|TP=0]: iteration: 3149 / 150000 | consumed_samples: 201536 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787867E+05 | tokens_per_sec_per_gpu: 9.043543E+03 | global_batch_size: 64 | lm_loss: 1.080365E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 303709.875 [ip-26-0-154-121:0]:06/21/2023 18:00:39 [INFO|DP=0|PP=0|TP=0]: iteration: 3150 / 150000 | consumed_samples: 201600 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785420E+05 | tokens_per_sec_per_gpu: 9.039719E+03 | global_batch_size: 64 | lm_loss: 1.080356E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 723344.688 [ip-26-0-154-121:0]:06/21/2023 18:00:40 [INFO|DP=0|PP=0|TP=0]: iteration: 3151 / 150000 | consumed_samples: 201664 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785879E+05 | tokens_per_sec_per_gpu: 9.040435E+03 | global_batch_size: 64 | lm_loss: 1.080356E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 341174.688 [ip-26-0-154-121:0]:06/21/2023 18:00:41 [INFO|DP=0|PP=0|TP=0]: iteration: 3152 / 150000 | consumed_samples: 201728 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789161E+05 | tokens_per_sec_per_gpu: 9.045564E+03 | global_batch_size: 64 | lm_loss: 1.080358E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 386530.031 [ip-26-0-154-121:0]:06/21/2023 18:00:42 [INFO|DP=0|PP=0|TP=0]: iteration: 3153 / 150000 | consumed_samples: 201792 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786331E+05 | tokens_per_sec_per_gpu: 9.041142E+03 | global_batch_size: 64 | lm_loss: 1.080357E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 383882.062 [ip-26-0-154-121:0]:06/21/2023 18:00:42 [INFO|DP=0|PP=0|TP=0]: iteration: 3154 / 150000 | consumed_samples: 201856 | elapsed_time_per_iteration_ms: 910.8 | tokens_per_sec: 5.756623E+05 | tokens_per_sec_per_gpu: 8.994724E+03 | global_batch_size: 64 | lm_loss: 1.080362E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 103.90 | hardware_tflops_per_gpu: 103.90 | grad_norm: 402243.625 [ip-26-0-154-121:0]:06/21/2023 18:00:43 [INFO|DP=0|PP=0|TP=0]: iteration: 3155 / 150000 | consumed_samples: 201920 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785166E+05 | tokens_per_sec_per_gpu: 9.039322E+03 | global_batch_size: 64 | lm_loss: 1.080354E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 310444.250 [ip-26-0-154-121:0]:06/21/2023 18:00:44 [INFO|DP=0|PP=0|TP=0]: iteration: 3156 / 150000 | consumed_samples: 201984 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786622E+05 | tokens_per_sec_per_gpu: 9.041596E+03 | global_batch_size: 64 | lm_loss: 1.080355E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 298185.781 [ip-26-0-154-121:0]:06/21/2023 18:00:45 [INFO|DP=0|PP=0|TP=0]: iteration: 3157 / 150000 | consumed_samples: 202048 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788119E+05 | tokens_per_sec_per_gpu: 9.043936E+03 | global_batch_size: 64 | lm_loss: 1.080359E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 421320.500 [ip-26-0-154-121:0]:06/21/2023 18:00:46 [INFO|DP=0|PP=0|TP=0]: iteration: 3158 / 150000 | consumed_samples: 202112 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789464E+05 | tokens_per_sec_per_gpu: 9.046038E+03 | global_batch_size: 64 | lm_loss: 1.080354E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 337410.656 [ip-26-0-154-121:0]:06/21/2023 18:00:47 [INFO|DP=0|PP=0|TP=0]: iteration: 3159 / 150000 | consumed_samples: 202176 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787995E+05 | tokens_per_sec_per_gpu: 9.043743E+03 | global_batch_size: 64 | lm_loss: 1.080350E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 305068.375 [ip-26-0-154-121:0]:06/21/2023 18:00:48 [INFO|DP=0|PP=0|TP=0]: iteration: 3160 / 150000 | consumed_samples: 202240 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783871E+05 | tokens_per_sec_per_gpu: 9.037299E+03 | global_batch_size: 64 | lm_loss: 1.080350E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 295722.000 [ip-26-0-154-121:0]:06/21/2023 18:00:49 [INFO|DP=0|PP=0|TP=0]: iteration: 3161 / 150000 | consumed_samples: 202304 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786521E+05 | tokens_per_sec_per_gpu: 9.041439E+03 | global_batch_size: 64 | lm_loss: 1.080358E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 344151.000 [ip-26-0-154-121:0]:06/21/2023 18:00:50 [INFO|DP=0|PP=0|TP=0]: iteration: 3162 / 150000 | consumed_samples: 202368 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787616E+05 | tokens_per_sec_per_gpu: 9.043150E+03 | global_batch_size: 64 | lm_loss: 1.080349E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 1546917.875 [ip-26-0-154-121:0]:06/21/2023 18:00:51 [INFO|DP=0|PP=0|TP=0]: iteration: 3163 / 150000 | consumed_samples: 202432 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787389E+05 | tokens_per_sec_per_gpu: 9.042796E+03 | global_batch_size: 64 | lm_loss: 1.080345E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 745378.312 [ip-26-0-154-121:0]:06/21/2023 18:00:51 [INFO|DP=0|PP=0|TP=0]: iteration: 3164 / 150000 | consumed_samples: 202496 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789571E+05 | tokens_per_sec_per_gpu: 9.046205E+03 | global_batch_size: 64 | lm_loss: 1.080331E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 529666.938 [ip-26-0-154-121:0]:06/21/2023 18:00:52 [INFO|DP=0|PP=0|TP=0]: iteration: 3165 / 150000 | consumed_samples: 202560 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788084E+05 | tokens_per_sec_per_gpu: 9.043881E+03 | global_batch_size: 64 | lm_loss: 1.080347E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 336965.062 [ip-26-0-154-121:0]:06/21/2023 18:00:53 [INFO|DP=0|PP=0|TP=0]: iteration: 3166 / 150000 | consumed_samples: 202624 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789557E+05 | tokens_per_sec_per_gpu: 9.046183E+03 | global_batch_size: 64 | lm_loss: 1.080340E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 1271195.875 [ip-26-0-154-121:0]:06/21/2023 18:00:54 [INFO|DP=0|PP=0|TP=0]: iteration: 3167 / 150000 | consumed_samples: 202688 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786740E+05 | tokens_per_sec_per_gpu: 9.041782E+03 | global_batch_size: 64 | lm_loss: 1.080345E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 745046.062 [ip-26-0-154-121:0]:06/21/2023 18:00:55 [INFO|DP=0|PP=0|TP=0]: iteration: 3168 / 150000 | consumed_samples: 202752 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784331E+05 | tokens_per_sec_per_gpu: 9.038017E+03 | global_batch_size: 64 | lm_loss: 1.080340E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 554785.938 [ip-26-0-154-121:0]:06/21/2023 18:00:56 [INFO|DP=0|PP=0|TP=0]: iteration: 3169 / 150000 | consumed_samples: 202816 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785061E+05 | tokens_per_sec_per_gpu: 9.039158E+03 | global_batch_size: 64 | lm_loss: 1.080339E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 458509.750 [ip-26-0-154-121:0]:06/21/2023 18:00:57 [INFO|DP=0|PP=0|TP=0]: iteration: 3170 / 150000 | consumed_samples: 202880 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784258E+05 | tokens_per_sec_per_gpu: 9.037903E+03 | global_batch_size: 64 | lm_loss: 1.080334E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 350105.031 [ip-26-0-154-121:0]:06/21/2023 18:00:58 [INFO|DP=0|PP=0|TP=0]: iteration: 3171 / 150000 | consumed_samples: 202944 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784080E+05 | tokens_per_sec_per_gpu: 9.037625E+03 | global_batch_size: 64 | lm_loss: 1.080334E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 345167.281 [ip-26-0-154-121:0]:06/21/2023 18:00:59 [INFO|DP=0|PP=0|TP=0]: iteration: 3172 / 150000 | consumed_samples: 203008 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783009E+05 | tokens_per_sec_per_gpu: 9.035951E+03 | global_batch_size: 64 | lm_loss: 1.080339E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 426909.406 [ip-26-0-154-121:0]:06/21/2023 18:01:00 [INFO|DP=0|PP=0|TP=0]: iteration: 3173 / 150000 | consumed_samples: 203072 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784179E+05 | tokens_per_sec_per_gpu: 9.037779E+03 | global_batch_size: 64 | lm_loss: 1.080342E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 434091.250 [ip-26-0-154-121:0]:06/21/2023 18:01:01 [INFO|DP=0|PP=0|TP=0]: iteration: 3174 / 150000 | consumed_samples: 203136 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784787E+05 | tokens_per_sec_per_gpu: 9.038730E+03 | global_batch_size: 64 | lm_loss: 1.080333E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 461979.938 [ip-26-0-154-121:0]:06/21/2023 18:01:01 [INFO|DP=0|PP=0|TP=0]: iteration: 3175 / 150000 | consumed_samples: 203200 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784787E+05 | tokens_per_sec_per_gpu: 9.038730E+03 | global_batch_size: 64 | lm_loss: 1.080334E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 461355.656 [ip-26-0-154-121:0]:06/21/2023 18:01:02 [INFO|DP=0|PP=0|TP=0]: iteration: 3176 / 150000 | consumed_samples: 203264 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788009E+05 | tokens_per_sec_per_gpu: 9.043764E+03 | global_batch_size: 64 | lm_loss: 1.080313E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 388048.062 [ip-26-0-154-121:0]:06/21/2023 18:01:03 [INFO|DP=0|PP=0|TP=0]: iteration: 3177 / 150000 | consumed_samples: 203328 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784437E+05 | tokens_per_sec_per_gpu: 9.038183E+03 | global_batch_size: 64 | lm_loss: 1.080334E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 361493.625 [ip-26-0-154-121:0]:06/21/2023 18:01:04 [INFO|DP=0|PP=0|TP=0]: iteration: 3178 / 150000 | consumed_samples: 203392 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787246E+05 | tokens_per_sec_per_gpu: 9.042572E+03 | global_batch_size: 64 | lm_loss: 1.080333E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 497671.094 [ip-26-0-154-121:0]:06/21/2023 18:01:05 [INFO|DP=0|PP=0|TP=0]: iteration: 3179 / 150000 | consumed_samples: 203456 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785366E+05 | tokens_per_sec_per_gpu: 9.039634E+03 | global_batch_size: 64 | lm_loss: 1.080332E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 468009.250 [ip-26-0-154-121:0]:06/21/2023 18:01:06 [INFO|DP=0|PP=0|TP=0]: iteration: 3180 / 150000 | consumed_samples: 203520 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785335E+05 | tokens_per_sec_per_gpu: 9.039586E+03 | global_batch_size: 64 | lm_loss: 1.080330E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 419603.812 [ip-26-0-154-121:0]:06/21/2023 18:01:07 [INFO|DP=0|PP=0|TP=0]: iteration: 3181 / 150000 | consumed_samples: 203584 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783914E+05 | tokens_per_sec_per_gpu: 9.037366E+03 | global_batch_size: 64 | lm_loss: 1.080328E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 378715.156 [ip-26-0-154-121:0]:06/21/2023 18:01:08 [INFO|DP=0|PP=0|TP=0]: iteration: 3182 / 150000 | consumed_samples: 203648 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787825E+05 | tokens_per_sec_per_gpu: 9.043476E+03 | global_batch_size: 64 | lm_loss: 1.080333E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 425751.062 [ip-26-0-154-121:0]:06/21/2023 18:01:09 [INFO|DP=0|PP=0|TP=0]: iteration: 3183 / 150000 | consumed_samples: 203712 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786640E+05 | tokens_per_sec_per_gpu: 9.041625E+03 | global_batch_size: 64 | lm_loss: 1.080329E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 346740.875 [ip-26-0-154-121:0]:06/21/2023 18:01:10 [INFO|DP=0|PP=0|TP=0]: iteration: 3184 / 150000 | consumed_samples: 203776 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789195E+05 | tokens_per_sec_per_gpu: 9.045617E+03 | global_batch_size: 64 | lm_loss: 1.080330E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 366657.719 [ip-26-0-154-121:0]:06/21/2023 18:01:11 [INFO|DP=0|PP=0|TP=0]: iteration: 3185 / 150000 | consumed_samples: 203840 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786579E+05 | tokens_per_sec_per_gpu: 9.041530E+03 | global_batch_size: 64 | lm_loss: 1.080334E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 350066.188 [ip-26-0-154-121:0]:06/21/2023 18:01:11 [INFO|DP=0|PP=0|TP=0]: iteration: 3186 / 150000 | consumed_samples: 203904 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782772E+05 | tokens_per_sec_per_gpu: 9.035581E+03 | global_batch_size: 64 | lm_loss: 1.080328E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 319536.250 [ip-26-0-154-121:0]:06/21/2023 18:01:12 [INFO|DP=0|PP=0|TP=0]: iteration: 3187 / 150000 | consumed_samples: 203968 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788602E+05 | tokens_per_sec_per_gpu: 9.044690E+03 | global_batch_size: 64 | lm_loss: 1.080324E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 320762.000 [ip-26-0-154-121:0]:06/21/2023 18:01:13 [INFO|DP=0|PP=0|TP=0]: iteration: 3188 / 150000 | consumed_samples: 204032 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786882E+05 | tokens_per_sec_per_gpu: 9.042003E+03 | global_batch_size: 64 | lm_loss: 1.080321E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 391764.188 [ip-26-0-154-121:0]:06/21/2023 18:01:14 [INFO|DP=0|PP=0|TP=0]: iteration: 3189 / 150000 | consumed_samples: 204096 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786635E+05 | tokens_per_sec_per_gpu: 9.041618E+03 | global_batch_size: 64 | lm_loss: 1.080307E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 414449.219 [ip-26-0-154-121:0]:06/21/2023 18:01:15 [INFO|DP=0|PP=0|TP=0]: iteration: 3190 / 150000 | consumed_samples: 204160 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784378E+05 | tokens_per_sec_per_gpu: 9.038091E+03 | global_batch_size: 64 | lm_loss: 1.080321E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 365083.312 [ip-26-0-154-121:0]:06/21/2023 18:01:16 [INFO|DP=0|PP=0|TP=0]: iteration: 3191 / 150000 | consumed_samples: 204224 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784065E+05 | tokens_per_sec_per_gpu: 9.037601E+03 | global_batch_size: 64 | lm_loss: 1.080325E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 338332.125 [ip-26-0-154-121:0]:06/21/2023 18:01:17 [INFO|DP=0|PP=0|TP=0]: iteration: 3192 / 150000 | consumed_samples: 204288 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786596E+05 | tokens_per_sec_per_gpu: 9.041556E+03 | global_batch_size: 64 | lm_loss: 1.080331E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 362002.156 [ip-26-0-154-121:0]:06/21/2023 18:01:18 [INFO|DP=0|PP=0|TP=0]: iteration: 3193 / 150000 | consumed_samples: 204352 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783724E+05 | tokens_per_sec_per_gpu: 9.037068E+03 | global_batch_size: 64 | lm_loss: 1.080318E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 408944.406 [ip-26-0-154-121:0]:06/21/2023 18:01:19 [INFO|DP=0|PP=0|TP=0]: iteration: 3194 / 150000 | consumed_samples: 204416 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784259E+05 | tokens_per_sec_per_gpu: 9.037905E+03 | global_batch_size: 64 | lm_loss: 1.080321E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 325841.906 [ip-26-0-154-121:0]:06/21/2023 18:01:20 [INFO|DP=0|PP=0|TP=0]: iteration: 3195 / 150000 | consumed_samples: 204480 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784812E+05 | tokens_per_sec_per_gpu: 9.038768E+03 | global_batch_size: 64 | lm_loss: 1.080320E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 335414.969 [ip-26-0-154-121:0]:06/21/2023 18:01:21 [INFO|DP=0|PP=0|TP=0]: iteration: 3196 / 150000 | consumed_samples: 204544 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785681E+05 | tokens_per_sec_per_gpu: 9.040126E+03 | global_batch_size: 64 | lm_loss: 1.080326E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 410660.656 [ip-26-0-154-121:0]:06/21/2023 18:01:21 [INFO|DP=0|PP=0|TP=0]: iteration: 3197 / 150000 | consumed_samples: 204608 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787613E+05 | tokens_per_sec_per_gpu: 9.043145E+03 | global_batch_size: 64 | lm_loss: 1.080320E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 365593.312 [ip-26-0-154-121:0]:06/21/2023 18:01:22 [INFO|DP=0|PP=0|TP=0]: iteration: 3198 / 150000 | consumed_samples: 204672 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788366E+05 | tokens_per_sec_per_gpu: 9.044321E+03 | global_batch_size: 64 | lm_loss: 1.080318E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 327071.625 [ip-26-0-154-121:0]:06/21/2023 18:01:23 [INFO|DP=0|PP=0|TP=0]: iteration: 3199 / 150000 | consumed_samples: 204736 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784953E+05 | tokens_per_sec_per_gpu: 9.038989E+03 | global_batch_size: 64 | lm_loss: 1.080320E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 336199.094 [ip-26-0-154-121:0]:06/21/2023 18:01:24 [INFO|DP=0|PP=0|TP=0]: iteration: 3200 / 150000 | consumed_samples: 204800 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787516E+05 | tokens_per_sec_per_gpu: 9.042993E+03 | global_batch_size: 64 | lm_loss: 1.080319E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 411649.906 [ip-26-0-154-121:0]:06/21/2023 18:01:25 [INFO|DP=0|PP=0|TP=0]: iteration: 3201 / 150000 | consumed_samples: 204864 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788209E+05 | tokens_per_sec_per_gpu: 9.044076E+03 | global_batch_size: 64 | lm_loss: 1.080315E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 502618.625 [ip-26-0-154-121:0]:06/21/2023 18:01:26 [INFO|DP=0|PP=0|TP=0]: iteration: 3202 / 150000 | consumed_samples: 204928 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787449E+05 | tokens_per_sec_per_gpu: 9.042888E+03 | global_batch_size: 64 | lm_loss: 1.080311E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 376965.438 [ip-26-0-154-121:0]:06/21/2023 18:01:27 [INFO|DP=0|PP=0|TP=0]: iteration: 3203 / 150000 | consumed_samples: 204992 | elapsed_time_per_iteration_ms: 905.5 | tokens_per_sec: 5.789795E+05 | tokens_per_sec_per_gpu: 9.046555E+03 | global_batch_size: 64 | lm_loss: 1.080315E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.50 | hardware_tflops_per_gpu: 104.50 | grad_norm: 366949.938 [ip-26-0-154-121:0]:06/21/2023 18:01:28 [INFO|DP=0|PP=0|TP=0]: iteration: 3204 / 150000 | consumed_samples: 205056 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788180E+05 | tokens_per_sec_per_gpu: 9.044031E+03 | global_batch_size: 64 | lm_loss: 1.080306E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 444423.406 [ip-26-0-154-121:0]:06/21/2023 18:01:29 [INFO|DP=0|PP=0|TP=0]: iteration: 3205 / 150000 | consumed_samples: 205120 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787764E+05 | tokens_per_sec_per_gpu: 9.043381E+03 | global_batch_size: 64 | lm_loss: 1.080302E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 397198.906 [ip-26-0-154-121:0]:06/21/2023 18:01:30 [INFO|DP=0|PP=0|TP=0]: iteration: 3206 / 150000 | consumed_samples: 205184 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788835E+05 | tokens_per_sec_per_gpu: 9.045055E+03 | global_batch_size: 64 | lm_loss: 1.080311E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 427082.500 [ip-26-0-154-121:0]:06/21/2023 18:01:31 [INFO|DP=0|PP=0|TP=0]: iteration: 3207 / 150000 | consumed_samples: 205248 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785279E+05 | tokens_per_sec_per_gpu: 9.039498E+03 | global_batch_size: 64 | lm_loss: 1.080311E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 373201.094 [ip-26-0-154-121:0]:06/21/2023 18:01:31 [INFO|DP=0|PP=0|TP=0]: iteration: 3208 / 150000 | consumed_samples: 205312 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784542E+05 | tokens_per_sec_per_gpu: 9.038347E+03 | global_batch_size: 64 | lm_loss: 1.080315E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 378172.594 [ip-26-0-154-121:0]:06/21/2023 18:01:32 [INFO|DP=0|PP=0|TP=0]: iteration: 3209 / 150000 | consumed_samples: 205376 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786136E+05 | tokens_per_sec_per_gpu: 9.040837E+03 | global_batch_size: 64 | lm_loss: 1.080312E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 433278.531 [ip-26-0-154-121:0]:06/21/2023 18:01:33 [INFO|DP=0|PP=0|TP=0]: iteration: 3210 / 150000 | consumed_samples: 205440 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789243E+05 | tokens_per_sec_per_gpu: 9.045693E+03 | global_batch_size: 64 | lm_loss: 1.080305E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 430361.750 [ip-26-0-154-121:0]:06/21/2023 18:01:34 [INFO|DP=0|PP=0|TP=0]: iteration: 3211 / 150000 | consumed_samples: 205504 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785795E+05 | tokens_per_sec_per_gpu: 9.040304E+03 | global_batch_size: 64 | lm_loss: 1.080308E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 396575.750 [ip-26-0-154-121:0]:06/21/2023 18:01:35 [INFO|DP=0|PP=0|TP=0]: iteration: 3212 / 150000 | consumed_samples: 205568 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785702E+05 | tokens_per_sec_per_gpu: 9.040159E+03 | global_batch_size: 64 | lm_loss: 1.080291E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 379395.375 [ip-26-0-154-121:0]:06/21/2023 18:01:36 [INFO|DP=0|PP=0|TP=0]: iteration: 3213 / 150000 | consumed_samples: 205632 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785410E+05 | tokens_per_sec_per_gpu: 9.039703E+03 | global_batch_size: 64 | lm_loss: 1.080302E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 398921.812 [ip-26-0-154-121:0]:06/21/2023 18:01:37 [INFO|DP=0|PP=0|TP=0]: iteration: 3214 / 150000 | consumed_samples: 205696 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784031E+05 | tokens_per_sec_per_gpu: 9.037549E+03 | global_batch_size: 64 | lm_loss: 1.080307E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 370401.500 [ip-26-0-154-121:0]:06/21/2023 18:01:38 [INFO|DP=0|PP=0|TP=0]: iteration: 3215 / 150000 | consumed_samples: 205760 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786523E+05 | tokens_per_sec_per_gpu: 9.041442E+03 | global_batch_size: 64 | lm_loss: 1.080301E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 369374.156 [ip-26-0-154-121:0]:06/21/2023 18:01:39 [INFO|DP=0|PP=0|TP=0]: iteration: 3216 / 150000 | consumed_samples: 205824 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786121E+05 | tokens_per_sec_per_gpu: 9.040814E+03 | global_batch_size: 64 | lm_loss: 1.080307E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 404334.594 [ip-26-0-154-121:0]:06/21/2023 18:01:40 [INFO|DP=0|PP=0|TP=0]: iteration: 3217 / 150000 | consumed_samples: 205888 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786063E+05 | tokens_per_sec_per_gpu: 9.040723E+03 | global_batch_size: 64 | lm_loss: 1.080308E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 419555.844 [ip-26-0-154-121:0]:06/21/2023 18:01:40 [INFO|DP=0|PP=0|TP=0]: iteration: 3218 / 150000 | consumed_samples: 205952 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785352E+05 | tokens_per_sec_per_gpu: 9.039612E+03 | global_batch_size: 64 | lm_loss: 1.080310E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 406619.750 [ip-26-0-154-121:0]:06/21/2023 18:01:41 [INFO|DP=0|PP=0|TP=0]: iteration: 3219 / 150000 | consumed_samples: 206016 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784769E+05 | tokens_per_sec_per_gpu: 9.038702E+03 | global_batch_size: 64 | lm_loss: 1.080311E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 431159.344 [ip-26-0-154-121:0]:06/21/2023 18:01:42 [INFO|DP=0|PP=0|TP=0]: iteration: 3220 / 150000 | consumed_samples: 206080 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788165E+05 | tokens_per_sec_per_gpu: 9.044007E+03 | global_batch_size: 64 | lm_loss: 1.080303E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 351532.500 [ip-26-0-154-121:0]:06/21/2023 18:01:43 [INFO|DP=0|PP=0|TP=0]: iteration: 3221 / 150000 | consumed_samples: 206144 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785172E+05 | tokens_per_sec_per_gpu: 9.039332E+03 | global_batch_size: 64 | lm_loss: 1.080303E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 490260.906 [ip-26-0-154-121:0]:06/21/2023 18:01:44 [INFO|DP=0|PP=0|TP=0]: iteration: 3222 / 150000 | consumed_samples: 206208 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786497E+05 | tokens_per_sec_per_gpu: 9.041401E+03 | global_batch_size: 64 | lm_loss: 1.080303E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 387336.656 [ip-26-0-154-121:0]:06/21/2023 18:01:45 [INFO|DP=0|PP=0|TP=0]: iteration: 3223 / 150000 | consumed_samples: 206272 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784594E+05 | tokens_per_sec_per_gpu: 9.038428E+03 | global_batch_size: 64 | lm_loss: 1.080306E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 364310.500 [ip-26-0-154-121:0]:06/21/2023 18:01:46 [INFO|DP=0|PP=0|TP=0]: iteration: 3224 / 150000 | consumed_samples: 206336 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787066E+05 | tokens_per_sec_per_gpu: 9.042291E+03 | global_batch_size: 64 | lm_loss: 1.080304E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 328759.906 [ip-26-0-154-121:0]:06/21/2023 18:01:47 [INFO|DP=0|PP=0|TP=0]: iteration: 3225 / 150000 | consumed_samples: 206400 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784252E+05 | tokens_per_sec_per_gpu: 9.037893E+03 | global_batch_size: 64 | lm_loss: 1.080300E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 351672.438 [ip-26-0-154-121:0]:06/21/2023 18:01:48 [INFO|DP=0|PP=0|TP=0]: iteration: 3226 / 150000 | consumed_samples: 206464 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786364E+05 | tokens_per_sec_per_gpu: 9.041194E+03 | global_batch_size: 64 | lm_loss: 1.080302E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 362416.562 [ip-26-0-154-121:0]:06/21/2023 18:01:49 [INFO|DP=0|PP=0|TP=0]: iteration: 3227 / 150000 | consumed_samples: 206528 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784276E+05 | tokens_per_sec_per_gpu: 9.037931E+03 | global_batch_size: 64 | lm_loss: 1.080303E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 388706.219 [ip-26-0-154-121:0]:06/21/2023 18:01:50 [INFO|DP=0|PP=0|TP=0]: iteration: 3228 / 150000 | consumed_samples: 206592 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783012E+05 | tokens_per_sec_per_gpu: 9.035956E+03 | global_batch_size: 64 | lm_loss: 1.080299E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 408368.469 [ip-26-0-154-121:0]:06/21/2023 18:01:50 [INFO|DP=0|PP=0|TP=0]: iteration: 3229 / 150000 | consumed_samples: 206656 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787226E+05 | tokens_per_sec_per_gpu: 9.042541E+03 | global_batch_size: 64 | lm_loss: 1.080302E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 352519.344 [ip-26-0-154-121:0]:06/21/2023 18:01:51 [INFO|DP=0|PP=0|TP=0]: iteration: 3230 / 150000 | consumed_samples: 206720 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784341E+05 | tokens_per_sec_per_gpu: 9.038034E+03 | global_batch_size: 64 | lm_loss: 1.080298E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 374028.906 [ip-26-0-154-121:0]:06/21/2023 18:01:52 [INFO|DP=0|PP=0|TP=0]: iteration: 3231 / 150000 | consumed_samples: 206784 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789024E+05 | tokens_per_sec_per_gpu: 9.045350E+03 | global_batch_size: 64 | lm_loss: 1.080303E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 436264.125 [ip-26-0-154-121:0]:06/21/2023 18:01:53 [INFO|DP=0|PP=0|TP=0]: iteration: 3232 / 150000 | consumed_samples: 206848 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.783979E+05 | tokens_per_sec_per_gpu: 9.037468E+03 | global_batch_size: 64 | lm_loss: 1.080300E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 395957.844 [ip-26-0-154-121:0]:06/21/2023 18:01:54 [INFO|DP=0|PP=0|TP=0]: iteration: 3233 / 150000 | consumed_samples: 206912 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785264E+05 | tokens_per_sec_per_gpu: 9.039474E+03 | global_batch_size: 64 | lm_loss: 1.080301E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 381504.188 [ip-26-0-154-121:0]:06/21/2023 18:01:55 [INFO|DP=0|PP=0|TP=0]: iteration: 3234 / 150000 | consumed_samples: 206976 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784985E+05 | tokens_per_sec_per_gpu: 9.039039E+03 | global_batch_size: 64 | lm_loss: 1.080296E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 376788.031 [ip-26-0-154-121:0]:06/21/2023 18:01:56 [INFO|DP=0|PP=0|TP=0]: iteration: 3235 / 150000 | consumed_samples: 207040 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790692E+05 | tokens_per_sec_per_gpu: 9.047956E+03 | global_batch_size: 64 | lm_loss: 1.080298E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 370968.125 [ip-26-0-154-121:0]:06/21/2023 18:01:57 [INFO|DP=0|PP=0|TP=0]: iteration: 3236 / 150000 | consumed_samples: 207104 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786306E+05 | tokens_per_sec_per_gpu: 9.041104E+03 | global_batch_size: 64 | lm_loss: 1.080295E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 374783.188 [ip-26-0-154-121:0]:06/21/2023 18:01:58 [INFO|DP=0|PP=0|TP=0]: iteration: 3237 / 150000 | consumed_samples: 207168 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787357E+05 | tokens_per_sec_per_gpu: 9.042746E+03 | global_batch_size: 64 | lm_loss: 1.080226E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 388315.750 [ip-26-0-154-121:0]:06/21/2023 18:01:59 [INFO|DP=0|PP=0|TP=0]: iteration: 3238 / 150000 | consumed_samples: 207232 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788649E+05 | tokens_per_sec_per_gpu: 9.044764E+03 | global_batch_size: 64 | lm_loss: 1.080294E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 378780.312 [ip-26-0-154-121:0]:06/21/2023 18:02:00 [INFO|DP=0|PP=0|TP=0]: iteration: 3239 / 150000 | consumed_samples: 207296 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787631E+05 | tokens_per_sec_per_gpu: 9.043174E+03 | global_batch_size: 64 | lm_loss: 1.080293E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 367647.719 [ip-26-0-154-121:0]:06/21/2023 18:02:00 [INFO|DP=0|PP=0|TP=0]: iteration: 3240 / 150000 | consumed_samples: 207360 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788609E+05 | tokens_per_sec_per_gpu: 9.044702E+03 | global_batch_size: 64 | lm_loss: 1.080299E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 362254.938 [ip-26-0-154-121:0]:06/21/2023 18:02:01 [INFO|DP=0|PP=0|TP=0]: iteration: 3241 / 150000 | consumed_samples: 207424 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784947E+05 | tokens_per_sec_per_gpu: 9.038980E+03 | global_batch_size: 64 | lm_loss: 1.080293E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 394978.969 [ip-26-0-154-121:0]:06/21/2023 18:02:03 [INFO|DP=0|PP=0|TP=0]: iteration: 3242 / 150000 | consumed_samples: 207488 | elapsed_time_per_iteration_ms: 1234.6 | tokens_per_sec: 4.246588E+05 | tokens_per_sec_per_gpu: 6.635293E+03 | global_batch_size: 64 | lm_loss: 1.080308E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 76.65 | hardware_tflops_per_gpu: 76.65 | grad_norm: 364677.781 [ip-26-0-154-121:0]:06/21/2023 18:02:04 [INFO|DP=0|PP=0|TP=0]: iteration: 3243 / 150000 | consumed_samples: 207552 | elapsed_time_per_iteration_ms: 961.8 | tokens_per_sec: 5.451290E+05 | tokens_per_sec_per_gpu: 8.517641E+03 | global_batch_size: 64 | lm_loss: 1.080293E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 98.39 | hardware_tflops_per_gpu: 98.39 | grad_norm: 325898.406 [ip-26-0-154-121:0]:06/21/2023 18:02:05 [INFO|DP=0|PP=0|TP=0]: iteration: 3244 / 150000 | consumed_samples: 207616 | elapsed_time_per_iteration_ms: 964.4 | tokens_per_sec: 5.436228E+05 | tokens_per_sec_per_gpu: 8.494106E+03 | global_batch_size: 64 | lm_loss: 1.080291E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 98.12 | hardware_tflops_per_gpu: 98.12 | grad_norm: 286915.688 [ip-26-0-154-121:0]:06/21/2023 18:02:05 [INFO|DP=0|PP=0|TP=0]: iteration: 3245 / 150000 | consumed_samples: 207680 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784701E+05 | tokens_per_sec_per_gpu: 9.038595E+03 | global_batch_size: 64 | lm_loss: 1.080291E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 280595.125 [ip-26-0-154-121:0]:06/21/2023 18:02:06 [INFO|DP=0|PP=0|TP=0]: iteration: 3246 / 150000 | consumed_samples: 207744 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789246E+05 | tokens_per_sec_per_gpu: 9.045698E+03 | global_batch_size: 64 | lm_loss: 1.080295E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 287264.438 [ip-26-0-154-121:0]:06/21/2023 18:02:07 [INFO|DP=0|PP=0|TP=0]: iteration: 3247 / 150000 | consumed_samples: 207808 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787225E+05 | tokens_per_sec_per_gpu: 9.042539E+03 | global_batch_size: 64 | lm_loss: 1.080287E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 286663.312 [ip-26-0-154-121:0]:06/21/2023 18:02:08 [INFO|DP=0|PP=0|TP=0]: iteration: 3248 / 150000 | consumed_samples: 207872 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783922E+05 | tokens_per_sec_per_gpu: 9.037377E+03 | global_batch_size: 64 | lm_loss: 1.080290E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 427262.000 [ip-26-0-154-121:0]:06/21/2023 18:02:09 [INFO|DP=0|PP=0|TP=0]: iteration: 3249 / 150000 | consumed_samples: 207936 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788221E+05 | tokens_per_sec_per_gpu: 9.044095E+03 | global_batch_size: 64 | lm_loss: 1.080291E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 373230.406 [ip-26-0-154-121:0]:06/21/2023 18:02:10 [INFO|DP=0|PP=0|TP=0]: iteration: 3250 / 150000 | consumed_samples: 208000 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788285E+05 | tokens_per_sec_per_gpu: 9.044195E+03 | global_batch_size: 64 | lm_loss: 1.080282E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 343579.312 [ip-26-0-154-121:0]:06/21/2023 18:02:11 [INFO|DP=0|PP=0|TP=0]: iteration: 3251 / 150000 | consumed_samples: 208064 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782364E+05 | tokens_per_sec_per_gpu: 9.034944E+03 | global_batch_size: 64 | lm_loss: 1.080282E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 290551.438 [ip-26-0-154-121:0]:06/21/2023 18:02:12 [INFO|DP=0|PP=0|TP=0]: iteration: 3252 / 150000 | consumed_samples: 208128 | elapsed_time_per_iteration_ms: 936.6 | tokens_per_sec: 5.597677E+05 | tokens_per_sec_per_gpu: 8.746370E+03 | global_batch_size: 64 | lm_loss: 1.080294E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 101.03 | hardware_tflops_per_gpu: 101.03 | grad_norm: 327306.156 [ip-26-0-154-121:0]:06/21/2023 18:02:13 [INFO|DP=0|PP=0|TP=0]: iteration: 3253 / 150000 | consumed_samples: 208192 | elapsed_time_per_iteration_ms: 923.4 | tokens_per_sec: 5.677636E+05 | tokens_per_sec_per_gpu: 8.871306E+03 | global_batch_size: 64 | lm_loss: 1.080287E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 102.47 | hardware_tflops_per_gpu: 102.47 | grad_norm: 473180.844 [ip-26-0-154-121:0]:06/21/2023 18:02:14 [INFO|DP=0|PP=0|TP=0]: iteration: 3254 / 150000 | consumed_samples: 208256 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785693E+05 | tokens_per_sec_per_gpu: 9.040145E+03 | global_batch_size: 64 | lm_loss: 1.080289E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 353291.531 [ip-26-0-154-121:0]:06/21/2023 18:02:15 [INFO|DP=0|PP=0|TP=0]: iteration: 3255 / 150000 | consumed_samples: 208320 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781843E+05 | tokens_per_sec_per_gpu: 9.034129E+03 | global_batch_size: 64 | lm_loss: 1.080287E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.35 | hardware_tflops_per_gpu: 104.35 | grad_norm: 355992.938 [ip-26-0-154-121:0]:06/21/2023 18:02:15 [INFO|DP=0|PP=0|TP=0]: iteration: 3256 / 150000 | consumed_samples: 208384 | elapsed_time_per_iteration_ms: 908.3 | tokens_per_sec: 5.772155E+05 | tokens_per_sec_per_gpu: 9.018993E+03 | global_batch_size: 64 | lm_loss: 1.080288E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.18 | hardware_tflops_per_gpu: 104.18 | grad_norm: 298346.969 [ip-26-0-154-121:0]:06/21/2023 18:02:16 [INFO|DP=0|PP=0|TP=0]: iteration: 3257 / 150000 | consumed_samples: 208448 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785393E+05 | tokens_per_sec_per_gpu: 9.039677E+03 | global_batch_size: 64 | lm_loss: 1.080288E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 304679.219 [ip-26-0-154-121:0]:06/21/2023 18:02:17 [INFO|DP=0|PP=0|TP=0]: iteration: 3258 / 150000 | consumed_samples: 208512 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785142E+05 | tokens_per_sec_per_gpu: 9.039284E+03 | global_batch_size: 64 | lm_loss: 1.080284E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 329471.562 [ip-26-0-154-121:0]:06/21/2023 18:02:18 [INFO|DP=0|PP=0|TP=0]: iteration: 3259 / 150000 | consumed_samples: 208576 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788701E+05 | tokens_per_sec_per_gpu: 9.044845E+03 | global_batch_size: 64 | lm_loss: 1.080287E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 322893.188 [ip-26-0-154-121:0]:06/21/2023 18:02:19 [INFO|DP=0|PP=0|TP=0]: iteration: 3260 / 150000 | consumed_samples: 208640 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788896E+05 | tokens_per_sec_per_gpu: 9.045150E+03 | global_batch_size: 64 | lm_loss: 1.080287E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 361503.719 [ip-26-0-154-121:0]:06/21/2023 18:02:20 [INFO|DP=0|PP=0|TP=0]: iteration: 3261 / 150000 | consumed_samples: 208704 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787633E+05 | tokens_per_sec_per_gpu: 9.043176E+03 | global_batch_size: 64 | lm_loss: 1.080287E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 330865.000 [ip-26-0-154-121:0]:06/21/2023 18:02:21 [INFO|DP=0|PP=0|TP=0]: iteration: 3262 / 150000 | consumed_samples: 208768 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788632E+05 | tokens_per_sec_per_gpu: 9.044738E+03 | global_batch_size: 64 | lm_loss: 1.080277E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 331849.375 [ip-26-0-154-121:0]:06/21/2023 18:02:22 [INFO|DP=0|PP=0|TP=0]: iteration: 3263 / 150000 | consumed_samples: 208832 | elapsed_time_per_iteration_ms: 951.6 | tokens_per_sec: 5.509755E+05 | tokens_per_sec_per_gpu: 8.608993E+03 | global_batch_size: 64 | lm_loss: 1.080284E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 99.44 | hardware_tflops_per_gpu: 99.44 | grad_norm: 301814.531 [ip-26-0-154-121:0]:06/21/2023 18:02:23 [INFO|DP=0|PP=0|TP=0]: iteration: 3264 / 150000 | consumed_samples: 208896 | elapsed_time_per_iteration_ms: 918.1 | tokens_per_sec: 5.710782E+05 | tokens_per_sec_per_gpu: 8.923096E+03 | global_batch_size: 64 | lm_loss: 1.080286E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 103.07 | hardware_tflops_per_gpu: 103.07 | grad_norm: 339525.688 [ip-26-0-154-121:0]:06/21/2023 18:02:24 [INFO|DP=0|PP=0|TP=0]: iteration: 3265 / 150000 | consumed_samples: 208960 | elapsed_time_per_iteration_ms: 913.9 | tokens_per_sec: 5.737032E+05 | tokens_per_sec_per_gpu: 8.964112E+03 | global_batch_size: 64 | lm_loss: 1.080285E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 103.55 | hardware_tflops_per_gpu: 103.55 | grad_norm: 308434.469 [ip-26-0-154-121:0]:06/21/2023 18:02:25 [INFO|DP=0|PP=0|TP=0]: iteration: 3266 / 150000 | consumed_samples: 209024 | elapsed_time_per_iteration_ms: 1051.8 | tokens_per_sec: 4.984766E+05 | tokens_per_sec_per_gpu: 7.788698E+03 | global_batch_size: 64 | lm_loss: 1.080295E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 89.97 | hardware_tflops_per_gpu: 89.97 | grad_norm: 313612.594 [ip-26-0-154-121:0]:06/21/2023 18:02:26 [INFO|DP=0|PP=0|TP=0]: iteration: 3267 / 150000 | consumed_samples: 209088 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784745E+05 | tokens_per_sec_per_gpu: 9.038664E+03 | global_batch_size: 64 | lm_loss: 1.080274E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 329258.000 [ip-26-0-154-121:0]:06/21/2023 18:02:27 [INFO|DP=0|PP=0|TP=0]: iteration: 3268 / 150000 | consumed_samples: 209152 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785624E+05 | tokens_per_sec_per_gpu: 9.040038E+03 | global_batch_size: 64 | lm_loss: 1.080280E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 341308.344 [ip-26-0-154-121:0]:06/21/2023 18:02:27 [INFO|DP=0|PP=0|TP=0]: iteration: 3269 / 150000 | consumed_samples: 209216 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785032E+05 | tokens_per_sec_per_gpu: 9.039113E+03 | global_batch_size: 64 | lm_loss: 1.080279E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 348036.000 [ip-26-0-154-121:0]:06/21/2023 18:02:28 [INFO|DP=0|PP=0|TP=0]: iteration: 3270 / 150000 | consumed_samples: 209280 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786791E+05 | tokens_per_sec_per_gpu: 9.041860E+03 | global_batch_size: 64 | lm_loss: 1.080282E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 433931.906 [ip-26-0-154-121:0]:06/21/2023 18:02:29 [INFO|DP=0|PP=0|TP=0]: iteration: 3271 / 150000 | consumed_samples: 209344 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784296E+05 | tokens_per_sec_per_gpu: 9.037962E+03 | global_batch_size: 64 | lm_loss: 1.080277E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 407927.156 [ip-26-0-154-121:0]:06/21/2023 18:02:30 [INFO|DP=0|PP=0|TP=0]: iteration: 3272 / 150000 | consumed_samples: 209408 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778664E+05 | tokens_per_sec_per_gpu: 9.029163E+03 | global_batch_size: 64 | lm_loss: 1.080273E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 397681.875 [ip-26-0-154-121:0]:06/21/2023 18:02:31 [INFO|DP=0|PP=0|TP=0]: iteration: 3273 / 150000 | consumed_samples: 209472 | elapsed_time_per_iteration_ms: 915.5 | tokens_per_sec: 5.726948E+05 | tokens_per_sec_per_gpu: 8.948357E+03 | global_batch_size: 64 | lm_loss: 1.080273E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 103.36 | hardware_tflops_per_gpu: 103.36 | grad_norm: 355246.562 [ip-26-0-154-121:0]:06/21/2023 18:02:32 [INFO|DP=0|PP=0|TP=0]: iteration: 3274 / 150000 | consumed_samples: 209536 | elapsed_time_per_iteration_ms: 914.7 | tokens_per_sec: 5.731723E+05 | tokens_per_sec_per_gpu: 8.955818E+03 | global_batch_size: 64 | lm_loss: 1.080274E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 103.45 | hardware_tflops_per_gpu: 103.45 | grad_norm: 337554.062 [ip-26-0-154-121:0]:06/21/2023 18:02:33 [INFO|DP=0|PP=0|TP=0]: iteration: 3275 / 150000 | consumed_samples: 209600 | elapsed_time_per_iteration_ms: 928.3 | tokens_per_sec: 5.647768E+05 | tokens_per_sec_per_gpu: 8.824637E+03 | global_batch_size: 64 | lm_loss: 1.080280E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 101.93 | hardware_tflops_per_gpu: 101.93 | grad_norm: 347130.906 [ip-26-0-154-121:0]:06/21/2023 18:02:34 [INFO|DP=0|PP=0|TP=0]: iteration: 3276 / 150000 | consumed_samples: 209664 | elapsed_time_per_iteration_ms: 912.1 | tokens_per_sec: 5.748174E+05 | tokens_per_sec_per_gpu: 8.981522E+03 | global_batch_size: 64 | lm_loss: 1.080272E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 103.75 | hardware_tflops_per_gpu: 103.75 | grad_norm: 341953.500 [ip-26-0-154-121:0]:06/21/2023 18:02:35 [INFO|DP=0|PP=0|TP=0]: iteration: 3277 / 150000 | consumed_samples: 209728 | elapsed_time_per_iteration_ms: 913.8 | tokens_per_sec: 5.737277E+05 | tokens_per_sec_per_gpu: 8.964496E+03 | global_batch_size: 64 | lm_loss: 1.080274E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 103.55 | hardware_tflops_per_gpu: 103.55 | grad_norm: 326996.875 [ip-26-0-154-121:0]:06/21/2023 18:02:36 [INFO|DP=0|PP=0|TP=0]: iteration: 3278 / 150000 | consumed_samples: 209792 | elapsed_time_per_iteration_ms: 993.8 | tokens_per_sec: 5.275476E+05 | tokens_per_sec_per_gpu: 8.242932E+03 | global_batch_size: 64 | lm_loss: 1.080273E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 95.22 | hardware_tflops_per_gpu: 95.22 | grad_norm: 387129.844 [ip-26-0-154-121:0]:06/21/2023 18:02:37 [INFO|DP=0|PP=0|TP=0]: iteration: 3279 / 150000 | consumed_samples: 209856 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787264E+05 | tokens_per_sec_per_gpu: 9.042600E+03 | global_batch_size: 64 | lm_loss: 1.080277E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 426911.344 [ip-26-0-154-121:0]:06/21/2023 18:02:38 [INFO|DP=0|PP=0|TP=0]: iteration: 3280 / 150000 | consumed_samples: 209920 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788331E+05 | tokens_per_sec_per_gpu: 9.044267E+03 | global_batch_size: 64 | lm_loss: 1.080273E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 375128.281 [ip-26-0-154-121:0]:06/21/2023 18:02:39 [INFO|DP=0|PP=0|TP=0]: iteration: 3281 / 150000 | consumed_samples: 209984 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787970E+05 | tokens_per_sec_per_gpu: 9.043702E+03 | global_batch_size: 64 | lm_loss: 1.080272E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 341400.406 [ip-26-0-154-121:0]:06/21/2023 18:02:39 [INFO|DP=0|PP=0|TP=0]: iteration: 3282 / 150000 | consumed_samples: 210048 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785286E+05 | tokens_per_sec_per_gpu: 9.039510E+03 | global_batch_size: 64 | lm_loss: 1.080269E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 375276.594 [ip-26-0-154-121:0]:06/21/2023 18:02:40 [INFO|DP=0|PP=0|TP=0]: iteration: 3283 / 150000 | consumed_samples: 210112 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786003E+05 | tokens_per_sec_per_gpu: 9.040630E+03 | global_batch_size: 64 | lm_loss: 1.080269E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 335726.656 [ip-26-0-154-121:0]:06/21/2023 18:02:41 [INFO|DP=0|PP=0|TP=0]: iteration: 3284 / 150000 | consumed_samples: 210176 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788002E+05 | tokens_per_sec_per_gpu: 9.043752E+03 | global_batch_size: 64 | lm_loss: 1.080266E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 297550.812 [ip-26-0-154-121:0]:06/21/2023 18:02:42 [INFO|DP=0|PP=0|TP=0]: iteration: 3285 / 150000 | consumed_samples: 210240 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784941E+05 | tokens_per_sec_per_gpu: 9.038970E+03 | global_batch_size: 64 | lm_loss: 1.080271E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 299284.469 [ip-26-0-154-121:0]:06/21/2023 18:02:43 [INFO|DP=0|PP=0|TP=0]: iteration: 3286 / 150000 | consumed_samples: 210304 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787925E+05 | tokens_per_sec_per_gpu: 9.043633E+03 | global_batch_size: 64 | lm_loss: 1.080267E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 281138.625 [ip-26-0-154-121:0]:06/21/2023 18:02:44 [INFO|DP=0|PP=0|TP=0]: iteration: 3287 / 150000 | consumed_samples: 210368 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784650E+05 | tokens_per_sec_per_gpu: 9.038516E+03 | global_batch_size: 64 | lm_loss: 1.080264E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 278501.219 [ip-26-0-154-121:0]:06/21/2023 18:02:45 [INFO|DP=0|PP=0|TP=0]: iteration: 3288 / 150000 | consumed_samples: 210432 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787091E+05 | tokens_per_sec_per_gpu: 9.042329E+03 | global_batch_size: 64 | lm_loss: 1.080262E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 272525.625 [ip-26-0-154-121:0]:06/21/2023 18:02:46 [INFO|DP=0|PP=0|TP=0]: iteration: 3289 / 150000 | consumed_samples: 210496 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786198E+05 | tokens_per_sec_per_gpu: 9.040935E+03 | global_batch_size: 64 | lm_loss: 1.080263E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 272613.094 [ip-26-0-154-121:0]:06/21/2023 18:02:47 [INFO|DP=0|PP=0|TP=0]: iteration: 3290 / 150000 | consumed_samples: 210560 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.785904E+05 | tokens_per_sec_per_gpu: 9.040476E+03 | global_batch_size: 64 | lm_loss: 1.080266E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 274658.875 [ip-26-0-154-121:0]:06/21/2023 18:02:48 [INFO|DP=0|PP=0|TP=0]: iteration: 3291 / 150000 | consumed_samples: 210624 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790579E+05 | tokens_per_sec_per_gpu: 9.047779E+03 | global_batch_size: 64 | lm_loss: 1.080262E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 268706.062 [ip-26-0-154-121:0]:06/21/2023 18:02:48 [INFO|DP=0|PP=0|TP=0]: iteration: 3292 / 150000 | consumed_samples: 210688 | elapsed_time_per_iteration_ms: 905.4 | tokens_per_sec: 5.790704E+05 | tokens_per_sec_per_gpu: 9.047975E+03 | global_batch_size: 64 | lm_loss: 1.080261E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.51 | hardware_tflops_per_gpu: 104.51 | grad_norm: 327980.344 [ip-26-0-154-121:0]:06/21/2023 18:02:49 [INFO|DP=0|PP=0|TP=0]: iteration: 3293 / 150000 | consumed_samples: 210752 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787121E+05 | tokens_per_sec_per_gpu: 9.042377E+03 | global_batch_size: 64 | lm_loss: 1.080259E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 326832.344 [ip-26-0-154-121:0]:06/21/2023 18:02:50 [INFO|DP=0|PP=0|TP=0]: iteration: 3294 / 150000 | consumed_samples: 210816 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787925E+05 | tokens_per_sec_per_gpu: 9.043633E+03 | global_batch_size: 64 | lm_loss: 1.080265E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 300807.906 [ip-26-0-154-121:0]:06/21/2023 18:02:51 [INFO|DP=0|PP=0|TP=0]: iteration: 3295 / 150000 | consumed_samples: 210880 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785096E+05 | tokens_per_sec_per_gpu: 9.039213E+03 | global_batch_size: 64 | lm_loss: 1.080255E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 308244.594 [ip-26-0-154-121:0]:06/21/2023 18:02:52 [INFO|DP=0|PP=0|TP=0]: iteration: 3296 / 150000 | consumed_samples: 210944 | elapsed_time_per_iteration_ms: 912.5 | tokens_per_sec: 5.745741E+05 | tokens_per_sec_per_gpu: 8.977720E+03 | global_batch_size: 64 | lm_loss: 1.080258E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 103.70 | hardware_tflops_per_gpu: 103.70 | grad_norm: 354694.500 [ip-26-0-154-121:0]:06/21/2023 18:02:53 [INFO|DP=0|PP=0|TP=0]: iteration: 3297 / 150000 | consumed_samples: 211008 | elapsed_time_per_iteration_ms: 909.3 | tokens_per_sec: 5.765655E+05 | tokens_per_sec_per_gpu: 9.008836E+03 | global_batch_size: 64 | lm_loss: 1.080256E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.06 | hardware_tflops_per_gpu: 104.06 | grad_norm: 350602.438 [ip-26-0-154-121:0]:06/21/2023 18:02:54 [INFO|DP=0|PP=0|TP=0]: iteration: 3298 / 150000 | consumed_samples: 211072 | elapsed_time_per_iteration_ms: 906.8 | tokens_per_sec: 5.781964E+05 | tokens_per_sec_per_gpu: 9.034319E+03 | global_batch_size: 64 | lm_loss: 1.080256E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.36 | hardware_tflops_per_gpu: 104.36 | grad_norm: 361371.500 [ip-26-0-154-121:0]:06/21/2023 18:02:55 [INFO|DP=0|PP=0|TP=0]: iteration: 3299 / 150000 | consumed_samples: 211136 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788385E+05 | tokens_per_sec_per_gpu: 9.044352E+03 | global_batch_size: 64 | lm_loss: 1.080249E+01 | lr: 3.000E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 373472.438 [ip-26-0-154-121:0]:06/21/2023 18:02:56 [INFO|DP=0|PP=0|TP=0]: iteration: 3300 / 150000 | consumed_samples: 211200 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788224E+05 | tokens_per_sec_per_gpu: 9.044100E+03 | global_batch_size: 64 | lm_loss: 1.080255E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 372702.156 [ip-26-0-154-121:0]:06/21/2023 18:02:57 [INFO|DP=0|PP=0|TP=0]: iteration: 3301 / 150000 | consumed_samples: 211264 | elapsed_time_per_iteration_ms: 910.3 | tokens_per_sec: 5.759597E+05 | tokens_per_sec_per_gpu: 8.999370E+03 | global_batch_size: 64 | lm_loss: 1.080253E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 103.95 | hardware_tflops_per_gpu: 103.95 | grad_norm: 352659.750 [ip-26-0-154-121:0]:06/21/2023 18:02:58 [INFO|DP=0|PP=0|TP=0]: iteration: 3302 / 150000 | consumed_samples: 211328 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788931E+05 | tokens_per_sec_per_gpu: 9.045205E+03 | global_batch_size: 64 | lm_loss: 1.080250E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 312164.031 [ip-26-0-154-121:0]:06/21/2023 18:02:58 [INFO|DP=0|PP=0|TP=0]: iteration: 3303 / 150000 | consumed_samples: 211392 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784632E+05 | tokens_per_sec_per_gpu: 9.038488E+03 | global_batch_size: 64 | lm_loss: 1.080252E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 286715.375 [ip-26-0-154-121:0]:06/21/2023 18:02:59 [INFO|DP=0|PP=0|TP=0]: iteration: 3304 / 150000 | consumed_samples: 211456 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789192E+05 | tokens_per_sec_per_gpu: 9.045612E+03 | global_batch_size: 64 | lm_loss: 1.080250E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 279773.156 [ip-26-0-154-121:0]:06/21/2023 18:03:00 [INFO|DP=0|PP=0|TP=0]: iteration: 3305 / 150000 | consumed_samples: 211520 | elapsed_time_per_iteration_ms: 986.7 | tokens_per_sec: 5.313442E+05 | tokens_per_sec_per_gpu: 8.302253E+03 | global_batch_size: 64 | lm_loss: 1.080252E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 95.90 | hardware_tflops_per_gpu: 95.90 | grad_norm: 275516.281 [ip-26-0-154-121:0]:06/21/2023 18:03:01 [INFO|DP=0|PP=0|TP=0]: iteration: 3306 / 150000 | consumed_samples: 211584 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788673E+05 | tokens_per_sec_per_gpu: 9.044802E+03 | global_batch_size: 64 | lm_loss: 1.080252E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 276817.875 [ip-26-0-154-121:0]:06/21/2023 18:03:02 [INFO|DP=0|PP=0|TP=0]: iteration: 3307 / 150000 | consumed_samples: 211648 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786576E+05 | tokens_per_sec_per_gpu: 9.041525E+03 | global_batch_size: 64 | lm_loss: 1.080252E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 253467.609 [ip-26-0-154-121:0]:06/21/2023 18:03:03 [INFO|DP=0|PP=0|TP=0]: iteration: 3308 / 150000 | consumed_samples: 211712 | elapsed_time_per_iteration_ms: 973.3 | tokens_per_sec: 5.386844E+05 | tokens_per_sec_per_gpu: 8.416943E+03 | global_batch_size: 64 | lm_loss: 1.080246E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 97.23 | hardware_tflops_per_gpu: 97.23 | grad_norm: 254829.484 [ip-26-0-154-121:0]:06/21/2023 18:03:04 [INFO|DP=0|PP=0|TP=0]: iteration: 3309 / 150000 | consumed_samples: 211776 | elapsed_time_per_iteration_ms: 950.8 | tokens_per_sec: 5.513922E+05 | tokens_per_sec_per_gpu: 8.615503E+03 | global_batch_size: 64 | lm_loss: 1.080248E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 99.52 | hardware_tflops_per_gpu: 99.52 | grad_norm: 318518.094 [ip-26-0-154-121:0]:06/21/2023 18:03:05 [INFO|DP=0|PP=0|TP=0]: iteration: 3310 / 150000 | consumed_samples: 211840 | elapsed_time_per_iteration_ms: 926.7 | tokens_per_sec: 5.657778E+05 | tokens_per_sec_per_gpu: 8.840278E+03 | global_batch_size: 64 | lm_loss: 1.080241E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 102.12 | hardware_tflops_per_gpu: 102.12 | grad_norm: 285081.562 [ip-26-0-154-121:0]:06/21/2023 18:03:06 [INFO|DP=0|PP=0|TP=0]: iteration: 3311 / 150000 | consumed_samples: 211904 | elapsed_time_per_iteration_ms: 909.3 | tokens_per_sec: 5.765631E+05 | tokens_per_sec_per_gpu: 9.008799E+03 | global_batch_size: 64 | lm_loss: 1.080248E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.06 | hardware_tflops_per_gpu: 104.06 | grad_norm: 359838.125 [ip-26-0-154-121:0]:06/21/2023 18:03:07 [INFO|DP=0|PP=0|TP=0]: iteration: 3312 / 150000 | consumed_samples: 211968 | elapsed_time_per_iteration_ms: 940.1 | tokens_per_sec: 5.577131E+05 | tokens_per_sec_per_gpu: 8.714268E+03 | global_batch_size: 64 | lm_loss: 1.080247E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 100.66 | hardware_tflops_per_gpu: 100.66 | grad_norm: 566658.500 [ip-26-0-154-121:0]:06/21/2023 18:03:08 [INFO|DP=0|PP=0|TP=0]: iteration: 3313 / 150000 | consumed_samples: 212032 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789013E+05 | tokens_per_sec_per_gpu: 9.045333E+03 | global_batch_size: 64 | lm_loss: 1.080247E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 435318.406 [ip-26-0-154-121:0]:06/21/2023 18:03:09 [INFO|DP=0|PP=0|TP=0]: iteration: 3314 / 150000 | consumed_samples: 212096 | elapsed_time_per_iteration_ms: 921.5 | tokens_per_sec: 5.689347E+05 | tokens_per_sec_per_gpu: 8.889605E+03 | global_batch_size: 64 | lm_loss: 1.080243E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 102.68 | hardware_tflops_per_gpu: 102.68 | grad_norm: 350393.344 [ip-26-0-154-121:0]:06/21/2023 18:03:10 [INFO|DP=0|PP=0|TP=0]: iteration: 3315 / 150000 | consumed_samples: 212160 | elapsed_time_per_iteration_ms: 909.4 | tokens_per_sec: 5.765388E+05 | tokens_per_sec_per_gpu: 9.008418E+03 | global_batch_size: 64 | lm_loss: 1.080243E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.06 | hardware_tflops_per_gpu: 104.06 | grad_norm: 284827.656 [ip-26-0-154-121:0]:06/21/2023 18:03:11 [INFO|DP=0|PP=0|TP=0]: iteration: 3316 / 150000 | consumed_samples: 212224 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788340E+05 | tokens_per_sec_per_gpu: 9.044281E+03 | global_batch_size: 64 | lm_loss: 1.080243E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 351056.438 [ip-26-0-154-121:0]:06/21/2023 18:03:11 [INFO|DP=0|PP=0|TP=0]: iteration: 3317 / 150000 | consumed_samples: 212288 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789577E+05 | tokens_per_sec_per_gpu: 9.046214E+03 | global_batch_size: 64 | lm_loss: 1.080245E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 303949.562 [ip-26-0-154-121:0]:06/21/2023 18:03:12 [INFO|DP=0|PP=0|TP=0]: iteration: 3318 / 150000 | consumed_samples: 212352 | elapsed_time_per_iteration_ms: 920.4 | tokens_per_sec: 5.696613E+05 | tokens_per_sec_per_gpu: 8.900959E+03 | global_batch_size: 64 | lm_loss: 1.080243E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 102.82 | hardware_tflops_per_gpu: 102.82 | grad_norm: 291718.281 [ip-26-0-154-121:0]:06/21/2023 18:03:13 [INFO|DP=0|PP=0|TP=0]: iteration: 3319 / 150000 | consumed_samples: 212416 | elapsed_time_per_iteration_ms: 913.1 | tokens_per_sec: 5.741688E+05 | tokens_per_sec_per_gpu: 8.971387E+03 | global_batch_size: 64 | lm_loss: 1.080245E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 103.63 | hardware_tflops_per_gpu: 103.63 | grad_norm: 261332.047 [ip-26-0-154-121:0]:06/21/2023 18:03:14 [INFO|DP=0|PP=0|TP=0]: iteration: 3320 / 150000 | consumed_samples: 212480 | elapsed_time_per_iteration_ms: 914.9 | tokens_per_sec: 5.730328E+05 | tokens_per_sec_per_gpu: 8.953638E+03 | global_batch_size: 64 | lm_loss: 1.080242E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 103.42 | hardware_tflops_per_gpu: 103.42 | grad_norm: 268837.875 [ip-26-0-154-121:0]:06/21/2023 18:03:15 [INFO|DP=0|PP=0|TP=0]: iteration: 3321 / 150000 | consumed_samples: 212544 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785528E+05 | tokens_per_sec_per_gpu: 9.039888E+03 | global_batch_size: 64 | lm_loss: 1.080242E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 270731.469 [ip-26-0-154-121:0]:06/21/2023 18:03:16 [INFO|DP=0|PP=0|TP=0]: iteration: 3322 / 150000 | consumed_samples: 212608 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786870E+05 | tokens_per_sec_per_gpu: 9.041984E+03 | global_batch_size: 64 | lm_loss: 1.080239E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 260654.938 [ip-26-0-154-121:0]:06/21/2023 18:03:17 [INFO|DP=0|PP=0|TP=0]: iteration: 3323 / 150000 | consumed_samples: 212672 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789364E+05 | tokens_per_sec_per_gpu: 9.045881E+03 | global_batch_size: 64 | lm_loss: 1.080241E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 251035.250 [ip-26-0-154-121:0]:06/21/2023 18:03:18 [INFO|DP=0|PP=0|TP=0]: iteration: 3324 / 150000 | consumed_samples: 212736 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785437E+05 | tokens_per_sec_per_gpu: 9.039746E+03 | global_batch_size: 64 | lm_loss: 1.080241E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 244069.297 [ip-26-0-154-121:0]:06/21/2023 18:03:19 [INFO|DP=0|PP=0|TP=0]: iteration: 3325 / 150000 | consumed_samples: 212800 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788996E+05 | tokens_per_sec_per_gpu: 9.045307E+03 | global_batch_size: 64 | lm_loss: 1.080240E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 248603.422 [ip-26-0-154-121:0]:06/21/2023 18:03:20 [INFO|DP=0|PP=0|TP=0]: iteration: 3326 / 150000 | consumed_samples: 212864 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785038E+05 | tokens_per_sec_per_gpu: 9.039122E+03 | global_batch_size: 64 | lm_loss: 1.080238E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 263234.969 [ip-26-0-154-121:0]:06/21/2023 18:03:21 [INFO|DP=0|PP=0|TP=0]: iteration: 3327 / 150000 | consumed_samples: 212928 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787241E+05 | tokens_per_sec_per_gpu: 9.042565E+03 | global_batch_size: 64 | lm_loss: 1.080238E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 266953.375 [ip-26-0-154-121:0]:06/21/2023 18:03:21 [INFO|DP=0|PP=0|TP=0]: iteration: 3328 / 150000 | consumed_samples: 212992 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787424E+05 | tokens_per_sec_per_gpu: 9.042850E+03 | global_batch_size: 64 | lm_loss: 1.080235E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 253746.703 [ip-26-0-154-121:0]:06/21/2023 18:03:22 [INFO|DP=0|PP=0|TP=0]: iteration: 3329 / 150000 | consumed_samples: 213056 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786756E+05 | tokens_per_sec_per_gpu: 9.041806E+03 | global_batch_size: 64 | lm_loss: 1.080239E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 264842.250 [ip-26-0-154-121:0]:06/21/2023 18:03:23 [INFO|DP=0|PP=0|TP=0]: iteration: 3330 / 150000 | consumed_samples: 213120 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786725E+05 | tokens_per_sec_per_gpu: 9.041758E+03 | global_batch_size: 64 | lm_loss: 1.080236E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 295217.531 [ip-26-0-154-121:0]:06/21/2023 18:03:24 [INFO|DP=0|PP=0|TP=0]: iteration: 3331 / 150000 | consumed_samples: 213184 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789100E+05 | tokens_per_sec_per_gpu: 9.045469E+03 | global_batch_size: 64 | lm_loss: 1.080233E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 272774.156 [ip-26-0-154-121:0]:06/21/2023 18:03:25 [INFO|DP=0|PP=0|TP=0]: iteration: 3332 / 150000 | consumed_samples: 213248 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785417E+05 | tokens_per_sec_per_gpu: 9.039715E+03 | global_batch_size: 64 | lm_loss: 1.080235E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 273997.781 [ip-26-0-154-121:0]:06/21/2023 18:03:26 [INFO|DP=0|PP=0|TP=0]: iteration: 3333 / 150000 | consumed_samples: 213312 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784600E+05 | tokens_per_sec_per_gpu: 9.038438E+03 | global_batch_size: 64 | lm_loss: 1.080232E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 255821.375 [ip-26-0-154-121:0]:06/21/2023 18:03:27 [INFO|DP=0|PP=0|TP=0]: iteration: 3334 / 150000 | consumed_samples: 213376 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786478E+05 | tokens_per_sec_per_gpu: 9.041373E+03 | global_batch_size: 64 | lm_loss: 1.080235E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 267496.125 [ip-26-0-154-121:0]:06/21/2023 18:03:28 [INFO|DP=0|PP=0|TP=0]: iteration: 3335 / 150000 | consumed_samples: 213440 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786996E+05 | tokens_per_sec_per_gpu: 9.042182E+03 | global_batch_size: 64 | lm_loss: 1.080234E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 249962.531 [ip-26-0-154-121:0]:06/21/2023 18:03:29 [INFO|DP=0|PP=0|TP=0]: iteration: 3336 / 150000 | consumed_samples: 213504 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787813E+05 | tokens_per_sec_per_gpu: 9.043457E+03 | global_batch_size: 64 | lm_loss: 1.080231E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 286014.000 [ip-26-0-154-121:0]:06/21/2023 18:03:30 [INFO|DP=0|PP=0|TP=0]: iteration: 3337 / 150000 | consumed_samples: 213568 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787531E+05 | tokens_per_sec_per_gpu: 9.043017E+03 | global_batch_size: 64 | lm_loss: 1.080230E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 267353.031 [ip-26-0-154-121:0]:06/21/2023 18:03:31 [INFO|DP=0|PP=0|TP=0]: iteration: 3338 / 150000 | consumed_samples: 213632 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783490E+05 | tokens_per_sec_per_gpu: 9.036702E+03 | global_batch_size: 64 | lm_loss: 1.080231E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 258293.297 [ip-26-0-154-121:0]:06/21/2023 18:03:31 [INFO|DP=0|PP=0|TP=0]: iteration: 3339 / 150000 | consumed_samples: 213696 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786587E+05 | tokens_per_sec_per_gpu: 9.041542E+03 | global_batch_size: 64 | lm_loss: 1.080231E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 260531.250 [ip-26-0-154-121:0]:06/21/2023 18:03:32 [INFO|DP=0|PP=0|TP=0]: iteration: 3340 / 150000 | consumed_samples: 213760 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783234E+05 | tokens_per_sec_per_gpu: 9.036303E+03 | global_batch_size: 64 | lm_loss: 1.080233E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 251258.656 [ip-26-0-154-121:0]:06/21/2023 18:03:33 [INFO|DP=0|PP=0|TP=0]: iteration: 3341 / 150000 | consumed_samples: 213824 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778837E+05 | tokens_per_sec_per_gpu: 9.029433E+03 | global_batch_size: 64 | lm_loss: 1.080230E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 246206.984 [ip-26-0-154-121:0]:06/21/2023 18:03:34 [INFO|DP=0|PP=0|TP=0]: iteration: 3342 / 150000 | consumed_samples: 213888 | elapsed_time_per_iteration_ms: 907.1 | tokens_per_sec: 5.779518E+05 | tokens_per_sec_per_gpu: 9.030496E+03 | global_batch_size: 64 | lm_loss: 1.080232E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.31 | hardware_tflops_per_gpu: 104.31 | grad_norm: 294238.562 [ip-26-0-154-121:0]:06/21/2023 18:03:35 [INFO|DP=0|PP=0|TP=0]: iteration: 3343 / 150000 | consumed_samples: 213952 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.782715E+05 | tokens_per_sec_per_gpu: 9.035493E+03 | global_batch_size: 64 | lm_loss: 1.080233E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 243506.328 [ip-26-0-154-121:0]:06/21/2023 18:03:36 [INFO|DP=0|PP=0|TP=0]: iteration: 3344 / 150000 | consumed_samples: 214016 | elapsed_time_per_iteration_ms: 906.7 | tokens_per_sec: 5.782665E+05 | tokens_per_sec_per_gpu: 9.035414E+03 | global_batch_size: 64 | lm_loss: 1.080233E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.37 | hardware_tflops_per_gpu: 104.37 | grad_norm: 221643.547 [ip-26-0-154-121:0]:06/21/2023 18:03:37 [INFO|DP=0|PP=0|TP=0]: iteration: 3345 / 150000 | consumed_samples: 214080 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784090E+05 | tokens_per_sec_per_gpu: 9.037641E+03 | global_batch_size: 64 | lm_loss: 1.080236E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 219038.672 [ip-26-0-154-121:0]:06/21/2023 18:03:38 [INFO|DP=0|PP=0|TP=0]: iteration: 3346 / 150000 | consumed_samples: 214144 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791172E+05 | tokens_per_sec_per_gpu: 9.048706E+03 | global_batch_size: 64 | lm_loss: 1.080235E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 245568.062 [ip-26-0-154-121:0]:06/21/2023 18:03:39 [INFO|DP=0|PP=0|TP=0]: iteration: 3347 / 150000 | consumed_samples: 214208 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784909E+05 | tokens_per_sec_per_gpu: 9.038920E+03 | global_batch_size: 64 | lm_loss: 1.080233E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 270958.781 [ip-26-0-154-121:0]:06/21/2023 18:03:40 [INFO|DP=0|PP=0|TP=0]: iteration: 3348 / 150000 | consumed_samples: 214272 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786360E+05 | tokens_per_sec_per_gpu: 9.041187E+03 | global_batch_size: 64 | lm_loss: 1.080230E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 374802.844 [ip-26-0-154-121:0]:06/21/2023 18:03:41 [INFO|DP=0|PP=0|TP=0]: iteration: 3349 / 150000 | consumed_samples: 214336 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788070E+05 | tokens_per_sec_per_gpu: 9.043859E+03 | global_batch_size: 64 | lm_loss: 1.080231E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 357815.781 [ip-26-0-154-121:0]:06/21/2023 18:03:41 [INFO|DP=0|PP=0|TP=0]: iteration: 3350 / 150000 | consumed_samples: 214400 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788006E+05 | tokens_per_sec_per_gpu: 9.043759E+03 | global_batch_size: 64 | lm_loss: 1.080231E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 296731.844 [ip-26-0-154-121:0]:06/21/2023 18:03:42 [INFO|DP=0|PP=0|TP=0]: iteration: 3351 / 150000 | consumed_samples: 214464 | elapsed_time_per_iteration_ms: 907.3 | tokens_per_sec: 5.778450E+05 | tokens_per_sec_per_gpu: 9.028828E+03 | global_batch_size: 64 | lm_loss: 1.080232E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.29 | hardware_tflops_per_gpu: 104.29 | grad_norm: 324563.406 [ip-26-0-154-121:0]:06/21/2023 18:03:43 [INFO|DP=0|PP=0|TP=0]: iteration: 3352 / 150000 | consumed_samples: 214528 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787781E+05 | tokens_per_sec_per_gpu: 9.043407E+03 | global_batch_size: 64 | lm_loss: 1.080229E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 299230.562 [ip-26-0-154-121:0]:06/21/2023 18:03:44 [INFO|DP=0|PP=0|TP=0]: iteration: 3353 / 150000 | consumed_samples: 214592 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787549E+05 | tokens_per_sec_per_gpu: 9.043045E+03 | global_batch_size: 64 | lm_loss: 1.080230E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 253473.016 [ip-26-0-154-121:0]:06/21/2023 18:03:45 [INFO|DP=0|PP=0|TP=0]: iteration: 3354 / 150000 | consumed_samples: 214656 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.787970E+05 | tokens_per_sec_per_gpu: 9.043702E+03 | global_batch_size: 64 | lm_loss: 1.080231E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 287192.969 [ip-26-0-154-121:0]:06/21/2023 18:03:46 [INFO|DP=0|PP=0|TP=0]: iteration: 3355 / 150000 | consumed_samples: 214720 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786708E+05 | tokens_per_sec_per_gpu: 9.041732E+03 | global_batch_size: 64 | lm_loss: 1.080232E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 327967.156 [ip-26-0-154-121:0]:06/21/2023 18:03:47 [INFO|DP=0|PP=0|TP=0]: iteration: 3356 / 150000 | consumed_samples: 214784 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783941E+05 | tokens_per_sec_per_gpu: 9.037408E+03 | global_batch_size: 64 | lm_loss: 1.080232E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 320635.969 [ip-26-0-154-121:0]:06/21/2023 18:03:48 [INFO|DP=0|PP=0|TP=0]: iteration: 3357 / 150000 | consumed_samples: 214848 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785600E+05 | tokens_per_sec_per_gpu: 9.040000E+03 | global_batch_size: 64 | lm_loss: 1.080231E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 284814.281 [ip-26-0-154-121:0]:06/21/2023 18:03:49 [INFO|DP=0|PP=0|TP=0]: iteration: 3358 / 150000 | consumed_samples: 214912 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787442E+05 | tokens_per_sec_per_gpu: 9.042879E+03 | global_batch_size: 64 | lm_loss: 1.080233E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 304369.969 [ip-26-0-154-121:0]:06/21/2023 18:03:50 [INFO|DP=0|PP=0|TP=0]: iteration: 3359 / 150000 | consumed_samples: 214976 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787744E+05 | tokens_per_sec_per_gpu: 9.043350E+03 | global_batch_size: 64 | lm_loss: 1.080233E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 287744.125 [ip-26-0-154-121:0]:06/21/2023 18:03:51 [INFO|DP=0|PP=0|TP=0]: iteration: 3360 / 150000 | consumed_samples: 215040 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788064E+05 | tokens_per_sec_per_gpu: 9.043850E+03 | global_batch_size: 64 | lm_loss: 1.080233E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 296066.094 [ip-26-0-154-121:0]:06/21/2023 18:03:51 [INFO|DP=0|PP=0|TP=0]: iteration: 3361 / 150000 | consumed_samples: 215104 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785699E+05 | tokens_per_sec_per_gpu: 9.040155E+03 | global_batch_size: 64 | lm_loss: 1.080230E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 251944.469 [ip-26-0-154-121:0]:06/21/2023 18:03:52 [INFO|DP=0|PP=0|TP=0]: iteration: 3362 / 150000 | consumed_samples: 215168 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786451E+05 | tokens_per_sec_per_gpu: 9.041330E+03 | global_batch_size: 64 | lm_loss: 1.080232E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 259130.141 [ip-26-0-154-121:0]:06/21/2023 18:03:53 [INFO|DP=0|PP=0|TP=0]: iteration: 3363 / 150000 | consumed_samples: 215232 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787573E+05 | tokens_per_sec_per_gpu: 9.043084E+03 | global_batch_size: 64 | lm_loss: 1.080234E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 253367.172 [ip-26-0-154-121:0]:06/21/2023 18:03:54 [INFO|DP=0|PP=0|TP=0]: iteration: 3364 / 150000 | consumed_samples: 215296 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787343E+05 | tokens_per_sec_per_gpu: 9.042724E+03 | global_batch_size: 64 | lm_loss: 1.080233E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 264777.938 [ip-26-0-154-121:0]:06/21/2023 18:03:55 [INFO|DP=0|PP=0|TP=0]: iteration: 3365 / 150000 | consumed_samples: 215360 | elapsed_time_per_iteration_ms: 905.3 | tokens_per_sec: 5.791297E+05 | tokens_per_sec_per_gpu: 9.048902E+03 | global_batch_size: 64 | lm_loss: 1.080233E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.52 | hardware_tflops_per_gpu: 104.52 | grad_norm: 227524.656 [ip-26-0-154-121:0]:06/21/2023 18:03:56 [INFO|DP=0|PP=0|TP=0]: iteration: 3366 / 150000 | consumed_samples: 215424 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784752E+05 | tokens_per_sec_per_gpu: 9.038675E+03 | global_batch_size: 64 | lm_loss: 1.080234E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 198148.688 [ip-26-0-154-121:0]:06/21/2023 18:03:57 [INFO|DP=0|PP=0|TP=0]: iteration: 3367 / 150000 | consumed_samples: 215488 | elapsed_time_per_iteration_ms: 905.6 | tokens_per_sec: 5.789179E+05 | tokens_per_sec_per_gpu: 9.045593E+03 | global_batch_size: 64 | lm_loss: 1.080233E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.49 | hardware_tflops_per_gpu: 104.49 | grad_norm: 204202.375 [ip-26-0-154-121:0]:06/21/2023 18:03:58 [INFO|DP=0|PP=0|TP=0]: iteration: 3368 / 150000 | consumed_samples: 215552 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786512E+05 | tokens_per_sec_per_gpu: 9.041425E+03 | global_batch_size: 64 | lm_loss: 1.080234E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 249028.719 [ip-26-0-154-121:0]:06/21/2023 18:03:59 [INFO|DP=0|PP=0|TP=0]: iteration: 3369 / 150000 | consumed_samples: 215616 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784188E+05 | tokens_per_sec_per_gpu: 9.037793E+03 | global_batch_size: 64 | lm_loss: 1.080232E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 200881.344 [ip-26-0-154-121:0]:06/21/2023 18:04:00 [INFO|DP=0|PP=0|TP=0]: iteration: 3370 / 150000 | consumed_samples: 215680 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.787046E+05 | tokens_per_sec_per_gpu: 9.042260E+03 | global_batch_size: 64 | lm_loss: 1.080231E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 188995.547 [ip-26-0-154-121:0]:06/21/2023 18:04:00 [INFO|DP=0|PP=0|TP=0]: iteration: 3371 / 150000 | consumed_samples: 215744 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783368E+05 | tokens_per_sec_per_gpu: 9.036512E+03 | global_batch_size: 64 | lm_loss: 1.080233E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 214451.703 [ip-26-0-154-121:0]:06/21/2023 18:04:01 [INFO|DP=0|PP=0|TP=0]: iteration: 3372 / 150000 | consumed_samples: 215808 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783786E+05 | tokens_per_sec_per_gpu: 9.037166E+03 | global_batch_size: 64 | lm_loss: 1.080235E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 219428.297 [ip-26-0-154-121:0]:06/21/2023 18:04:02 [INFO|DP=0|PP=0|TP=0]: iteration: 3373 / 150000 | consumed_samples: 215872 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788561E+05 | tokens_per_sec_per_gpu: 9.044626E+03 | global_batch_size: 64 | lm_loss: 1.080233E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 222718.156 [ip-26-0-154-121:0]:06/21/2023 18:04:03 [INFO|DP=0|PP=0|TP=0]: iteration: 3374 / 150000 | consumed_samples: 215936 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784853E+05 | tokens_per_sec_per_gpu: 9.038832E+03 | global_batch_size: 64 | lm_loss: 1.080233E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 265393.312 [ip-26-0-154-121:0]:06/21/2023 18:04:04 [INFO|DP=0|PP=0|TP=0]: iteration: 3375 / 150000 | consumed_samples: 216000 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.783984E+05 | tokens_per_sec_per_gpu: 9.037475E+03 | global_batch_size: 64 | lm_loss: 1.080235E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 262376.375 [ip-26-0-154-121:0]:06/21/2023 18:04:05 [INFO|DP=0|PP=0|TP=0]: iteration: 3376 / 150000 | consumed_samples: 216064 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784223E+05 | tokens_per_sec_per_gpu: 9.037848E+03 | global_batch_size: 64 | lm_loss: 1.080243E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 292170.656 [ip-26-0-154-121:0]:06/21/2023 18:04:06 [INFO|DP=0|PP=0|TP=0]: iteration: 3377 / 150000 | consumed_samples: 216128 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783922E+05 | tokens_per_sec_per_gpu: 9.037377E+03 | global_batch_size: 64 | lm_loss: 1.080236E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 268038.844 [ip-26-0-154-121:0]:06/21/2023 18:04:07 [INFO|DP=0|PP=0|TP=0]: iteration: 3378 / 150000 | consumed_samples: 216192 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785868E+05 | tokens_per_sec_per_gpu: 9.040419E+03 | global_batch_size: 64 | lm_loss: 1.080215E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.43 | hardware_tflops_per_gpu: 104.43 | grad_norm: 261998.438 [ip-26-0-154-121:0]:06/21/2023 18:04:08 [INFO|DP=0|PP=0|TP=0]: iteration: 3379 / 150000 | consumed_samples: 216256 | elapsed_time_per_iteration_ms: 906.1 | tokens_per_sec: 5.786436E+05 | tokens_per_sec_per_gpu: 9.041306E+03 | global_batch_size: 64 | lm_loss: 1.080235E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 248368.078 [ip-26-0-154-121:0]:06/21/2023 18:04:09 [INFO|DP=0|PP=0|TP=0]: iteration: 3380 / 150000 | consumed_samples: 216320 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785224E+05 | tokens_per_sec_per_gpu: 9.039413E+03 | global_batch_size: 64 | lm_loss: 1.080237E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 232487.547 [ip-26-0-154-121:0]:06/21/2023 18:04:10 [INFO|DP=0|PP=0|TP=0]: iteration: 3381 / 150000 | consumed_samples: 216384 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783858E+05 | tokens_per_sec_per_gpu: 9.037278E+03 | global_batch_size: 64 | lm_loss: 1.080235E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 265647.969 [ip-26-0-154-121:0]:06/21/2023 18:04:10 [INFO|DP=0|PP=0|TP=0]: iteration: 3382 / 150000 | consumed_samples: 216448 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783782E+05 | tokens_per_sec_per_gpu: 9.037159E+03 | global_batch_size: 64 | lm_loss: 1.080236E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 230901.078 [ip-26-0-154-121:0]:06/21/2023 18:04:11 [INFO|DP=0|PP=0|TP=0]: iteration: 3383 / 150000 | consumed_samples: 216512 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783903E+05 | tokens_per_sec_per_gpu: 9.037349E+03 | global_batch_size: 64 | lm_loss: 1.080234E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 236572.062 [ip-26-0-154-121:0]:06/21/2023 18:04:12 [INFO|DP=0|PP=0|TP=0]: iteration: 3384 / 150000 | consumed_samples: 216576 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781058E+05 | tokens_per_sec_per_gpu: 9.032904E+03 | global_batch_size: 64 | lm_loss: 1.080235E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 209515.109 [ip-26-0-154-121:0]:06/21/2023 18:04:13 [INFO|DP=0|PP=0|TP=0]: iteration: 3385 / 150000 | consumed_samples: 216640 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784376E+05 | tokens_per_sec_per_gpu: 9.038088E+03 | global_batch_size: 64 | lm_loss: 1.080236E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 228517.484 [ip-26-0-154-121:0]:06/21/2023 18:04:14 [INFO|DP=0|PP=0|TP=0]: iteration: 3386 / 150000 | consumed_samples: 216704 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786829E+05 | tokens_per_sec_per_gpu: 9.041920E+03 | global_batch_size: 64 | lm_loss: 1.080234E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 247621.469 [ip-26-0-154-121:0]:06/21/2023 18:04:15 [INFO|DP=0|PP=0|TP=0]: iteration: 3387 / 150000 | consumed_samples: 216768 | elapsed_time_per_iteration_ms: 906.9 | tokens_per_sec: 5.781171E+05 | tokens_per_sec_per_gpu: 9.033079E+03 | global_batch_size: 64 | lm_loss: 1.080236E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.34 | hardware_tflops_per_gpu: 104.34 | grad_norm: 205328.953 [ip-26-0-154-121:0]:06/21/2023 18:04:16 [INFO|DP=0|PP=0|TP=0]: iteration: 3388 / 150000 | consumed_samples: 216832 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787695E+05 | tokens_per_sec_per_gpu: 9.043274E+03 | global_batch_size: 64 | lm_loss: 1.080234E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 217121.906 [ip-26-0-154-121:0]:06/21/2023 18:04:17 [INFO|DP=0|PP=0|TP=0]: iteration: 3389 / 150000 | consumed_samples: 216896 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784862E+05 | tokens_per_sec_per_gpu: 9.038847E+03 | global_batch_size: 64 | lm_loss: 1.080236E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 224868.391 [ip-26-0-154-121:0]:06/21/2023 18:04:18 [INFO|DP=0|PP=0|TP=0]: iteration: 3390 / 150000 | consumed_samples: 216960 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788686E+05 | tokens_per_sec_per_gpu: 9.044821E+03 | global_batch_size: 64 | lm_loss: 1.080237E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 227982.703 [ip-26-0-154-121:0]:06/21/2023 18:04:19 [INFO|DP=0|PP=0|TP=0]: iteration: 3391 / 150000 | consumed_samples: 217024 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786848E+05 | tokens_per_sec_per_gpu: 9.041951E+03 | global_batch_size: 64 | lm_loss: 1.080242E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 213279.859 [ip-26-0-154-121:0]:06/21/2023 18:04:20 [INFO|DP=0|PP=0|TP=0]: iteration: 3392 / 150000 | consumed_samples: 217088 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786731E+05 | tokens_per_sec_per_gpu: 9.041768E+03 | global_batch_size: 64 | lm_loss: 1.080237E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 205900.734 [ip-26-0-154-121:0]:06/21/2023 18:04:20 [INFO|DP=0|PP=0|TP=0]: iteration: 3393 / 150000 | consumed_samples: 217152 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784195E+05 | tokens_per_sec_per_gpu: 9.037805E+03 | global_batch_size: 64 | lm_loss: 1.080239E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 207889.562 [ip-26-0-154-121:0]:06/21/2023 18:04:21 [INFO|DP=0|PP=0|TP=0]: iteration: 3394 / 150000 | consumed_samples: 217216 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786797E+05 | tokens_per_sec_per_gpu: 9.041870E+03 | global_batch_size: 64 | lm_loss: 1.080238E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 226986.781 [ip-26-0-154-121:0]:06/21/2023 18:04:22 [INFO|DP=0|PP=0|TP=0]: iteration: 3395 / 150000 | consumed_samples: 217280 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785655E+05 | tokens_per_sec_per_gpu: 9.040086E+03 | global_batch_size: 64 | lm_loss: 1.080240E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 230638.453 [ip-26-0-154-121:0]:06/21/2023 18:04:23 [INFO|DP=0|PP=0|TP=0]: iteration: 3396 / 150000 | consumed_samples: 217344 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784577E+05 | tokens_per_sec_per_gpu: 9.038402E+03 | global_batch_size: 64 | lm_loss: 1.080240E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 225177.547 [ip-26-0-154-121:0]:06/21/2023 18:04:24 [INFO|DP=0|PP=0|TP=0]: iteration: 3397 / 150000 | consumed_samples: 217408 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783756E+05 | tokens_per_sec_per_gpu: 9.037118E+03 | global_batch_size: 64 | lm_loss: 1.080241E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 228314.344 [ip-26-0-154-121:0]:06/21/2023 18:04:25 [INFO|DP=0|PP=0|TP=0]: iteration: 3398 / 150000 | consumed_samples: 217472 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784416E+05 | tokens_per_sec_per_gpu: 9.038150E+03 | global_batch_size: 64 | lm_loss: 1.080240E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 219838.562 [ip-26-0-154-121:0]:06/21/2023 18:04:26 [INFO|DP=0|PP=0|TP=0]: iteration: 3399 / 150000 | consumed_samples: 217536 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.784813E+05 | tokens_per_sec_per_gpu: 9.038771E+03 | global_batch_size: 64 | lm_loss: 1.080243E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 284624.312 [ip-26-0-154-121:0]:06/21/2023 18:04:27 [INFO|DP=0|PP=0|TP=0]: iteration: 3400 / 150000 | consumed_samples: 217600 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786809E+05 | tokens_per_sec_per_gpu: 9.041889E+03 | global_batch_size: 64 | lm_loss: 1.080243E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 296685.969 [ip-26-0-154-121:0]:06/21/2023 18:04:28 [INFO|DP=0|PP=0|TP=0]: iteration: 3401 / 150000 | consumed_samples: 217664 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787290E+05 | tokens_per_sec_per_gpu: 9.042641E+03 | global_batch_size: 64 | lm_loss: 1.080241E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.45 | hardware_tflops_per_gpu: 104.45 | grad_norm: 261668.125 [ip-26-0-154-121:0]:06/21/2023 18:04:29 [INFO|DP=0|PP=0|TP=0]: iteration: 3402 / 150000 | consumed_samples: 217728 | elapsed_time_per_iteration_ms: 906.0 | tokens_per_sec: 5.786666E+05 | tokens_per_sec_per_gpu: 9.041665E+03 | global_batch_size: 64 | lm_loss: 1.080245E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.44 | hardware_tflops_per_gpu: 104.44 | grad_norm: 323844.719 [ip-26-0-154-121:0]:06/21/2023 18:04:30 [INFO|DP=0|PP=0|TP=0]: iteration: 3403 / 150000 | consumed_samples: 217792 | elapsed_time_per_iteration_ms: 905.8 | tokens_per_sec: 5.788245E+05 | tokens_per_sec_per_gpu: 9.044133E+03 | global_batch_size: 64 | lm_loss: 1.080241E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.47 | hardware_tflops_per_gpu: 104.47 | grad_norm: 343353.562 [ip-26-0-154-121:0]:06/21/2023 18:04:30 [INFO|DP=0|PP=0|TP=0]: iteration: 3404 / 150000 | consumed_samples: 217856 | elapsed_time_per_iteration_ms: 907.2 | tokens_per_sec: 5.778900E+05 | tokens_per_sec_per_gpu: 9.029531E+03 | global_batch_size: 64 | lm_loss: 1.080243E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.30 | hardware_tflops_per_gpu: 104.30 | grad_norm: 309084.125 [ip-26-0-154-121:0]:06/21/2023 18:04:31 [INFO|DP=0|PP=0|TP=0]: iteration: 3405 / 150000 | consumed_samples: 217920 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784192E+05 | tokens_per_sec_per_gpu: 9.037801E+03 | global_batch_size: 64 | lm_loss: 1.080243E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 265440.094 [ip-26-0-154-121:0]:06/21/2023 18:04:32 [INFO|DP=0|PP=0|TP=0]: iteration: 3406 / 150000 | consumed_samples: 217984 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785527E+05 | tokens_per_sec_per_gpu: 9.039886E+03 | global_batch_size: 64 | lm_loss: 1.080241E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 215289.516 [ip-26-0-154-121:0]:06/21/2023 18:04:33 [INFO|DP=0|PP=0|TP=0]: iteration: 3407 / 150000 | consumed_samples: 218048 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785550E+05 | tokens_per_sec_per_gpu: 9.039922E+03 | global_batch_size: 64 | lm_loss: 1.080243E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 211289.859 [ip-26-0-154-121:0]:06/21/2023 18:04:34 [INFO|DP=0|PP=0|TP=0]: iteration: 3408 / 150000 | consumed_samples: 218112 | elapsed_time_per_iteration_ms: 906.3 | tokens_per_sec: 5.785128E+05 | tokens_per_sec_per_gpu: 9.039263E+03 | global_batch_size: 64 | lm_loss: 1.080244E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.41 | hardware_tflops_per_gpu: 104.41 | grad_norm: 230493.266 [ip-26-0-154-121:0]:06/21/2023 18:04:35 [INFO|DP=0|PP=0|TP=0]: iteration: 3409 / 150000 | consumed_samples: 218176 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787560E+05 | tokens_per_sec_per_gpu: 9.043062E+03 | global_batch_size: 64 | lm_loss: 1.080241E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 201328.828 [ip-26-0-154-121:0]:06/21/2023 18:04:36 [INFO|DP=0|PP=0|TP=0]: iteration: 3410 / 150000 | consumed_samples: 218240 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787785E+05 | tokens_per_sec_per_gpu: 9.043414E+03 | global_batch_size: 64 | lm_loss: 1.080241E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 183576.734 [ip-26-0-154-121:0]:06/21/2023 18:04:37 [INFO|DP=0|PP=0|TP=0]: iteration: 3411 / 150000 | consumed_samples: 218304 | elapsed_time_per_iteration_ms: 905.9 | tokens_per_sec: 5.787630E+05 | tokens_per_sec_per_gpu: 9.043172E+03 | global_batch_size: 64 | lm_loss: 1.080243E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.46 | hardware_tflops_per_gpu: 104.46 | grad_norm: 446203.031 [ip-26-0-154-121:0]:06/21/2023 18:04:38 [INFO|DP=0|PP=0|TP=0]: iteration: 3412 / 150000 | consumed_samples: 218368 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.788760E+05 | tokens_per_sec_per_gpu: 9.044938E+03 | global_batch_size: 64 | lm_loss: 1.080244E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 838851.562 [ip-26-0-154-121:0]:06/21/2023 18:04:39 [INFO|DP=0|PP=0|TP=0]: iteration: 3413 / 150000 | consumed_samples: 218432 | elapsed_time_per_iteration_ms: 906.2 | tokens_per_sec: 5.785455E+05 | tokens_per_sec_per_gpu: 9.039774E+03 | global_batch_size: 64 | lm_loss: 1.080243E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.42 | hardware_tflops_per_gpu: 104.42 | grad_norm: 743680.812 [ip-26-0-154-121:0]:06/21/2023 18:04:40 [INFO|DP=0|PP=0|TP=0]: iteration: 3414 / 150000 | consumed_samples: 218496 | elapsed_time_per_iteration_ms: 905.7 | tokens_per_sec: 5.789047E+05 | tokens_per_sec_per_gpu: 9.045386E+03 | global_batch_size: 64 | lm_loss: 1.080242E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.48 | hardware_tflops_per_gpu: 104.48 | grad_norm: 506483.219 [ip-26-0-154-121:0]:06/21/2023 18:04:40 [INFO|DP=0|PP=0|TP=0]: iteration: 3415 / 150000 | consumed_samples: 218560 | elapsed_time_per_iteration_ms: 906.6 | tokens_per_sec: 5.783201E+05 | tokens_per_sec_per_gpu: 9.036251E+03 | global_batch_size: 64 | lm_loss: 1.080243E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 359536.969 [ip-26-0-154-121:0]:06/21/2023 18:04:41 [INFO|DP=0|PP=0|TP=0]: iteration: 3416 / 150000 | consumed_samples: 218624 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783966E+05 | tokens_per_sec_per_gpu: 9.037446E+03 | global_batch_size: 64 | lm_loss: 1.080242E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.39 | hardware_tflops_per_gpu: 104.39 | grad_norm: 334761.312 [ip-26-0-154-121:0]:06/21/2023 18:04:42 [INFO|DP=0|PP=0|TP=0]: iteration: 3417 / 150000 | consumed_samples: 218688 | elapsed_time_per_iteration_ms: 906.5 | tokens_per_sec: 5.783406E+05 | tokens_per_sec_per_gpu: 9.036572E+03 | global_batch_size: 64 | lm_loss: 1.080244E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.38 | hardware_tflops_per_gpu: 104.38 | grad_norm: 306155.438 [ip-26-0-154-121:0]:06/21/2023 18:04:43 [INFO|DP=0|PP=0|TP=0]: iteration: 3418 / 150000 | consumed_samples: 218752 | elapsed_time_per_iteration_ms: 906.4 | tokens_per_sec: 5.784591E+05 | tokens_per_sec_per_gpu: 9.038423E+03 | global_batch_size: 64 | lm_loss: 1.080244E+01 | lr: 2.999E-04 | model_tflops_per_gpu: 104.40 | hardware_tflops_per_gpu: 104.40 | grad_norm: 273224.594 slurmstepd: error: *** JOB 161646 ON ip-26-0-154-80 CANCELLED AT 2023-06-21T18:04:43 *** slurmstepd: error: *** STEP 161646.0 ON ip-26-0-154-80 CANCELLED AT 2023-06-21T18:04:43 *** srun: Job step aborted: Waiting up to 32 seconds for job step to finish.