Training in progress, step 18000
Browse files- model.safetensors +1 -1
- wandb/debug-internal.log +0 -0
- wandb/run-20231119_105908-nwx0xdmm/files/output.log +1057 -0
- wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +1 -1
- wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log +0 -0
- wandb/run-20231119_105908-nwx0xdmm/run-nwx0xdmm.wandb +2 -2
model.safetensors
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 3219908024
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:1d63b3b7c1076ec597dcdcc6541ece450e677504bd6df3bf6b6d0efdff7161ba
|
3 |
size 3219908024
|
wandb/debug-internal.log
CHANGED
The diff for this file is too large to render.
See raw diff
|
|
wandb/run-20231119_105908-nwx0xdmm/files/output.log
CHANGED
@@ -10796,3 +10796,1060 @@ Reading metadata...: 1704it [00:00, 2782.72it/s]
|
|
10796 |
|
10797 |
|
10798 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
10796 |
|
10797 |
|
10798 |
|
10799 |
+
|
10800 |
+
|
10801 |
+
|
10802 |
+
|
10803 |
+
|
10804 |
+
|
10805 |
+
|
10806 |
+
|
10807 |
+
|
10808 |
+
|
10809 |
+
|
10810 |
+
|
10811 |
+
|
10812 |
+
|
10813 |
+
|
10814 |
+
|
10815 |
+
28%|ββββββββββββββββββββββββββ | 17079/60000 [32:19:07<129:01:51, 10.82s/it]
|
10816 |
+
|
10817 |
+
|
10818 |
+
|
10819 |
+
|
10820 |
+
|
10821 |
+
|
10822 |
+
|
10823 |
+
|
10824 |
+
|
10825 |
+
|
10826 |
+
|
10827 |
+
|
10828 |
+
|
10829 |
+
|
10830 |
+
|
10831 |
+
|
10832 |
+
|
10833 |
+
|
10834 |
+
|
10835 |
+
|
10836 |
+
|
10837 |
+
28%|ββββββββββββββββββββββββββ | 17100/60000 [32:23:09<119:08:31, 10.00s/it]
|
10838 |
+
|
10839 |
+
|
10840 |
+
|
10841 |
+
|
10842 |
+
|
10843 |
+
|
10844 |
+
|
10845 |
+
|
10846 |
+
|
10847 |
+
|
10848 |
+
|
10849 |
+
|
10850 |
+
|
10851 |
+
|
10852 |
+
|
10853 |
+
|
10854 |
+
|
10855 |
+
|
10856 |
+
|
10857 |
+
|
10858 |
+
29%|ββββββββββββββββββββββββββ | 17120/60000 [32:26:39<126:38:31, 10.63s/it]
|
10859 |
+
|
10860 |
+
|
10861 |
+
|
10862 |
+
|
10863 |
+
|
10864 |
+
|
10865 |
+
|
10866 |
+
|
10867 |
+
|
10868 |
+
|
10869 |
+
|
10870 |
+
|
10871 |
+
|
10872 |
+
|
10873 |
+
|
10874 |
+
|
10875 |
+
|
10876 |
+
|
10877 |
+
|
10878 |
+
29%|ββββββββββββββββββββββββββ | 17139/60000 [32:30:02<126:25:22, 10.62s/it]
|
10879 |
+
|
10880 |
+
|
10881 |
+
|
10882 |
+
|
10883 |
+
|
10884 |
+
|
10885 |
+
|
10886 |
+
|
10887 |
+
|
10888 |
+
|
10889 |
+
|
10890 |
+
|
10891 |
+
|
10892 |
+
|
10893 |
+
|
10894 |
+
|
10895 |
+
|
10896 |
+
|
10897 |
+
|
10898 |
+
|
10899 |
+
|
10900 |
+
29%|ββββββββββββββββββββββββββ | 17160/60000 [32:33:43<121:42:38, 10.23s/it]
|
10901 |
+
|
10902 |
+
|
10903 |
+
|
10904 |
+
|
10905 |
+
|
10906 |
+
|
10907 |
+
|
10908 |
+
|
10909 |
+
|
10910 |
+
|
10911 |
+
|
10912 |
+
|
10913 |
+
|
10914 |
+
|
10915 |
+
|
10916 |
+
|
10917 |
+
|
10918 |
+
|
10919 |
+
|
10920 |
+
29%|ββββββββββββββββββββββββββ | 17179/60000 [32:37:14<129:43:00, 10.91s/it]
|
10921 |
+
|
10922 |
+
|
10923 |
+
|
10924 |
+
|
10925 |
+
|
10926 |
+
|
10927 |
+
|
10928 |
+
|
10929 |
+
|
10930 |
+
|
10931 |
+
|
10932 |
+
|
10933 |
+
|
10934 |
+
|
10935 |
+
|
10936 |
+
|
10937 |
+
|
10938 |
+
|
10939 |
+
|
10940 |
+
|
10941 |
+
|
10942 |
+
29%|ββββββββββββββββββββββββββ | 17200/60000 [32:40:58<126:19:25, 10.63s/it]
|
10943 |
+
|
10944 |
+
|
10945 |
+
|
10946 |
+
|
10947 |
+
|
10948 |
+
|
10949 |
+
|
10950 |
+
|
10951 |
+
|
10952 |
+
|
10953 |
+
|
10954 |
+
|
10955 |
+
|
10956 |
+
|
10957 |
+
|
10958 |
+
|
10959 |
+
|
10960 |
+
|
10961 |
+
|
10962 |
+
|
10963 |
+
29%|ββββββββββββββββββββββββββ | 17220/60000 [32:44:48<121:41:20, 10.24s/it]
|
10964 |
+
|
10965 |
+
|
10966 |
+
|
10967 |
+
|
10968 |
+
|
10969 |
+
|
10970 |
+
|
10971 |
+
|
10972 |
+
|
10973 |
+
|
10974 |
+
|
10975 |
+
|
10976 |
+
|
10977 |
+
|
10978 |
+
|
10979 |
+
|
10980 |
+
|
10981 |
+
|
10982 |
+
|
10983 |
+
|
10984 |
+
29%|βββββββββββββββββββββββββββ | 17240/60000 [32:48:10<121:17:48, 10.21s/it]
|
10985 |
+
|
10986 |
+
|
10987 |
+
|
10988 |
+
|
10989 |
+
|
10990 |
+
|
10991 |
+
|
10992 |
+
|
10993 |
+
|
10994 |
+
|
10995 |
+
|
10996 |
+
|
10997 |
+
|
10998 |
+
|
10999 |
+
|
11000 |
+
|
11001 |
+
|
11002 |
+
|
11003 |
+
|
11004 |
+
|
11005 |
+
29%|βββββββββββββββββββββββββββ | 17260/60000 [32:51:46<133:43:41, 11.26s/it]
|
11006 |
+
|
11007 |
+
|
11008 |
+
|
11009 |
+
|
11010 |
+
|
11011 |
+
|
11012 |
+
|
11013 |
+
|
11014 |
+
|
11015 |
+
|
11016 |
+
|
11017 |
+
|
11018 |
+
|
11019 |
+
|
11020 |
+
|
11021 |
+
|
11022 |
+
|
11023 |
+
|
11024 |
+
|
11025 |
+
|
11026 |
+
29%|βββββββββββββββββββββββββββ | 17280/60000 [32:55:19<124:13:09, 10.47s/it]
|
11027 |
+
|
11028 |
+
|
11029 |
+
|
11030 |
+
|
11031 |
+
|
11032 |
+
|
11033 |
+
|
11034 |
+
|
11035 |
+
|
11036 |
+
|
11037 |
+
|
11038 |
+
|
11039 |
+
|
11040 |
+
|
11041 |
+
|
11042 |
+
|
11043 |
+
|
11044 |
+
|
11045 |
+
|
11046 |
+
|
11047 |
+
29%|βββββββββββββββββββββββββββ | 17300/60000 [32:58:50<115:22:34, 9.73s/it]
|
11048 |
+
|
11049 |
+
|
11050 |
+
|
11051 |
+
|
11052 |
+
|
11053 |
+
|
11054 |
+
|
11055 |
+
|
11056 |
+
|
11057 |
+
|
11058 |
+
|
11059 |
+
|
11060 |
+
|
11061 |
+
|
11062 |
+
|
11063 |
+
|
11064 |
+
|
11065 |
+
|
11066 |
+
|
11067 |
+
29%|βββββββββββββββββββββββββββ | 17319/60000 [33:01:58<119:29:26, 10.08s/it]
|
11068 |
+
|
11069 |
+
|
11070 |
+
|
11071 |
+
|
11072 |
+
|
11073 |
+
|
11074 |
+
|
11075 |
+
|
11076 |
+
|
11077 |
+
|
11078 |
+
|
11079 |
+
|
11080 |
+
|
11081 |
+
|
11082 |
+
|
11083 |
+
|
11084 |
+
|
11085 |
+
|
11086 |
+
|
11087 |
+
|
11088 |
+
|
11089 |
+
29%|βββββββββββββββββββββββββββ | 17340/60000 [33:05:27<116:57:49, 9.87s/it]
|
11090 |
+
|
11091 |
+
|
11092 |
+
|
11093 |
+
|
11094 |
+
|
11095 |
+
|
11096 |
+
|
11097 |
+
|
11098 |
+
|
11099 |
+
|
11100 |
+
|
11101 |
+
|
11102 |
+
|
11103 |
+
|
11104 |
+
|
11105 |
+
|
11106 |
+
|
11107 |
+
|
11108 |
+
|
11109 |
+
|
11110 |
+
29%|βββββββββββββββββββββββββββ | 17360/60000 [33:08:51<117:19:02, 9.90s/it]
|
11111 |
+
|
11112 |
+
|
11113 |
+
|
11114 |
+
|
11115 |
+
|
11116 |
+
|
11117 |
+
|
11118 |
+
|
11119 |
+
|
11120 |
+
|
11121 |
+
|
11122 |
+
|
11123 |
+
|
11124 |
+
|
11125 |
+
|
11126 |
+
|
11127 |
+
|
11128 |
+
|
11129 |
+
|
11130 |
+
29%|βββββββββββββββββββββββββββ | 17379/60000 [33:11:59<116:31:25, 9.84s/it]
|
11131 |
+
|
11132 |
+
|
11133 |
+
|
11134 |
+
|
11135 |
+
|
11136 |
+
|
11137 |
+
|
11138 |
+
|
11139 |
+
|
11140 |
+
|
11141 |
+
|
11142 |
+
|
11143 |
+
|
11144 |
+
|
11145 |
+
|
11146 |
+
|
11147 |
+
|
11148 |
+
|
11149 |
+
|
11150 |
+
|
11151 |
+
29%|βββββββββββββββββββββββββββ | 17399/60000 [33:15:15<115:06:32, 9.73s/it]
|
11152 |
+
|
11153 |
+
|
11154 |
+
|
11155 |
+
|
11156 |
+
|
11157 |
+
|
11158 |
+
|
11159 |
+
|
11160 |
+
|
11161 |
+
|
11162 |
+
|
11163 |
+
|
11164 |
+
|
11165 |
+
|
11166 |
+
|
11167 |
+
|
11168 |
+
|
11169 |
+
|
11170 |
+
|
11171 |
+
|
11172 |
+
29%|βββββββββββββββββββββββββββ | 17419/60000 [33:18:36<116:15:55, 9.83s/it]
|
11173 |
+
|
11174 |
+
|
11175 |
+
|
11176 |
+
|
11177 |
+
|
11178 |
+
|
11179 |
+
|
11180 |
+
|
11181 |
+
|
11182 |
+
|
11183 |
+
|
11184 |
+
|
11185 |
+
|
11186 |
+
|
11187 |
+
|
11188 |
+
|
11189 |
+
|
11190 |
+
|
11191 |
+
|
11192 |
+
|
11193 |
+
29%|βββββββββββββββββββββββββββ | 17439/60000 [33:22:01<122:21:14, 10.35s/it]
|
11194 |
+
|
11195 |
+
|
11196 |
+
|
11197 |
+
|
11198 |
+
|
11199 |
+
|
11200 |
+
|
11201 |
+
|
11202 |
+
|
11203 |
+
|
11204 |
+
|
11205 |
+
|
11206 |
+
|
11207 |
+
|
11208 |
+
|
11209 |
+
|
11210 |
+
|
11211 |
+
|
11212 |
+
|
11213 |
+
|
11214 |
+
29%|βββββββββββββββββββββββββββ | 17459/60000 [33:25:20<119:52:56, 10.14s/it]
|
11215 |
+
|
11216 |
+
|
11217 |
+
|
11218 |
+
|
11219 |
+
|
11220 |
+
|
11221 |
+
|
11222 |
+
|
11223 |
+
|
11224 |
+
|
11225 |
+
|
11226 |
+
|
11227 |
+
|
11228 |
+
|
11229 |
+
|
11230 |
+
|
11231 |
+
|
11232 |
+
|
11233 |
+
|
11234 |
+
|
11235 |
+
29%|βββββββββββββββββββββββββββ | 17479/60000 [33:28:38<117:50:22, 9.98s/it]
|
11236 |
+
|
11237 |
+
|
11238 |
+
|
11239 |
+
|
11240 |
+
|
11241 |
+
|
11242 |
+
|
11243 |
+
|
11244 |
+
|
11245 |
+
|
11246 |
+
|
11247 |
+
|
11248 |
+
|
11249 |
+
|
11250 |
+
|
11251 |
+
|
11252 |
+
|
11253 |
+
|
11254 |
+
|
11255 |
+
|
11256 |
+
29%|βββββββββββββββββββββββββββ | 17499/60000 [33:31:59<117:25:52, 9.95s/it]
|
11257 |
+
|
11258 |
+
|
11259 |
+
|
11260 |
+
|
11261 |
+
|
11262 |
+
|
11263 |
+
|
11264 |
+
|
11265 |
+
|
11266 |
+
|
11267 |
+
|
11268 |
+
|
11269 |
+
|
11270 |
+
|
11271 |
+
|
11272 |
+
|
11273 |
+
|
11274 |
+
|
11275 |
+
|
11276 |
+
|
11277 |
+
29%|βββββββββββββββββββββββββββ | 17519/60000 [33:35:21<118:30:37, 10.04s/it]
|
11278 |
+
|
11279 |
+
|
11280 |
+
|
11281 |
+
|
11282 |
+
|
11283 |
+
|
11284 |
+
|
11285 |
+
|
11286 |
+
|
11287 |
+
|
11288 |
+
|
11289 |
+
|
11290 |
+
|
11291 |
+
|
11292 |
+
|
11293 |
+
|
11294 |
+
|
11295 |
+
|
11296 |
+
|
11297 |
+
|
11298 |
+
|
11299 |
+
29%|βββββββββββββββββββββββββββ | 17540/60000 [33:38:54<118:32:10, 10.05s/it]
|
11300 |
+
|
11301 |
+
|
11302 |
+
|
11303 |
+
|
11304 |
+
|
11305 |
+
|
11306 |
+
|
11307 |
+
|
11308 |
+
|
11309 |
+
|
11310 |
+
|
11311 |
+
|
11312 |
+
|
11313 |
+
|
11314 |
+
|
11315 |
+
|
11316 |
+
|
11317 |
+
|
11318 |
+
|
11319 |
+
29%|βββββββββββββββββββββββββββ | 17559/60000 [33:42:04<117:47:53, 9.99s/it]
|
11320 |
+
|
11321 |
+
|
11322 |
+
|
11323 |
+
|
11324 |
+
|
11325 |
+
|
11326 |
+
|
11327 |
+
|
11328 |
+
|
11329 |
+
|
11330 |
+
|
11331 |
+
|
11332 |
+
|
11333 |
+
|
11334 |
+
|
11335 |
+
|
11336 |
+
|
11337 |
+
|
11338 |
+
|
11339 |
+
|
11340 |
+
29%|βββββββββββββββββββββββββββ | 17579/60000 [33:45:19<114:13:38, 9.69s/it]
|
11341 |
+
|
11342 |
+
|
11343 |
+
|
11344 |
+
|
11345 |
+
|
11346 |
+
|
11347 |
+
|
11348 |
+
|
11349 |
+
|
11350 |
+
|
11351 |
+
|
11352 |
+
|
11353 |
+
|
11354 |
+
|
11355 |
+
|
11356 |
+
|
11357 |
+
|
11358 |
+
|
11359 |
+
|
11360 |
+
|
11361 |
+
29%|βββββββββββββββββββββββββββ | 17599/60000 [33:48:37<114:49:50, 9.75s/it]
|
11362 |
+
|
11363 |
+
|
11364 |
+
|
11365 |
+
|
11366 |
+
|
11367 |
+
|
11368 |
+
|
11369 |
+
|
11370 |
+
|
11371 |
+
|
11372 |
+
|
11373 |
+
|
11374 |
+
|
11375 |
+
|
11376 |
+
|
11377 |
+
|
11378 |
+
|
11379 |
+
|
11380 |
+
|
11381 |
+
|
11382 |
+
|
11383 |
+
29%|βββββββββββββββββββββββββββ | 17620/60000 [33:53:09<136:30:35, 11.60s/it]
|
11384 |
+
|
11385 |
+
|
11386 |
+
|
11387 |
+
|
11388 |
+
|
11389 |
+
|
11390 |
+
|
11391 |
+
|
11392 |
+
|
11393 |
+
|
11394 |
+
|
11395 |
+
|
11396 |
+
|
11397 |
+
|
11398 |
+
|
11399 |
+
|
11400 |
+
|
11401 |
+
|
11402 |
+
|
11403 |
+
29%|βββββββββββββββββββββββββββ | 17639/60000 [33:56:29<115:37:18, 9.83s/it]
|
11404 |
+
|
11405 |
+
|
11406 |
+
|
11407 |
+
|
11408 |
+
|
11409 |
+
|
11410 |
+
|
11411 |
+
|
11412 |
+
|
11413 |
+
|
11414 |
+
|
11415 |
+
|
11416 |
+
|
11417 |
+
|
11418 |
+
|
11419 |
+
|
11420 |
+
|
11421 |
+
|
11422 |
+
|
11423 |
+
|
11424 |
+
29%|βββββββββββββββββββββββββββ | 17659/60000 [33:59:47<117:19:13, 9.98s/it]
|
11425 |
+
|
11426 |
+
|
11427 |
+
|
11428 |
+
|
11429 |
+
|
11430 |
+
|
11431 |
+
|
11432 |
+
|
11433 |
+
|
11434 |
+
|
11435 |
+
|
11436 |
+
|
11437 |
+
|
11438 |
+
|
11439 |
+
|
11440 |
+
|
11441 |
+
|
11442 |
+
|
11443 |
+
|
11444 |
+
|
11445 |
+
|
11446 |
+
29%|βββββββββββββββββββββββββββ | 17680/60000 [34:03:21<115:27:02, 9.82s/it]
|
11447 |
+
|
11448 |
+
|
11449 |
+
|
11450 |
+
|
11451 |
+
|
11452 |
+
|
11453 |
+
|
11454 |
+
|
11455 |
+
|
11456 |
+
|
11457 |
+
|
11458 |
+
|
11459 |
+
|
11460 |
+
|
11461 |
+
|
11462 |
+
|
11463 |
+
|
11464 |
+
|
11465 |
+
|
11466 |
+
|
11467 |
+
30%|βββββββββββββββββββββββββββ | 17700/60000 [34:06:49<135:01:39, 11.49s/it]
|
11468 |
+
|
11469 |
+
|
11470 |
+
|
11471 |
+
|
11472 |
+
|
11473 |
+
|
11474 |
+
|
11475 |
+
|
11476 |
+
|
11477 |
+
|
11478 |
+
|
11479 |
+
|
11480 |
+
|
11481 |
+
|
11482 |
+
|
11483 |
+
|
11484 |
+
|
11485 |
+
|
11486 |
+
|
11487 |
+
|
11488 |
+
30%|βββββββββββββββββββββββββββ | 17720/60000 [34:10:08<114:22:05, 9.74s/it]
|
11489 |
+
|
11490 |
+
|
11491 |
+
|
11492 |
+
|
11493 |
+
|
11494 |
+
|
11495 |
+
|
11496 |
+
|
11497 |
+
|
11498 |
+
|
11499 |
+
|
11500 |
+
|
11501 |
+
|
11502 |
+
|
11503 |
+
|
11504 |
+
|
11505 |
+
|
11506 |
+
|
11507 |
+
|
11508 |
+
30%|βββββββββββββββββββββββββββ | 17739/60000 [34:13:17<117:01:45, 9.97s/it]
|
11509 |
+
|
11510 |
+
|
11511 |
+
|
11512 |
+
|
11513 |
+
|
11514 |
+
|
11515 |
+
|
11516 |
+
|
11517 |
+
|
11518 |
+
|
11519 |
+
|
11520 |
+
|
11521 |
+
|
11522 |
+
|
11523 |
+
|
11524 |
+
|
11525 |
+
|
11526 |
+
|
11527 |
+
|
11528 |
+
|
11529 |
+
30%|βββββββββββββββββββββββββββ | 17759/60000 [34:16:37<118:57:18, 10.14s/it]
|
11530 |
+
|
11531 |
+
|
11532 |
+
|
11533 |
+
|
11534 |
+
|
11535 |
+
|
11536 |
+
|
11537 |
+
|
11538 |
+
|
11539 |
+
|
11540 |
+
|
11541 |
+
|
11542 |
+
|
11543 |
+
|
11544 |
+
|
11545 |
+
|
11546 |
+
|
11547 |
+
|
11548 |
+
|
11549 |
+
|
11550 |
+
30%|βββββββββββββββββββββββββββ | 17779/60000 [34:19:56<115:35:07, 9.86s/it]
|
11551 |
+
|
11552 |
+
|
11553 |
+
|
11554 |
+
|
11555 |
+
|
11556 |
+
|
11557 |
+
|
11558 |
+
|
11559 |
+
|
11560 |
+
|
11561 |
+
Reading metadata...: 1650it [00:00, 9802.57it/s] | 17790/60000 [34:21:51<122:47:12, 10.47s/it]
|
11562 |
+
|
11563 |
+
|
11564 |
+
|
11565 |
+
|
11566 |
+
|
11567 |
+
|
11568 |
+
|
11569 |
+
|
11570 |
+
|
11571 |
+
30%|βββββββββββββββββββββββββββ | 17799/60000 [34:23:21<114:52:46, 9.80s/it]
|
11572 |
+
|
11573 |
+
|
11574 |
+
|
11575 |
+
|
11576 |
+
|
11577 |
+
|
11578 |
+
|
11579 |
+
|
11580 |
+
|
11581 |
+
|
11582 |
+
|
11583 |
+
|
11584 |
+
|
11585 |
+
|
11586 |
+
|
11587 |
+
|
11588 |
+
|
11589 |
+
|
11590 |
+
|
11591 |
+
|
11592 |
+
|
11593 |
+
30%|βββββββββββββββββββββββββββ | 17820/60000 [34:27:33<130:34:55, 11.14s/it]
|
11594 |
+
|
11595 |
+
|
11596 |
+
|
11597 |
+
|
11598 |
+
|
11599 |
+
|
11600 |
+
|
11601 |
+
|
11602 |
+
|
11603 |
+
|
11604 |
+
|
11605 |
+
|
11606 |
+
|
11607 |
+
|
11608 |
+
|
11609 |
+
|
11610 |
+
|
11611 |
+
|
11612 |
+
|
11613 |
+
|
11614 |
+
30%|βββββββββββββββββββββββββββ | 17840/60000 [34:31:07<123:18:54, 10.53s/it]
|
11615 |
+
|
11616 |
+
|
11617 |
+
|
11618 |
+
|
11619 |
+
|
11620 |
+
|
11621 |
+
|
11622 |
+
|
11623 |
+
|
11624 |
+
|
11625 |
+
|
11626 |
+
|
11627 |
+
|
11628 |
+
|
11629 |
+
|
11630 |
+
|
11631 |
+
|
11632 |
+
|
11633 |
+
|
11634 |
+
30%|βββββββββββββββββββββββββββ | 17859/60000 [34:34:29<124:59:35, 10.68s/it]
|
11635 |
+
|
11636 |
+
|
11637 |
+
|
11638 |
+
|
11639 |
+
|
11640 |
+
|
11641 |
+
|
11642 |
+
|
11643 |
+
|
11644 |
+
|
11645 |
+
|
11646 |
+
|
11647 |
+
|
11648 |
+
|
11649 |
+
|
11650 |
+
|
11651 |
+
|
11652 |
+
|
11653 |
+
|
11654 |
+
|
11655 |
+
30%|βββββββββββββββββββββββββββ | 17879/60000 [34:37:57<122:33:15, 10.47s/it]
|
11656 |
+
|
11657 |
+
|
11658 |
+
|
11659 |
+
|
11660 |
+
|
11661 |
+
|
11662 |
+
|
11663 |
+
|
11664 |
+
|
11665 |
+
|
11666 |
+
|
11667 |
+
|
11668 |
+
|
11669 |
+
|
11670 |
+
|
11671 |
+
|
11672 |
+
|
11673 |
+
|
11674 |
+
|
11675 |
+
|
11676 |
+
|
11677 |
+
30%|ββββββββββββββββββββββββββββ | 17900/60000 [34:41:40<125:00:50, 10.69s/it]
|
11678 |
+
|
11679 |
+
|
11680 |
+
|
11681 |
+
|
11682 |
+
|
11683 |
+
|
11684 |
+
|
11685 |
+
|
11686 |
+
|
11687 |
+
|
11688 |
+
|
11689 |
+
|
11690 |
+
|
11691 |
+
|
11692 |
+
|
11693 |
+
|
11694 |
+
|
11695 |
+
|
11696 |
+
|
11697 |
+
|
11698 |
+
30%|ββββββββββββοΏ½οΏ½οΏ½βββββββββββββββ | 17920/60000 [34:45:28<132:19:19, 11.32s/it]
|
11699 |
+
|
11700 |
+
|
11701 |
+
|
11702 |
+
|
11703 |
+
|
11704 |
+
|
11705 |
+
|
11706 |
+
|
11707 |
+
|
11708 |
+
|
11709 |
+
|
11710 |
+
|
11711 |
+
|
11712 |
+
|
11713 |
+
|
11714 |
+
|
11715 |
+
|
11716 |
+
|
11717 |
+
|
11718 |
+
|
11719 |
+
30%|ββββββββββββββββββββββββββββ | 17940/60000 [34:49:09<129:39:29, 11.10s/it]
|
11720 |
+
|
11721 |
+
|
11722 |
+
|
11723 |
+
|
11724 |
+
|
11725 |
+
|
11726 |
+
|
11727 |
+
|
11728 |
+
|
11729 |
+
|
11730 |
+
|
11731 |
+
|
11732 |
+
|
11733 |
+
|
11734 |
+
|
11735 |
+
|
11736 |
+
|
11737 |
+
|
11738 |
+
|
11739 |
+
|
11740 |
+
30%|ββββββββββββββββββββββββββββ | 17960/60000 [34:53:03<130:33:00, 11.18s/it]
|
11741 |
+
|
11742 |
+
|
11743 |
+
|
11744 |
+
|
11745 |
+
|
11746 |
+
|
11747 |
+
|
11748 |
+
|
11749 |
+
|
11750 |
+
|
11751 |
+
|
11752 |
+
|
11753 |
+
|
11754 |
+
|
11755 |
+
|
11756 |
+
|
11757 |
+
|
11758 |
+
|
11759 |
+
|
11760 |
+
30%|ββββββββββββββββββββββββββββ | 17979/60000 [34:56:34<125:20:14, 10.74s/it]
|
11761 |
+
|
11762 |
+
|
11763 |
+
|
11764 |
+
|
11765 |
+
|
11766 |
+
|
11767 |
+
|
11768 |
+
|
11769 |
+
|
11770 |
+
|
11771 |
+
|
11772 |
+
|
11773 |
+
|
11774 |
+
|
11775 |
+
|
11776 |
+
|
11777 |
+
|
11778 |
+
|
11779 |
+
|
11780 |
+
|
11781 |
+
30%|ββββββββββββββββββββββββββββ | 17999/60000 [35:00:17<133:47:48, 11.47s/it]
|
11782 |
+
30%|ββββββββββββββββββββββββββββ | 18000/60000 [35:00:28<133:47:03, 11.47s/it][INFO|trainer.py:3173] 2023-11-20 21:59:43,380 >> ***** Running Evaluation *****
|
11783 |
+
[INFO|trainer.py:3177] 2023-11-20 21:59:43,387 >> Num examples: Unknown
|
11784 |
+
[INFO|trainer.py:3178] 2023-11-20 21:59:43,387 >> Batch size = 4
|
11785 |
+
|
11786 |
+
|
11787 |
+
[INFO|trainer_utils.py:759] 2023-11-20 21:59:47,673 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale. If client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale are not expected by `WhisperForConditionalGeneration.forward`, you can safely ignore this message.
|
11788 |
+
{'eval_loss': 0.1827392578125, 'eval_wer': 8.678854127402941, 'eval_runtime': 734.6885, 'eval_samples_per_second': 2.319, 'eval_steps_per_second': 0.58, 'epoch': 0.3}
|
11789 |
+
30%|ββββββββββββββββββββββββββββ | 18000/60000 [35:12:43<133:47:03, 11.47s/it][INFO|trainer.py:2896] 2023-11-20 22:12:31,234 >> Saving model checkpoint to ./checkpoint-18000
|
11790 |
+
[INFO|configuration_utils.py:462] 2023-11-20 22:12:31,268 >> Configuration saved in ./checkpoint-18000/config.json
|
11791 |
+
[INFO|configuration_utils.py:568] 2023-11-20 22:12:31,286 >> Configuration saved in ./checkpoint-18000/generation_config.json
|
11792 |
+
[2023-11-20 22:13:42,614] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step18000 is about to be saved!
|
11793 |
+
[2023-11-20 22:13:42,781] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-18000/global_step18000/mp_rank_00_model_states.pt
|
11794 |
+
[2023-11-20 22:13:42,781] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-18000/global_step18000/mp_rank_00_model_states.pt...
|
11795 |
+
[INFO|modeling_utils.py:2194] 2023-11-20 22:13:42,402 >> Model weights saved in ./checkpoint-18000/pytorch_model.bin
|
11796 |
+
[INFO|feature_extraction_utils.py:425] 2023-11-20 22:13:42,438 >> Feature extractor saved in ./checkpoint-18000/preprocessor_config.json
|
11797 |
+
[2023-11-20 22:13:58,098] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-18000/global_step18000/mp_rank_00_model_states.pt.
|
11798 |
+
[2023-11-20 22:13:58,157] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-18000/global_step18000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
|
11799 |
+
[2023-11-20 22:15:54,672] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-18000/global_step18000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
|
11800 |
+
[2023-11-20 22:15:54,682] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-18000/global_step18000/zero_pp_rank_0_mp_rank_00_optim_states.pt
|
11801 |
+
[2023-11-20 22:15:54,683] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step18000 is ready now!
|
11802 |
+
[INFO|feature_extraction_utils.py:425] 2023-11-20 22:17:07,828 >> Feature extractor saved in ./preprocessor_config.json
|
11803 |
+
|
11804 |
+
|
11805 |
+
|
11806 |
+
|
11807 |
+
|
11808 |
+
|
11809 |
+
|
11810 |
+
|
11811 |
+
|
11812 |
+
|
11813 |
+
|
11814 |
+
|
11815 |
+
|
11816 |
+
|
11817 |
+
|
11818 |
+
|
11819 |
+
|
11820 |
+
|
11821 |
+
30%|ββββββββββββββββββββββββββββ | 18019/60000 [35:22:13<147:24:35, 12.64s/it]
|
11822 |
+
|
11823 |
+
|
11824 |
+
|
11825 |
+
|
11826 |
+
|
11827 |
+
|
11828 |
+
|
11829 |
+
|
11830 |
+
|
11831 |
+
|
11832 |
+
|
11833 |
+
|
11834 |
+
|
11835 |
+
|
11836 |
+
|
11837 |
+
|
11838 |
+
|
11839 |
+
|
11840 |
+
|
11841 |
+
|
11842 |
+
30%|ββββββββββββββββββββββββββββ | 18039/60000 [35:26:16<145:51:45, 12.51s/it]
|
11843 |
+
|
11844 |
+
|
11845 |
+
|
11846 |
+
|
11847 |
+
|
11848 |
+
|
11849 |
+
|
11850 |
+
|
11851 |
+
|
11852 |
+
|
11853 |
+
|
11854 |
+
|
11855 |
+
|
wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json
CHANGED
@@ -1 +1 @@
|
|
1 |
-
{"train/loss": 0.
|
|
|
1 |
+
{"train/loss": 0.1355, "train/learning_rate": 2.1349322033898305e-06, "train/epoch": 0.3, "train/global_step": 18040, "_timestamp": 1700511965.7339036, "_runtime": 127616.6998167038, "_step": 562, "eval/loss": 0.1827392578125, "eval/wer": 8.678854127402941, "eval/runtime": 734.6885, "eval/samples_per_second": 2.319, "eval/steps_per_second": 0.58}
|
wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log
CHANGED
The diff for this file is too large to render.
See raw diff
|
|
wandb/run-20231119_105908-nwx0xdmm/run-nwx0xdmm.wandb
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:c661509eff524fa6fc61166ff8fb70b3921709b9feb3fff3ae66325b5480e695
|
3 |
+
size 8003044
|