Saving weights and logs of step 75000
Browse filesThis view is limited to 50 files because it contains too many changes.
See raw diff
- .gitattributes +1 -1
- 100k_flax_model.msgpack +3 -0
- 50k_steps_flax_model.msgpack +3 -0
- checkpoint_100000 → checkpoint_75000 +1 -1
- events.out.tfevents.1636318087.t1v-n-ba7aa1a5-w-0.20231.0.v2 +2 -2
- events.out.tfevents.1636357015.t1v-n-ba7aa1a5-w-0.29588.0.v2 +3 -0
- events.out.tfevents.1636358735.t1v-n-ba7aa1a5-w-0.3899.0.v2 +3 -0
- events.out.tfevents.1636358897.t1v-n-ba7aa1a5-w-0.5478.0.v2 +3 -0
- events.out.tfevents.1636358979.t1v-n-ba7aa1a5-w-0.6978.0.v2 +3 -0
- events.out.tfevents.1636359665.t1v-n-ba7aa1a5-w-0.3923.0.v2 +3 -0
- file_order +325 -0
- flax_model.msgpack +1 -1
- muril-bigbird/flax_model.msgpack +1 -1
- run_muril_bb_flax.sh +4 -3
- run_muril_bb_flax_mlm.py +1 -1
- wandb/debug-internal.log +1 -1
- wandb/debug.log +1 -1
- wandb/latest-run +1 -1
- wandb/run-20211107_204808-2ns3lo5n/files/config.yaml +6 -0
- wandb/run-20211107_204808-2ns3lo5n/files/output.log +5414 -0
- wandb/run-20211107_204808-2ns3lo5n/files/wandb-summary.json +1 -1
- wandb/run-20211107_204808-2ns3lo5n/logs/debug-internal.log +0 -0
- wandb/run-20211107_204808-2ns3lo5n/logs/debug.log +122 -0
- wandb/run-20211107_204808-2ns3lo5n/run-2ns3lo5n.wandb +0 -0
- wandb/run-20211108_073656-hyqam6zi/files/config.yaml +347 -0
- wandb/run-20211108_073656-hyqam6zi/files/output.log +15 -0
- wandb/run-20211108_073656-hyqam6zi/files/requirements.txt +101 -0
- wandb/run-20211108_073656-hyqam6zi/files/wandb-metadata.json +69 -0
- wandb/run-20211108_073656-hyqam6zi/files/wandb-summary.json +1 -0
- wandb/run-20211108_073656-hyqam6zi/logs/debug-internal.log +145 -0
- wandb/run-20211108_073656-hyqam6zi/logs/debug.log +129 -0
- wandb/run-20211108_073656-hyqam6zi/run-hyqam6zi.wandb +0 -0
- wandb/run-20211108_080537-1gq01aek/files/config.yaml +347 -0
- wandb/run-20211108_080537-1gq01aek/files/output.log +16 -0
- wandb/run-20211108_080537-1gq01aek/files/requirements.txt +101 -0
- wandb/run-20211108_080537-1gq01aek/files/wandb-metadata.json +69 -0
- wandb/run-20211108_080537-1gq01aek/files/wandb-summary.json +1 -0
- wandb/run-20211108_080537-1gq01aek/logs/debug-internal.log +159 -0
- wandb/run-20211108_080537-1gq01aek/logs/debug.log +137 -0
- wandb/run-20211108_080537-1gq01aek/run-1gq01aek.wandb +0 -0
- wandb/run-20211108_080817-3dz8poa1/files/config.yaml +347 -0
- wandb/run-20211108_080817-3dz8poa1/files/output.log +120 -0
- wandb/run-20211108_080817-3dz8poa1/files/requirements.txt +101 -0
- wandb/run-20211108_080817-3dz8poa1/files/wandb-metadata.json +69 -0
- wandb/run-20211108_080817-3dz8poa1/files/wandb-summary.json +1 -0
- wandb/run-20211108_080817-3dz8poa1/logs/debug-internal.log +142 -0
- wandb/run-20211108_080817-3dz8poa1/logs/debug.log +129 -0
- wandb/run-20211108_080817-3dz8poa1/run-3dz8poa1.wandb +0 -0
- wandb/run-20211108_080940-r077t6tw/files/config.yaml +347 -0
- wandb/run-20211108_080940-r077t6tw/files/output.log +108 -0
.gitattributes
CHANGED
@@ -25,4 +25,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
25 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
26 |
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
27 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
28 |
-
|
|
|
25 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
26 |
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
27 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
28 |
+
checkpoint_75000 filter=lfs diff=lfs merge=lfs -text
|
100k_flax_model.msgpack
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:9a27a51818882c80dc60c00a6758070d78bd85e45a66c91a5c31cf6f491516c7
|
3 |
+
size 962037499
|
50k_steps_flax_model.msgpack
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:92b4eb4b41a4fdd29fa296ca969236da8c1c0c923dab34bfff0448484f2a8228
|
3 |
+
size 962037499
|
checkpoint_100000 → checkpoint_75000
RENAMED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 2886112612
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:c386829fc369ca03d6970bef9037c52c9956540ef0a3585392813a01e37be848
|
3 |
size 2886112612
|
events.out.tfevents.1636318087.t1v-n-ba7aa1a5-w-0.20231.0.v2
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:dbdb3cab3b1246125c212bd6a06cf039f52a25061a02bbfc94a2bf6f91ecbd55
|
3 |
+
size 10628868
|
events.out.tfevents.1636357015.t1v-n-ba7aa1a5-w-0.29588.0.v2
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:4a4801ba99bdb8213d87b8d4f451b1c0149720e5c96f5efd5fd5752302afd7a4
|
3 |
+
size 40
|
events.out.tfevents.1636358735.t1v-n-ba7aa1a5-w-0.3899.0.v2
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:07930aa307734cd757b8ae37bcff122d28ce01678833a1835e60f45715d53ea1
|
3 |
+
size 40
|
events.out.tfevents.1636358897.t1v-n-ba7aa1a5-w-0.5478.0.v2
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:28340cce916b145b1c5caa26857d4ff0c75aa4dd29667cd2a7fe510949a01db6
|
3 |
+
size 40
|
events.out.tfevents.1636358979.t1v-n-ba7aa1a5-w-0.6978.0.v2
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:6673d0330fb01bef206c1e839b76ade7ce1265b2ddf6e8f8f7cafd23136f88d8
|
3 |
+
size 40
|
events.out.tfevents.1636359665.t1v-n-ba7aa1a5-w-0.3923.0.v2
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:320e33fc26d13a5019c71f012f5d91e73064e4518fda8c450a307606d34d4b0f
|
3 |
+
size 3727500
|
file_order
ADDED
@@ -0,0 +1,325 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
/dev/shm/data-files/telugu-data/9.parquet
|
2 |
+
/dev/shm/data-files/telugu-data/8.parquet
|
3 |
+
/dev/shm/data-files/telugu-data/7.parquet
|
4 |
+
/dev/shm/data-files/telugu-data/6.parquet
|
5 |
+
/dev/shm/data-files/telugu-data/5.parquet
|
6 |
+
/dev/shm/data-files/telugu-data/4.parquet
|
7 |
+
/dev/shm/data-files/telugu-data/3.parquet
|
8 |
+
/dev/shm/data-files/telugu-data/23.parquet
|
9 |
+
/dev/shm/data-files/telugu-data/22.parquet
|
10 |
+
/dev/shm/data-files/telugu-data/21.parquet
|
11 |
+
/dev/shm/data-files/telugu-data/20.parquet
|
12 |
+
/dev/shm/data-files/telugu-data/2.parquet
|
13 |
+
/dev/shm/data-files/telugu-data/19.parquet
|
14 |
+
/dev/shm/data-files/telugu-data/18.parquet
|
15 |
+
/dev/shm/data-files/telugu-data/17.parquet
|
16 |
+
/dev/shm/data-files/telugu-data/16.parquet
|
17 |
+
/dev/shm/data-files/telugu-data/15.parquet
|
18 |
+
/dev/shm/data-files/telugu-data/14.parquet
|
19 |
+
/dev/shm/data-files/telugu-data/13.parquet
|
20 |
+
/dev/shm/data-files/telugu-data/12.parquet
|
21 |
+
/dev/shm/data-files/telugu-data/11.parquet
|
22 |
+
/dev/shm/data-files/telugu-data/10.parquet
|
23 |
+
/dev/shm/data-files/telugu-data/1.parquet
|
24 |
+
/dev/shm/data-files/telugu-data/0.parquet
|
25 |
+
/dev/shm/data-files/tamil-data/9.parquet
|
26 |
+
/dev/shm/data-files/tamil-data/8.parquet
|
27 |
+
/dev/shm/data-files/tamil-data/7.parquet
|
28 |
+
/dev/shm/data-files/tamil-data/65.parquet
|
29 |
+
/dev/shm/data-files/tamil-data/64.parquet
|
30 |
+
/dev/shm/data-files/tamil-data/63.parquet
|
31 |
+
/dev/shm/data-files/tamil-data/62.parquet
|
32 |
+
/dev/shm/data-files/tamil-data/61.parquet
|
33 |
+
/dev/shm/data-files/tamil-data/60.parquet
|
34 |
+
/dev/shm/data-files/tamil-data/6.parquet
|
35 |
+
/dev/shm/data-files/tamil-data/59.parquet
|
36 |
+
/dev/shm/data-files/tamil-data/58.parquet
|
37 |
+
/dev/shm/data-files/tamil-data/57.parquet
|
38 |
+
/dev/shm/data-files/tamil-data/56.parquet
|
39 |
+
/dev/shm/data-files/tamil-data/55.parquet
|
40 |
+
/dev/shm/data-files/tamil-data/54.parquet
|
41 |
+
/dev/shm/data-files/tamil-data/53.parquet
|
42 |
+
/dev/shm/data-files/tamil-data/52.parquet
|
43 |
+
/dev/shm/data-files/tamil-data/51.parquet
|
44 |
+
/dev/shm/data-files/tamil-data/50.parquet
|
45 |
+
/dev/shm/data-files/tamil-data/5.parquet
|
46 |
+
/dev/shm/data-files/tamil-data/49.parquet
|
47 |
+
/dev/shm/data-files/tamil-data/48.parquet
|
48 |
+
/dev/shm/data-files/tamil-data/47.parquet
|
49 |
+
/dev/shm/data-files/tamil-data/46.parquet
|
50 |
+
/dev/shm/data-files/tamil-data/45.parquet
|
51 |
+
/dev/shm/data-files/tamil-data/44.parquet
|
52 |
+
/dev/shm/data-files/tamil-data/43.parquet
|
53 |
+
/dev/shm/data-files/tamil-data/42.parquet
|
54 |
+
/dev/shm/data-files/tamil-data/41.parquet
|
55 |
+
/dev/shm/data-files/tamil-data/40.parquet
|
56 |
+
/dev/shm/data-files/tamil-data/4.parquet
|
57 |
+
/dev/shm/data-files/tamil-data/39.parquet
|
58 |
+
/dev/shm/data-files/tamil-data/38.parquet
|
59 |
+
/dev/shm/data-files/tamil-data/37.parquet
|
60 |
+
/dev/shm/data-files/tamil-data/36.parquet
|
61 |
+
/dev/shm/data-files/tamil-data/35.parquet
|
62 |
+
/dev/shm/data-files/tamil-data/34.parquet
|
63 |
+
/dev/shm/data-files/tamil-data/33.parquet
|
64 |
+
/dev/shm/data-files/tamil-data/32.parquet
|
65 |
+
/dev/shm/data-files/tamil-data/31.parquet
|
66 |
+
/dev/shm/data-files/tamil-data/30.parquet
|
67 |
+
/dev/shm/data-files/tamil-data/3.parquet
|
68 |
+
/dev/shm/data-files/tamil-data/29.parquet
|
69 |
+
/dev/shm/data-files/tamil-data/28.parquet
|
70 |
+
/dev/shm/data-files/tamil-data/27.parquet
|
71 |
+
/dev/shm/data-files/tamil-data/26.parquet
|
72 |
+
/dev/shm/data-files/tamil-data/25.parquet
|
73 |
+
/dev/shm/data-files/tamil-data/24.parquet
|
74 |
+
/dev/shm/data-files/tamil-data/23.parquet
|
75 |
+
/dev/shm/data-files/tamil-data/22.parquet
|
76 |
+
/dev/shm/data-files/tamil-data/21.parquet
|
77 |
+
/dev/shm/data-files/tamil-data/20.parquet
|
78 |
+
/dev/shm/data-files/tamil-data/2.parquet
|
79 |
+
/dev/shm/data-files/tamil-data/19.parquet
|
80 |
+
/dev/shm/data-files/tamil-data/18.parquet
|
81 |
+
/dev/shm/data-files/tamil-data/17.parquet
|
82 |
+
/dev/shm/data-files/tamil-data/16.parquet
|
83 |
+
/dev/shm/data-files/tamil-data/15.parquet
|
84 |
+
/dev/shm/data-files/tamil-data/14.parquet
|
85 |
+
/dev/shm/data-files/tamil-data/13.parquet
|
86 |
+
/dev/shm/data-files/tamil-data/12.parquet
|
87 |
+
/dev/shm/data-files/tamil-data/11.parquet
|
88 |
+
/dev/shm/data-files/tamil-data/10.parquet
|
89 |
+
/dev/shm/data-files/tamil-data/1.parquet
|
90 |
+
/dev/shm/data-files/tamil-data/0.parquet
|
91 |
+
/dev/shm/data-files/hindi-data3/99.parquet
|
92 |
+
/dev/shm/data-files/hindi-data3/98.parquet
|
93 |
+
/dev/shm/data-files/hindi-data3/97.parquet
|
94 |
+
/dev/shm/data-files/hindi-data3/96.parquet
|
95 |
+
/dev/shm/data-files/hindi-data3/95.parquet
|
96 |
+
/dev/shm/data-files/hindi-data3/94.parquet
|
97 |
+
/dev/shm/data-files/hindi-data3/93.parquet
|
98 |
+
/dev/shm/data-files/hindi-data3/92.parquet
|
99 |
+
/dev/shm/data-files/hindi-data3/91.parquet
|
100 |
+
/dev/shm/data-files/hindi-data3/90.parquet
|
101 |
+
/dev/shm/data-files/hindi-data3/9.parquet
|
102 |
+
/dev/shm/data-files/hindi-data3/89.parquet
|
103 |
+
/dev/shm/data-files/hindi-data3/88.parquet
|
104 |
+
/dev/shm/data-files/hindi-data3/87.parquet
|
105 |
+
/dev/shm/data-files/hindi-data3/86.parquet
|
106 |
+
/dev/shm/data-files/hindi-data3/85.parquet
|
107 |
+
/dev/shm/data-files/hindi-data3/84.parquet
|
108 |
+
/dev/shm/data-files/hindi-data3/83.parquet
|
109 |
+
/dev/shm/data-files/hindi-data3/82.parquet
|
110 |
+
/dev/shm/data-files/hindi-data3/81.parquet
|
111 |
+
/dev/shm/data-files/hindi-data3/80.parquet
|
112 |
+
/dev/shm/data-files/hindi-data3/8.parquet
|
113 |
+
/dev/shm/data-files/hindi-data3/79.parquet
|
114 |
+
/dev/shm/data-files/hindi-data3/78.parquet
|
115 |
+
/dev/shm/data-files/hindi-data3/77.parquet
|
116 |
+
/dev/shm/data-files/hindi-data3/76.parquet
|
117 |
+
/dev/shm/data-files/hindi-data3/75.parquet
|
118 |
+
/dev/shm/data-files/hindi-data3/74.parquet
|
119 |
+
/dev/shm/data-files/hindi-data3/73.parquet
|
120 |
+
/dev/shm/data-files/hindi-data3/72.parquet
|
121 |
+
/dev/shm/data-files/hindi-data3/71.parquet
|
122 |
+
/dev/shm/data-files/hindi-data3/70.parquet
|
123 |
+
/dev/shm/data-files/hindi-data3/7.parquet
|
124 |
+
/dev/shm/data-files/hindi-data3/69.parquet
|
125 |
+
/dev/shm/data-files/hindi-data3/68.parquet
|
126 |
+
/dev/shm/data-files/hindi-data3/67.parquet
|
127 |
+
/dev/shm/data-files/hindi-data3/66.parquet
|
128 |
+
/dev/shm/data-files/hindi-data3/65.parquet
|
129 |
+
/dev/shm/data-files/hindi-data3/64.parquet
|
130 |
+
/dev/shm/data-files/hindi-data3/63.parquet
|
131 |
+
/dev/shm/data-files/hindi-data3/62.parquet
|
132 |
+
/dev/shm/data-files/hindi-data3/61.parquet
|
133 |
+
/dev/shm/data-files/hindi-data3/60.parquet
|
134 |
+
/dev/shm/data-files/hindi-data3/6.parquet
|
135 |
+
/dev/shm/data-files/hindi-data3/59.parquet
|
136 |
+
/dev/shm/data-files/hindi-data3/58.parquet
|
137 |
+
/dev/shm/data-files/hindi-data3/57.parquet
|
138 |
+
/dev/shm/data-files/hindi-data3/56.parquet
|
139 |
+
/dev/shm/data-files/hindi-data3/55.parquet
|
140 |
+
/dev/shm/data-files/hindi-data3/54.parquet
|
141 |
+
/dev/shm/data-files/hindi-data3/53.parquet
|
142 |
+
/dev/shm/data-files/hindi-data3/52.parquet
|
143 |
+
/dev/shm/data-files/hindi-data3/51.parquet
|
144 |
+
/dev/shm/data-files/hindi-data3/50.parquet
|
145 |
+
/dev/shm/data-files/hindi-data3/5.parquet
|
146 |
+
/dev/shm/data-files/hindi-data3/49.parquet
|
147 |
+
/dev/shm/data-files/hindi-data3/48.parquet
|
148 |
+
/dev/shm/data-files/hindi-data3/47.parquet
|
149 |
+
/dev/shm/data-files/hindi-data3/46.parquet
|
150 |
+
/dev/shm/data-files/hindi-data3/45.parquet
|
151 |
+
/dev/shm/data-files/hindi-data3/44.parquet
|
152 |
+
/dev/shm/data-files/hindi-data3/43.parquet
|
153 |
+
/dev/shm/data-files/hindi-data3/42.parquet
|
154 |
+
/dev/shm/data-files/hindi-data3/41.parquet
|
155 |
+
/dev/shm/data-files/hindi-data3/40.parquet
|
156 |
+
/dev/shm/data-files/hindi-data3/4.parquet
|
157 |
+
/dev/shm/data-files/hindi-data3/39.parquet
|
158 |
+
/dev/shm/data-files/hindi-data3/38.parquet
|
159 |
+
/dev/shm/data-files/hindi-data3/37.parquet
|
160 |
+
/dev/shm/data-files/hindi-data3/36.parquet
|
161 |
+
/dev/shm/data-files/hindi-data3/35.parquet
|
162 |
+
/dev/shm/data-files/hindi-data3/34.parquet
|
163 |
+
/dev/shm/data-files/hindi-data3/33.parquet
|
164 |
+
/dev/shm/data-files/hindi-data3/32.parquet
|
165 |
+
/dev/shm/data-files/hindi-data3/31.parquet
|
166 |
+
/dev/shm/data-files/hindi-data3/30.parquet
|
167 |
+
/dev/shm/data-files/hindi-data3/3.parquet
|
168 |
+
/dev/shm/data-files/hindi-data3/29.parquet
|
169 |
+
/dev/shm/data-files/hindi-data3/28.parquet
|
170 |
+
/dev/shm/data-files/hindi-data3/27.parquet
|
171 |
+
/dev/shm/data-files/hindi-data3/26.parquet
|
172 |
+
/dev/shm/data-files/hindi-data3/25.parquet
|
173 |
+
/dev/shm/data-files/hindi-data3/24.parquet
|
174 |
+
/dev/shm/data-files/hindi-data3/23.parquet
|
175 |
+
/dev/shm/data-files/hindi-data3/22.parquet
|
176 |
+
/dev/shm/data-files/hindi-data3/21.parquet
|
177 |
+
/dev/shm/data-files/hindi-data3/20.parquet
|
178 |
+
/dev/shm/data-files/hindi-data3/2.parquet
|
179 |
+
/dev/shm/data-files/hindi-data3/19.parquet
|
180 |
+
/dev/shm/data-files/hindi-data3/18.parquet
|
181 |
+
/dev/shm/data-files/hindi-data3/17.parquet
|
182 |
+
/dev/shm/data-files/hindi-data3/16.parquet
|
183 |
+
/dev/shm/data-files/hindi-data3/15.parquet
|
184 |
+
/dev/shm/data-files/hindi-data3/14.parquet
|
185 |
+
/dev/shm/data-files/hindi-data3/13.parquet
|
186 |
+
/dev/shm/data-files/hindi-data3/12.parquet
|
187 |
+
/dev/shm/data-files/hindi-data3/11.parquet
|
188 |
+
/dev/shm/data-files/hindi-data3/103.parquet
|
189 |
+
/dev/shm/data-files/hindi-data3/102.parquet
|
190 |
+
/dev/shm/data-files/hindi-data3/101.parquet
|
191 |
+
/dev/shm/data-files/hindi-data3/100.parquet
|
192 |
+
/dev/shm/data-files/hindi-data3/10.parquet
|
193 |
+
/dev/shm/data-files/hindi-data3/1.parquet
|
194 |
+
/dev/shm/data-files/hindi-data3/0.parquet
|
195 |
+
/dev/shm/data-files/hindi-data2/9.parquet
|
196 |
+
/dev/shm/data-files/hindi-data2/8.parquet
|
197 |
+
/dev/shm/data-files/hindi-data2/7.parquet
|
198 |
+
/dev/shm/data-files/hindi-data2/6.parquet
|
199 |
+
/dev/shm/data-files/hindi-data2/5.parquet
|
200 |
+
/dev/shm/data-files/hindi-data2/4.parquet
|
201 |
+
/dev/shm/data-files/hindi-data2/3.parquet
|
202 |
+
/dev/shm/data-files/hindi-data2/25.parquet
|
203 |
+
/dev/shm/data-files/hindi-data2/24.parquet
|
204 |
+
/dev/shm/data-files/hindi-data2/23.parquet
|
205 |
+
/dev/shm/data-files/hindi-data2/22.parquet
|
206 |
+
/dev/shm/data-files/hindi-data2/21.parquet
|
207 |
+
/dev/shm/data-files/hindi-data2/20.parquet
|
208 |
+
/dev/shm/data-files/hindi-data2/2.parquet
|
209 |
+
/dev/shm/data-files/hindi-data2/19.parquet
|
210 |
+
/dev/shm/data-files/hindi-data2/18.parquet
|
211 |
+
/dev/shm/data-files/hindi-data2/17.parquet
|
212 |
+
/dev/shm/data-files/hindi-data2/16.parquet
|
213 |
+
/dev/shm/data-files/hindi-data2/15.parquet
|
214 |
+
/dev/shm/data-files/hindi-data2/14.parquet
|
215 |
+
/dev/shm/data-files/hindi-data2/13.parquet
|
216 |
+
/dev/shm/data-files/hindi-data2/12.parquet
|
217 |
+
/dev/shm/data-files/hindi-data2/11.parquet
|
218 |
+
/dev/shm/data-files/hindi-data2/10.parquet
|
219 |
+
/dev/shm/data-files/hindi-data2/1.parquet
|
220 |
+
/dev/shm/data-files/hindi-data2/0.parquet
|
221 |
+
/dev/shm/data-files/hindi-data1/9.parquet
|
222 |
+
/dev/shm/data-files/hindi-data1/8.parquet
|
223 |
+
/dev/shm/data-files/hindi-data1/7.parquet
|
224 |
+
/dev/shm/data-files/hindi-data1/6.parquet
|
225 |
+
/dev/shm/data-files/hindi-data1/5.parquet
|
226 |
+
/dev/shm/data-files/hindi-data1/46.parquet
|
227 |
+
/dev/shm/data-files/hindi-data1/45.parquet
|
228 |
+
/dev/shm/data-files/hindi-data1/44.parquet
|
229 |
+
/dev/shm/data-files/hindi-data1/43.parquet
|
230 |
+
/dev/shm/data-files/hindi-data1/42.parquet
|
231 |
+
/dev/shm/data-files/hindi-data1/41.parquet
|
232 |
+
/dev/shm/data-files/hindi-data1/40.parquet
|
233 |
+
/dev/shm/data-files/hindi-data1/4.parquet
|
234 |
+
/dev/shm/data-files/hindi-data1/39.parquet
|
235 |
+
/dev/shm/data-files/hindi-data1/38.parquet
|
236 |
+
/dev/shm/data-files/hindi-data1/37.parquet
|
237 |
+
/dev/shm/data-files/hindi-data1/36.parquet
|
238 |
+
/dev/shm/data-files/hindi-data1/35.parquet
|
239 |
+
/dev/shm/data-files/hindi-data1/34.parquet
|
240 |
+
/dev/shm/data-files/hindi-data1/33.parquet
|
241 |
+
/dev/shm/data-files/hindi-data1/32.parquet
|
242 |
+
/dev/shm/data-files/hindi-data1/31.parquet
|
243 |
+
/dev/shm/data-files/hindi-data1/30.parquet
|
244 |
+
/dev/shm/data-files/hindi-data1/3.parquet
|
245 |
+
/dev/shm/data-files/hindi-data1/29.parquet
|
246 |
+
/dev/shm/data-files/hindi-data1/28.parquet
|
247 |
+
/dev/shm/data-files/hindi-data1/27.parquet
|
248 |
+
/dev/shm/data-files/hindi-data1/26.parquet
|
249 |
+
/dev/shm/data-files/hindi-data1/25.parquet
|
250 |
+
/dev/shm/data-files/hindi-data1/24.parquet
|
251 |
+
/dev/shm/data-files/hindi-data1/23.parquet
|
252 |
+
/dev/shm/data-files/hindi-data1/22.parquet
|
253 |
+
/dev/shm/data-files/hindi-data1/21.parquet
|
254 |
+
/dev/shm/data-files/hindi-data1/20.parquet
|
255 |
+
/dev/shm/data-files/hindi-data1/2.parquet
|
256 |
+
/dev/shm/data-files/hindi-data1/19.parquet
|
257 |
+
/dev/shm/data-files/hindi-data1/18.parquet
|
258 |
+
/dev/shm/data-files/hindi-data1/17.parquet
|
259 |
+
/dev/shm/data-files/hindi-data1/16.parquet
|
260 |
+
/dev/shm/data-files/hindi-data1/15.parquet
|
261 |
+
/dev/shm/data-files/hindi-data1/14.parquet
|
262 |
+
/dev/shm/data-files/hindi-data1/13.parquet
|
263 |
+
/dev/shm/data-files/hindi-data1/12.parquet
|
264 |
+
/dev/shm/data-files/hindi-data1/11.parquet
|
265 |
+
/dev/shm/data-files/hindi-data1/10.parquet
|
266 |
+
/dev/shm/data-files/hindi-data1/1.parquet
|
267 |
+
/dev/shm/data-files/hindi-data1/0.parquet
|
268 |
+
/dev/shm/data-files/english-data/9.parquet
|
269 |
+
/dev/shm/data-files/english-data/8.parquet
|
270 |
+
/dev/shm/data-files/english-data/7.parquet
|
271 |
+
/dev/shm/data-files/english-data/6.parquet
|
272 |
+
/dev/shm/data-files/english-data/57.parquet
|
273 |
+
/dev/shm/data-files/english-data/56.parquet
|
274 |
+
/dev/shm/data-files/english-data/55.parquet
|
275 |
+
/dev/shm/data-files/english-data/54.parquet
|
276 |
+
/dev/shm/data-files/english-data/53.parquet
|
277 |
+
/dev/shm/data-files/english-data/52.parquet
|
278 |
+
/dev/shm/data-files/english-data/51.parquet
|
279 |
+
/dev/shm/data-files/english-data/50.parquet
|
280 |
+
/dev/shm/data-files/english-data/5.parquet
|
281 |
+
/dev/shm/data-files/english-data/49.parquet
|
282 |
+
/dev/shm/data-files/english-data/48.parquet
|
283 |
+
/dev/shm/data-files/english-data/47.parquet
|
284 |
+
/dev/shm/data-files/english-data/46.parquet
|
285 |
+
/dev/shm/data-files/english-data/45.parquet
|
286 |
+
/dev/shm/data-files/english-data/44.parquet
|
287 |
+
/dev/shm/data-files/english-data/43.parquet
|
288 |
+
/dev/shm/data-files/english-data/42.parquet
|
289 |
+
/dev/shm/data-files/english-data/41.parquet
|
290 |
+
/dev/shm/data-files/english-data/40.parquet
|
291 |
+
/dev/shm/data-files/english-data/4.parquet
|
292 |
+
/dev/shm/data-files/english-data/39.parquet
|
293 |
+
/dev/shm/data-files/english-data/38.parquet
|
294 |
+
/dev/shm/data-files/english-data/37.parquet
|
295 |
+
/dev/shm/data-files/english-data/36.parquet
|
296 |
+
/dev/shm/data-files/english-data/35.parquet
|
297 |
+
/dev/shm/data-files/english-data/34.parquet
|
298 |
+
/dev/shm/data-files/english-data/33.parquet
|
299 |
+
/dev/shm/data-files/english-data/32.parquet
|
300 |
+
/dev/shm/data-files/english-data/31.parquet
|
301 |
+
/dev/shm/data-files/english-data/30.parquet
|
302 |
+
/dev/shm/data-files/english-data/3.parquet
|
303 |
+
/dev/shm/data-files/english-data/29.parquet
|
304 |
+
/dev/shm/data-files/english-data/28.parquet
|
305 |
+
/dev/shm/data-files/english-data/27.parquet
|
306 |
+
/dev/shm/data-files/english-data/26.parquet
|
307 |
+
/dev/shm/data-files/english-data/25.parquet
|
308 |
+
/dev/shm/data-files/english-data/24.parquet
|
309 |
+
/dev/shm/data-files/english-data/23.parquet
|
310 |
+
/dev/shm/data-files/english-data/22.parquet
|
311 |
+
/dev/shm/data-files/english-data/21.parquet
|
312 |
+
/dev/shm/data-files/english-data/20.parquet
|
313 |
+
/dev/shm/data-files/english-data/2.parquet
|
314 |
+
/dev/shm/data-files/english-data/19.parquet
|
315 |
+
/dev/shm/data-files/english-data/18.parquet
|
316 |
+
/dev/shm/data-files/english-data/17.parquet
|
317 |
+
/dev/shm/data-files/english-data/16.parquet
|
318 |
+
/dev/shm/data-files/english-data/15.parquet
|
319 |
+
/dev/shm/data-files/english-data/14.parquet
|
320 |
+
/dev/shm/data-files/english-data/13.parquet
|
321 |
+
/dev/shm/data-files/english-data/12.parquet
|
322 |
+
/dev/shm/data-files/english-data/11.parquet
|
323 |
+
/dev/shm/data-files/english-data/10.parquet
|
324 |
+
/dev/shm/data-files/english-data/1.parquet
|
325 |
+
/dev/shm/data-files/english-data/0.parquet
|
flax_model.msgpack
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 962037499
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:9d967d79b8d0e4b4a23656fec185bbb9e68940ad2e013e9a7b0f0f77b5c0a666
|
3 |
size 962037499
|
muril-bigbird/flax_model.msgpack
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 962037499
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:9a27a51818882c80dc60c00a6758070d78bd85e45a66c91a5c31cf6f491516c7
|
3 |
size 962037499
|
run_muril_bb_flax.sh
CHANGED
@@ -5,6 +5,7 @@ python run_muril_bb_flax_mlm.py \
|
|
5 |
--tokenizer_name muril-bigbird \
|
6 |
--dtype bfloat16 \
|
7 |
--pretokenized \
|
|
|
8 |
--train_file "/dev/shm/data-files" \
|
9 |
--validation_file "." \
|
10 |
--max_seq_length 4096 \
|
@@ -20,9 +21,9 @@ python run_muril_bb_flax_mlm.py \
|
|
20 |
--weight_decay 0.01 \
|
21 |
--num_train_epochs 3 \
|
22 |
--warmup_steps 0 \
|
23 |
-
--logging_steps
|
24 |
-
--save_steps
|
25 |
--save_total_limit 1 \
|
26 |
--seed 18 \
|
27 |
--push_to_hub \
|
28 |
-
--run_name "en-ta-hi-te-from-
|
|
|
5 |
--tokenizer_name muril-bigbird \
|
6 |
--dtype bfloat16 \
|
7 |
--pretokenized \
|
8 |
+
--resume_from_checkpoint "checkpoint_100000" \
|
9 |
--train_file "/dev/shm/data-files" \
|
10 |
--validation_file "." \
|
11 |
--max_seq_length 4096 \
|
|
|
21 |
--weight_decay 0.01 \
|
22 |
--num_train_epochs 3 \
|
23 |
--warmup_steps 0 \
|
24 |
+
--logging_steps 1000 \
|
25 |
+
--save_steps 25000 \
|
26 |
--save_total_limit 1 \
|
27 |
--seed 18 \
|
28 |
--push_to_hub \
|
29 |
+
--run_name "en-ta-hi-te-from-100k"
|
run_muril_bb_flax_mlm.py
CHANGED
@@ -343,7 +343,7 @@ if __name__ == "__main__":
|
|
343 |
files = p.glob("*/*.parquet")
|
344 |
files = [str(x) for x in files]
|
345 |
random.shuffle(files)
|
346 |
-
files = files[
|
347 |
datasets= load_dataset("parquet", data_files=files, split="train", cache_dir="/dev/shm")
|
348 |
datasets = datasets.shuffle(seed=training_args.seed)
|
349 |
datasets = datasets.train_test_split(test_size = 0.02, seed=training_args.seed)
|
|
|
343 |
files = p.glob("*/*.parquet")
|
344 |
files = [str(x) for x in files]
|
345 |
random.shuffle(files)
|
346 |
+
files = files[100:150]
|
347 |
datasets= load_dataset("parquet", data_files=files, split="train", cache_dir="/dev/shm")
|
348 |
datasets = datasets.shuffle(seed=training_args.seed)
|
349 |
datasets = datasets.train_test_split(test_size = 0.02, seed=training_args.seed)
|
wandb/debug-internal.log
CHANGED
@@ -1 +1 @@
|
|
1 |
-
run-
|
|
|
1 |
+
run-20211108_082107-2idnuxcc/logs/debug-internal.log
|
wandb/debug.log
CHANGED
@@ -1 +1 @@
|
|
1 |
-
run-
|
|
|
1 |
+
run-20211108_082107-2idnuxcc/logs/debug.log
|
wandb/latest-run
CHANGED
@@ -1 +1 @@
|
|
1 |
-
run-
|
|
|
1 |
+
run-20211108_082107-2idnuxcc
|
wandb/run-20211107_204808-2ns3lo5n/files/config.yaml
CHANGED
@@ -23,6 +23,12 @@ _wandb:
|
|
23 |
- 3
|
24 |
- 11
|
25 |
- 12
|
|
|
|
|
|
|
|
|
|
|
|
|
26 |
3:
|
27 |
- 13
|
28 |
4: 3.8.10
|
|
|
23 |
- 3
|
24 |
- 11
|
25 |
- 12
|
26 |
+
2:
|
27 |
+
- 1
|
28 |
+
- 2
|
29 |
+
- 3
|
30 |
+
- 11
|
31 |
+
- 12
|
32 |
3:
|
33 |
- 13
|
34 |
4: 3.8.10
|
wandb/run-20211107_204808-2ns3lo5n/files/output.log
CHANGED
@@ -12484,3 +12484,5417 @@ tcmalloc: large alloc 2886115328 bytes == 0x5c1898000 @ 0x7fee676b5680 0x7fee67
|
|
12484 |
[04:14:14] - INFO - absl - Saved checkpoint at ./checkpoint_100000
|
12485 |
[04:14:14] - INFO - absl - Removing checkpoint at ./checkpoint_50000
|
12486 |
Adding files tracked by Git LFS: ['checkpoint_100000']. This may take a bit of time if the files are large.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
12484 |
[04:14:14] - INFO - absl - Saved checkpoint at ./checkpoint_100000
|
12485 |
[04:14:14] - INFO - absl - Removing checkpoint at ./checkpoint_50000
|
12486 |
Adding files tracked by Git LFS: ['checkpoint_100000']. This may take a bit of time if the files are large.
|
12487 |
+
[04:14:39] - WARNING - huggingface_hub.repository - Adding files tracked by Git LFS: ['checkpoint_100000']. This may take a bit of time if the files are large.
|
12488 |
+
|
12489 |
+
|
12490 |
+
|
12491 |
+
|
12492 |
+
|
12493 |
+
|
12494 |
+
|
12495 |
+
|
12496 |
+
|
12497 |
+
|
12498 |
+
|
12499 |
+
|
12500 |
+
|
12501 |
+
|
12502 |
+
|
12503 |
+
|
12504 |
+
|
12505 |
+
|
12506 |
+
|
12507 |
+
|
12508 |
+
|
12509 |
+
|
12510 |
+
|
12511 |
+
|
12512 |
+
|
12513 |
+
|
12514 |
+
|
12515 |
+
|
12516 |
+
|
12517 |
+
|
12518 |
+
|
12519 |
+
|
12520 |
+
|
12521 |
+
|
12522 |
+
|
12523 |
+
|
12524 |
+
|
12525 |
+
|
12526 |
+
|
12527 |
+
|
12528 |
+
|
12529 |
+
|
12530 |
+
|
12531 |
+
|
12532 |
+
|
12533 |
+
|
12534 |
+
|
12535 |
+
|
12536 |
+
|
12537 |
+
|
12538 |
+
|
12539 |
+
|
12540 |
+
|
12541 |
+
|
12542 |
+
|
12543 |
+
|
12544 |
+
|
12545 |
+
|
12546 |
+
|
12547 |
+
|
12548 |
+
|
12549 |
+
|
12550 |
+
|
12551 |
+
|
12552 |
+
|
12553 |
+
|
12554 |
+
|
12555 |
+
|
12556 |
+
|
12557 |
+
|
12558 |
+
|
12559 |
+
|
12560 |
+
|
12561 |
+
|
12562 |
+
|
12563 |
+
|
12564 |
+
|
12565 |
+
|
12566 |
+
|
12567 |
+
|
12568 |
+
|
12569 |
+
|
12570 |
+
|
12571 |
+
|
12572 |
+
|
12573 |
+
|
12574 |
+
|
12575 |
+
|
12576 |
+
|
12577 |
+
|
12578 |
+
|
12579 |
+
|
12580 |
+
|
12581 |
+
|
12582 |
+
|
12583 |
+
|
12584 |
+
|
12585 |
+
|
12586 |
+
|
12587 |
+
|
12588 |
+
|
12589 |
+
|
12590 |
+
|
12591 |
+
|
12592 |
+
|
12593 |
+
|
12594 |
+
|
12595 |
+
|
12596 |
+
|
12597 |
+
|
12598 |
+
|
12599 |
+
|
12600 |
+
|
12601 |
+
|
12602 |
+
Training...: 82%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 50503/61250 [7:29:37<8:37:19, 2.89s/it]
|
12603 |
+
|
12604 |
+
|
12605 |
+
|
12606 |
+
|
12607 |
+
|
12608 |
+
|
12609 |
+
|
12610 |
+
|
12611 |
+
|
12612 |
+
|
12613 |
+
|
12614 |
+
|
12615 |
+
|
12616 |
+
|
12617 |
+
|
12618 |
+
|
12619 |
+
|
12620 |
+
|
12621 |
+
|
12622 |
+
|
12623 |
+
|
12624 |
+
|
12625 |
+
|
12626 |
+
|
12627 |
+
|
12628 |
+
|
12629 |
+
|
12630 |
+
|
12631 |
+
|
12632 |
+
|
12633 |
+
|
12634 |
+
|
12635 |
+
|
12636 |
+
|
12637 |
+
|
12638 |
+
|
12639 |
+
|
12640 |
+
|
12641 |
+
|
12642 |
+
|
12643 |
+
|
12644 |
+
|
12645 |
+
|
12646 |
+
|
12647 |
+
|
12648 |
+
|
12649 |
+
|
12650 |
+
|
12651 |
+
|
12652 |
+
|
12653 |
+
|
12654 |
+
|
12655 |
+
|
12656 |
+
|
12657 |
+
|
12658 |
+
|
12659 |
+
|
12660 |
+
|
12661 |
+
|
12662 |
+
|
12663 |
+
|
12664 |
+
|
12665 |
+
|
12666 |
+
|
12667 |
+
|
12668 |
+
|
12669 |
+
|
12670 |
+
|
12671 |
+
|
12672 |
+
|
12673 |
+
|
12674 |
+
|
12675 |
+
|
12676 |
+
|
12677 |
+
|
12678 |
+
|
12679 |
+
|
12680 |
+
|
12681 |
+
|
12682 |
+
|
12683 |
+
|
12684 |
+
|
12685 |
+
|
12686 |
+
|
12687 |
+
|
12688 |
+
|
12689 |
+
|
12690 |
+
|
12691 |
+
|
12692 |
+
|
12693 |
+
|
12694 |
+
|
12695 |
+
|
12696 |
+
|
12697 |
+
|
12698 |
+
|
12699 |
+
|
12700 |
+
|
12701 |
+
|
12702 |
+
|
12703 |
+
|
12704 |
+
|
12705 |
+
|
12706 |
+
|
12707 |
+
|
12708 |
+
|
12709 |
+
|
12710 |
+
|
12711 |
+
|
12712 |
+
|
12713 |
+
|
12714 |
+
|
12715 |
+
|
12716 |
+
Training...: 83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 51000/61250 [7:33:23<1:22:50, 2.06it/s]
|
12717 |
+
|
12718 |
+
|
12719 |
+
|
12720 |
+
|
12721 |
+
|
12722 |
+
|
12723 |
+
|
12724 |
+
|
12725 |
+
|
12726 |
+
|
12727 |
+
|
12728 |
+
|
12729 |
+
|
12730 |
+
|
12731 |
+
|
12732 |
+
|
12733 |
+
|
12734 |
+
|
12735 |
+
|
12736 |
+
|
12737 |
+
|
12738 |
+
|
12739 |
+
|
12740 |
+
|
12741 |
+
|
12742 |
+
|
12743 |
+
|
12744 |
+
|
12745 |
+
|
12746 |
+
|
12747 |
+
|
12748 |
+
|
12749 |
+
|
12750 |
+
|
12751 |
+
|
12752 |
+
|
12753 |
+
|
12754 |
+
|
12755 |
+
|
12756 |
+
|
12757 |
+
|
12758 |
+
|
12759 |
+
|
12760 |
+
|
12761 |
+
|
12762 |
+
|
12763 |
+
|
12764 |
+
|
12765 |
+
|
12766 |
+
|
12767 |
+
|
12768 |
+
|
12769 |
+
|
12770 |
+
|
12771 |
+
|
12772 |
+
|
12773 |
+
|
12774 |
+
|
12775 |
+
|
12776 |
+
|
12777 |
+
|
12778 |
+
|
12779 |
+
|
12780 |
+
|
12781 |
+
|
12782 |
+
|
12783 |
+
|
12784 |
+
|
12785 |
+
|
12786 |
+
|
12787 |
+
|
12788 |
+
|
12789 |
+
|
12790 |
+
|
12791 |
+
|
12792 |
+
|
12793 |
+
|
12794 |
+
|
12795 |
+
|
12796 |
+
|
12797 |
+
|
12798 |
+
|
12799 |
+
|
12800 |
+
|
12801 |
+
|
12802 |
+
|
12803 |
+
|
12804 |
+
|
12805 |
+
|
12806 |
+
|
12807 |
+
|
12808 |
+
|
12809 |
+
|
12810 |
+
|
12811 |
+
|
12812 |
+
|
12813 |
+
|
12814 |
+
|
12815 |
+
|
12816 |
+
|
12817 |
+
|
12818 |
+
|
12819 |
+
|
12820 |
+
|
12821 |
+
|
12822 |
+
|
12823 |
+
|
12824 |
+
|
12825 |
+
|
12826 |
+
|
12827 |
+
|
12828 |
+
|
12829 |
+
|
12830 |
+
Training...: 84%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 51500/61250 [7:37:26<1:18:48, 2.06it/s]
|
12831 |
+
|
12832 |
+
|
12833 |
+
|
12834 |
+
|
12835 |
+
|
12836 |
+
|
12837 |
+
|
12838 |
+
|
12839 |
+
|
12840 |
+
|
12841 |
+
|
12842 |
+
|
12843 |
+
|
12844 |
+
|
12845 |
+
|
12846 |
+
|
12847 |
+
|
12848 |
+
|
12849 |
+
|
12850 |
+
|
12851 |
+
|
12852 |
+
|
12853 |
+
|
12854 |
+
|
12855 |
+
|
12856 |
+
|
12857 |
+
|
12858 |
+
|
12859 |
+
|
12860 |
+
|
12861 |
+
|
12862 |
+
|
12863 |
+
|
12864 |
+
|
12865 |
+
|
12866 |
+
|
12867 |
+
|
12868 |
+
|
12869 |
+
|
12870 |
+
|
12871 |
+
|
12872 |
+
|
12873 |
+
|
12874 |
+
|
12875 |
+
|
12876 |
+
|
12877 |
+
|
12878 |
+
|
12879 |
+
|
12880 |
+
|
12881 |
+
|
12882 |
+
|
12883 |
+
|
12884 |
+
|
12885 |
+
|
12886 |
+
|
12887 |
+
|
12888 |
+
|
12889 |
+
|
12890 |
+
|
12891 |
+
|
12892 |
+
|
12893 |
+
|
12894 |
+
|
12895 |
+
|
12896 |
+
|
12897 |
+
|
12898 |
+
|
12899 |
+
|
12900 |
+
|
12901 |
+
|
12902 |
+
|
12903 |
+
|
12904 |
+
|
12905 |
+
|
12906 |
+
|
12907 |
+
|
12908 |
+
|
12909 |
+
|
12910 |
+
|
12911 |
+
|
12912 |
+
|
12913 |
+
|
12914 |
+
|
12915 |
+
|
12916 |
+
|
12917 |
+
|
12918 |
+
|
12919 |
+
|
12920 |
+
|
12921 |
+
|
12922 |
+
|
12923 |
+
|
12924 |
+
|
12925 |
+
|
12926 |
+
|
12927 |
+
|
12928 |
+
|
12929 |
+
|
12930 |
+
|
12931 |
+
|
12932 |
+
|
12933 |
+
|
12934 |
+
|
12935 |
+
|
12936 |
+
|
12937 |
+
|
12938 |
+
|
12939 |
+
|
12940 |
+
|
12941 |
+
|
12942 |
+
|
12943 |
+
|
12944 |
+
Training...: 85%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 52000/61250 [7:41:29<1:14:47, 2.06it/s]
|
12945 |
+
|
12946 |
+
|
12947 |
+
|
12948 |
+
|
12949 |
+
|
12950 |
+
|
12951 |
+
|
12952 |
+
|
12953 |
+
|
12954 |
+
|
12955 |
+
|
12956 |
+
|
12957 |
+
|
12958 |
+
|
12959 |
+
|
12960 |
+
|
12961 |
+
|
12962 |
+
|
12963 |
+
|
12964 |
+
|
12965 |
+
|
12966 |
+
|
12967 |
+
|
12968 |
+
|
12969 |
+
|
12970 |
+
|
12971 |
+
|
12972 |
+
|
12973 |
+
|
12974 |
+
|
12975 |
+
|
12976 |
+
|
12977 |
+
|
12978 |
+
|
12979 |
+
|
12980 |
+
|
12981 |
+
|
12982 |
+
|
12983 |
+
|
12984 |
+
|
12985 |
+
|
12986 |
+
|
12987 |
+
|
12988 |
+
|
12989 |
+
|
12990 |
+
|
12991 |
+
|
12992 |
+
|
12993 |
+
|
12994 |
+
|
12995 |
+
|
12996 |
+
|
12997 |
+
|
12998 |
+
|
12999 |
+
|
13000 |
+
|
13001 |
+
|
13002 |
+
|
13003 |
+
|
13004 |
+
|
13005 |
+
|
13006 |
+
|
13007 |
+
|
13008 |
+
|
13009 |
+
|
13010 |
+
|
13011 |
+
|
13012 |
+
|
13013 |
+
|
13014 |
+
|
13015 |
+
|
13016 |
+
|
13017 |
+
|
13018 |
+
|
13019 |
+
|
13020 |
+
|
13021 |
+
|
13022 |
+
|
13023 |
+
|
13024 |
+
|
13025 |
+
|
13026 |
+
|
13027 |
+
|
13028 |
+
|
13029 |
+
|
13030 |
+
|
13031 |
+
|
13032 |
+
|
13033 |
+
|
13034 |
+
|
13035 |
+
|
13036 |
+
|
13037 |
+
|
13038 |
+
|
13039 |
+
|
13040 |
+
|
13041 |
+
|
13042 |
+
|
13043 |
+
|
13044 |
+
|
13045 |
+
|
13046 |
+
|
13047 |
+
|
13048 |
+
|
13049 |
+
|
13050 |
+
|
13051 |
+
|
13052 |
+
|
13053 |
+
|
13054 |
+
|
13055 |
+
|
13056 |
+
|
13057 |
+
|
13058 |
+
|
13059 |
+
Training...: 86%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 52500/61250 [7:45:32<1:10:43, 2.06it/s]
|
13060 |
+
|
13061 |
+
|
13062 |
+
|
13063 |
+
|
13064 |
+
|
13065 |
+
|
13066 |
+
|
13067 |
+
|
13068 |
+
|
13069 |
+
|
13070 |
+
|
13071 |
+
|
13072 |
+
|
13073 |
+
|
13074 |
+
|
13075 |
+
|
13076 |
+
|
13077 |
+
|
13078 |
+
|
13079 |
+
|
13080 |
+
|
13081 |
+
|
13082 |
+
|
13083 |
+
|
13084 |
+
|
13085 |
+
|
13086 |
+
|
13087 |
+
|
13088 |
+
|
13089 |
+
|
13090 |
+
|
13091 |
+
|
13092 |
+
|
13093 |
+
|
13094 |
+
|
13095 |
+
|
13096 |
+
|
13097 |
+
|
13098 |
+
|
13099 |
+
|
13100 |
+
|
13101 |
+
|
13102 |
+
|
13103 |
+
|
13104 |
+
|
13105 |
+
|
13106 |
+
|
13107 |
+
|
13108 |
+
|
13109 |
+
|
13110 |
+
|
13111 |
+
|
13112 |
+
|
13113 |
+
|
13114 |
+
|
13115 |
+
|
13116 |
+
|
13117 |
+
|
13118 |
+
|
13119 |
+
|
13120 |
+
|
13121 |
+
|
13122 |
+
|
13123 |
+
|
13124 |
+
|
13125 |
+
|
13126 |
+
|
13127 |
+
|
13128 |
+
|
13129 |
+
|
13130 |
+
|
13131 |
+
|
13132 |
+
|
13133 |
+
|
13134 |
+
|
13135 |
+
|
13136 |
+
|
13137 |
+
|
13138 |
+
|
13139 |
+
|
13140 |
+
|
13141 |
+
|
13142 |
+
|
13143 |
+
|
13144 |
+
|
13145 |
+
|
13146 |
+
|
13147 |
+
|
13148 |
+
|
13149 |
+
|
13150 |
+
|
13151 |
+
|
13152 |
+
|
13153 |
+
|
13154 |
+
|
13155 |
+
|
13156 |
+
|
13157 |
+
|
13158 |
+
|
13159 |
+
|
13160 |
+
|
13161 |
+
|
13162 |
+
|
13163 |
+
|
13164 |
+
|
13165 |
+
|
13166 |
+
|
13167 |
+
|
13168 |
+
|
13169 |
+
|
13170 |
+
|
13171 |
+
|
13172 |
+
|
13173 |
+
|
13174 |
+
|
13175 |
+
Training...: 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████��██████████████████▊ | 53019/61250 [7:49:53<23:27, 5.85it/s]
|
13176 |
+
|
13177 |
+
|
13178 |
+
|
13179 |
+
|
13180 |
+
|
13181 |
+
|
13182 |
+
|
13183 |
+
|
13184 |
+
|
13185 |
+
|
13186 |
+
|
13187 |
+
|
13188 |
+
|
13189 |
+
|
13190 |
+
|
13191 |
+
|
13192 |
+
|
13193 |
+
|
13194 |
+
|
13195 |
+
|
13196 |
+
|
13197 |
+
|
13198 |
+
|
13199 |
+
|
13200 |
+
|
13201 |
+
|
13202 |
+
|
13203 |
+
|
13204 |
+
|
13205 |
+
|
13206 |
+
|
13207 |
+
|
13208 |
+
|
13209 |
+
|
13210 |
+
|
13211 |
+
|
13212 |
+
|
13213 |
+
|
13214 |
+
|
13215 |
+
|
13216 |
+
|
13217 |
+
|
13218 |
+
|
13219 |
+
|
13220 |
+
|
13221 |
+
|
13222 |
+
|
13223 |
+
|
13224 |
+
|
13225 |
+
|
13226 |
+
|
13227 |
+
|
13228 |
+
|
13229 |
+
|
13230 |
+
|
13231 |
+
|
13232 |
+
|
13233 |
+
|
13234 |
+
|
13235 |
+
|
13236 |
+
|
13237 |
+
|
13238 |
+
|
13239 |
+
|
13240 |
+
|
13241 |
+
|
13242 |
+
|
13243 |
+
|
13244 |
+
|
13245 |
+
|
13246 |
+
|
13247 |
+
|
13248 |
+
|
13249 |
+
|
13250 |
+
|
13251 |
+
|
13252 |
+
|
13253 |
+
|
13254 |
+
|
13255 |
+
|
13256 |
+
|
13257 |
+
|
13258 |
+
|
13259 |
+
|
13260 |
+
|
13261 |
+
|
13262 |
+
|
13263 |
+
|
13264 |
+
|
13265 |
+
|
13266 |
+
|
13267 |
+
|
13268 |
+
|
13269 |
+
|
13270 |
+
|
13271 |
+
|
13272 |
+
|
13273 |
+
|
13274 |
+
|
13275 |
+
|
13276 |
+
|
13277 |
+
|
13278 |
+
|
13279 |
+
|
13280 |
+
|
13281 |
+
|
13282 |
+
|
13283 |
+
|
13284 |
+
|
13285 |
+
|
13286 |
+
|
13287 |
+
|
13288 |
+
|
13289 |
+
|
13290 |
+
Training...: 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 53519/61250 [7:53:56<21:59, 5.86it/s]
|
13291 |
+
|
13292 |
+
|
13293 |
+
|
13294 |
+
|
13295 |
+
|
13296 |
+
|
13297 |
+
|
13298 |
+
|
13299 |
+
|
13300 |
+
|
13301 |
+
|
13302 |
+
|
13303 |
+
|
13304 |
+
|
13305 |
+
|
13306 |
+
|
13307 |
+
|
13308 |
+
|
13309 |
+
|
13310 |
+
|
13311 |
+
|
13312 |
+
|
13313 |
+
|
13314 |
+
|
13315 |
+
|
13316 |
+
|
13317 |
+
|
13318 |
+
|
13319 |
+
|
13320 |
+
|
13321 |
+
|
13322 |
+
|
13323 |
+
|
13324 |
+
|
13325 |
+
|
13326 |
+
|
13327 |
+
|
13328 |
+
|
13329 |
+
|
13330 |
+
|
13331 |
+
|
13332 |
+
|
13333 |
+
|
13334 |
+
|
13335 |
+
|
13336 |
+
|
13337 |
+
|
13338 |
+
|
13339 |
+
|
13340 |
+
|
13341 |
+
|
13342 |
+
|
13343 |
+
|
13344 |
+
|
13345 |
+
|
13346 |
+
|
13347 |
+
|
13348 |
+
|
13349 |
+
|
13350 |
+
|
13351 |
+
|
13352 |
+
|
13353 |
+
|
13354 |
+
|
13355 |
+
|
13356 |
+
|
13357 |
+
|
13358 |
+
|
13359 |
+
|
13360 |
+
|
13361 |
+
|
13362 |
+
|
13363 |
+
|
13364 |
+
|
13365 |
+
|
13366 |
+
|
13367 |
+
|
13368 |
+
|
13369 |
+
|
13370 |
+
|
13371 |
+
|
13372 |
+
|
13373 |
+
|
13374 |
+
|
13375 |
+
|
13376 |
+
|
13377 |
+
|
13378 |
+
|
13379 |
+
|
13380 |
+
|
13381 |
+
|
13382 |
+
|
13383 |
+
|
13384 |
+
|
13385 |
+
|
13386 |
+
|
13387 |
+
|
13388 |
+
|
13389 |
+
|
13390 |
+
|
13391 |
+
|
13392 |
+
|
13393 |
+
|
13394 |
+
|
13395 |
+
|
13396 |
+
|
13397 |
+
|
13398 |
+
|
13399 |
+
|
13400 |
+
|
13401 |
+
|
13402 |
+
|
13403 |
+
|
13404 |
+
Training...: 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 54000/61250 [7:57:42<58:36, 2.06it/s]
|
13405 |
+
|
13406 |
+
|
13407 |
+
|
13408 |
+
|
13409 |
+
|
13410 |
+
|
13411 |
+
|
13412 |
+
|
13413 |
+
|
13414 |
+
|
13415 |
+
|
13416 |
+
|
13417 |
+
|
13418 |
+
|
13419 |
+
|
13420 |
+
|
13421 |
+
|
13422 |
+
|
13423 |
+
|
13424 |
+
|
13425 |
+
|
13426 |
+
|
13427 |
+
|
13428 |
+
|
13429 |
+
|
13430 |
+
|
13431 |
+
|
13432 |
+
|
13433 |
+
|
13434 |
+
|
13435 |
+
|
13436 |
+
|
13437 |
+
|
13438 |
+
|
13439 |
+
|
13440 |
+
|
13441 |
+
|
13442 |
+
|
13443 |
+
|
13444 |
+
|
13445 |
+
|
13446 |
+
|
13447 |
+
|
13448 |
+
|
13449 |
+
|
13450 |
+
|
13451 |
+
|
13452 |
+
|
13453 |
+
|
13454 |
+
|
13455 |
+
|
13456 |
+
|
13457 |
+
|
13458 |
+
|
13459 |
+
|
13460 |
+
|
13461 |
+
|
13462 |
+
|
13463 |
+
|
13464 |
+
|
13465 |
+
|
13466 |
+
|
13467 |
+
|
13468 |
+
|
13469 |
+
|
13470 |
+
|
13471 |
+
|
13472 |
+
|
13473 |
+
|
13474 |
+
|
13475 |
+
|
13476 |
+
|
13477 |
+
|
13478 |
+
|
13479 |
+
|
13480 |
+
|
13481 |
+
|
13482 |
+
|
13483 |
+
|
13484 |
+
|
13485 |
+
|
13486 |
+
|
13487 |
+
|
13488 |
+
|
13489 |
+
|
13490 |
+
|
13491 |
+
|
13492 |
+
|
13493 |
+
|
13494 |
+
|
13495 |
+
|
13496 |
+
|
13497 |
+
|
13498 |
+
|
13499 |
+
|
13500 |
+
|
13501 |
+
|
13502 |
+
|
13503 |
+
|
13504 |
+
|
13505 |
+
|
13506 |
+
|
13507 |
+
|
13508 |
+
|
13509 |
+
|
13510 |
+
|
13511 |
+
|
13512 |
+
|
13513 |
+
|
13514 |
+
|
13515 |
+
|
13516 |
+
|
13517 |
+
|
13518 |
+
|
13519 |
+
Training...: 89%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 54500/61250 [8:01:45<54:34, 2.06it/s]
|
13520 |
+
|
13521 |
+
|
13522 |
+
|
13523 |
+
|
13524 |
+
|
13525 |
+
|
13526 |
+
|
13527 |
+
|
13528 |
+
|
13529 |
+
|
13530 |
+
|
13531 |
+
|
13532 |
+
|
13533 |
+
|
13534 |
+
|
13535 |
+
|
13536 |
+
|
13537 |
+
|
13538 |
+
|
13539 |
+
|
13540 |
+
|
13541 |
+
|
13542 |
+
|
13543 |
+
|
13544 |
+
|
13545 |
+
|
13546 |
+
|
13547 |
+
|
13548 |
+
|
13549 |
+
|
13550 |
+
|
13551 |
+
|
13552 |
+
|
13553 |
+
|
13554 |
+
|
13555 |
+
|
13556 |
+
|
13557 |
+
|
13558 |
+
|
13559 |
+
|
13560 |
+
|
13561 |
+
|
13562 |
+
|
13563 |
+
|
13564 |
+
|
13565 |
+
|
13566 |
+
|
13567 |
+
|
13568 |
+
|
13569 |
+
|
13570 |
+
|
13571 |
+
|
13572 |
+
|
13573 |
+
|
13574 |
+
|
13575 |
+
|
13576 |
+
|
13577 |
+
|
13578 |
+
|
13579 |
+
|
13580 |
+
|
13581 |
+
|
13582 |
+
|
13583 |
+
|
13584 |
+
|
13585 |
+
|
13586 |
+
|
13587 |
+
|
13588 |
+
|
13589 |
+
|
13590 |
+
|
13591 |
+
|
13592 |
+
|
13593 |
+
|
13594 |
+
|
13595 |
+
|
13596 |
+
|
13597 |
+
|
13598 |
+
|
13599 |
+
|
13600 |
+
|
13601 |
+
|
13602 |
+
|
13603 |
+
|
13604 |
+
|
13605 |
+
|
13606 |
+
|
13607 |
+
|
13608 |
+
|
13609 |
+
|
13610 |
+
|
13611 |
+
|
13612 |
+
|
13613 |
+
|
13614 |
+
|
13615 |
+
|
13616 |
+
|
13617 |
+
|
13618 |
+
|
13619 |
+
|
13620 |
+
|
13621 |
+
|
13622 |
+
|
13623 |
+
|
13624 |
+
|
13625 |
+
|
13626 |
+
|
13627 |
+
|
13628 |
+
|
13629 |
+
|
13630 |
+
|
13631 |
+
|
13632 |
+
|
13633 |
+
|
13634 |
+
|
13635 |
+
Training...: 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 55000/61250 [8:06:05<50:30, 2.06it/s]
|
13636 |
+
Step... (105000 | Loss: 1.4761005640029907, Learning Rate: 3.5034012398682535e-05)
|
13637 |
+
|
13638 |
+
|
13639 |
+
|
13640 |
+
|
13641 |
+
|
13642 |
+
|
13643 |
+
|
13644 |
+
|
13645 |
+
|
13646 |
+
|
13647 |
+
|
13648 |
+
|
13649 |
+
|
13650 |
+
|
13651 |
+
|
13652 |
+
|
13653 |
+
|
13654 |
+
|
13655 |
+
|
13656 |
+
|
13657 |
+
|
13658 |
+
|
13659 |
+
|
13660 |
+
|
13661 |
+
|
13662 |
+
|
13663 |
+
|
13664 |
+
|
13665 |
+
|
13666 |
+
|
13667 |
+
|
13668 |
+
|
13669 |
+
|
13670 |
+
|
13671 |
+
|
13672 |
+
|
13673 |
+
|
13674 |
+
|
13675 |
+
|
13676 |
+
|
13677 |
+
|
13678 |
+
|
13679 |
+
|
13680 |
+
|
13681 |
+
|
13682 |
+
|
13683 |
+
|
13684 |
+
|
13685 |
+
|
13686 |
+
|
13687 |
+
|
13688 |
+
|
13689 |
+
|
13690 |
+
|
13691 |
+
|
13692 |
+
|
13693 |
+
|
13694 |
+
|
13695 |
+
|
13696 |
+
|
13697 |
+
|
13698 |
+
|
13699 |
+
|
13700 |
+
|
13701 |
+
|
13702 |
+
|
13703 |
+
|
13704 |
+
|
13705 |
+
|
13706 |
+
|
13707 |
+
|
13708 |
+
|
13709 |
+
|
13710 |
+
|
13711 |
+
|
13712 |
+
|
13713 |
+
|
13714 |
+
|
13715 |
+
|
13716 |
+
|
13717 |
+
|
13718 |
+
|
13719 |
+
|
13720 |
+
|
13721 |
+
|
13722 |
+
|
13723 |
+
|
13724 |
+
|
13725 |
+
|
13726 |
+
|
13727 |
+
|
13728 |
+
|
13729 |
+
|
13730 |
+
|
13731 |
+
|
13732 |
+
|
13733 |
+
|
13734 |
+
|
13735 |
+
|
13736 |
+
|
13737 |
+
|
13738 |
+
|
13739 |
+
|
13740 |
+
|
13741 |
+
|
13742 |
+
|
13743 |
+
|
13744 |
+
|
13745 |
+
|
13746 |
+
|
13747 |
+
|
13748 |
+
|
13749 |
+
|
13750 |
+
|
13751 |
+
|
13752 |
+
|
13753 |
+
|
13754 |
+
|
13755 |
+
|
13756 |
+
|
13757 |
+
|
13758 |
+
|
13759 |
+
|
13760 |
+
|
13761 |
+
|
13762 |
+
|
13763 |
+
|
13764 |
+
|
13765 |
+
|
13766 |
+
|
13767 |
+
|
13768 |
+
|
13769 |
+
|
13770 |
+
|
13771 |
+
|
13772 |
+
|
13773 |
+
|
13774 |
+
|
13775 |
+
|
13776 |
+
|
13777 |
+
|
13778 |
+
|
13779 |
+
|
13780 |
+
|
13781 |
+
|
13782 |
+
|
13783 |
+
|
13784 |
+
|
13785 |
+
|
13786 |
+
|
13787 |
+
|
13788 |
+
|
13789 |
+
|
13790 |
+
|
13791 |
+
|
13792 |
+
|
13793 |
+
|
13794 |
+
|
13795 |
+
|
13796 |
+
|
13797 |
+
|
13798 |
+
|
13799 |
+
|
13800 |
+
|
13801 |
+
|
13802 |
+
|
13803 |
+
|
13804 |
+
|
13805 |
+
|
13806 |
+
|
13807 |
+
|
13808 |
+
|
13809 |
+
|
13810 |
+
|
13811 |
+
|
13812 |
+
|
13813 |
+
|
13814 |
+
|
13815 |
+
|
13816 |
+
|
13817 |
+
|
13818 |
+
|
13819 |
+
|
13820 |
+
|
13821 |
+
|
13822 |
+
|
13823 |
+
|
13824 |
+
|
13825 |
+
|
13826 |
+
|
13827 |
+
|
13828 |
+
|
13829 |
+
|
13830 |
+
|
13831 |
+
|
13832 |
+
|
13833 |
+
|
13834 |
+
|
13835 |
+
|
13836 |
+
|
13837 |
+
|
13838 |
+
|
13839 |
+
Training...: 91%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 55517/61250 [8:13:17<20:53, 4.58it/s]
|
13840 |
+
|
13841 |
+
|
13842 |
+
|
13843 |
+
|
13844 |
+
|
13845 |
+
|
13846 |
+
|
13847 |
+
|
13848 |
+
|
13849 |
+
|
13850 |
+
|
13851 |
+
|
13852 |
+
|
13853 |
+
|
13854 |
+
|
13855 |
+
|
13856 |
+
|
13857 |
+
|
13858 |
+
|
13859 |
+
|
13860 |
+
|
13861 |
+
|
13862 |
+
|
13863 |
+
|
13864 |
+
|
13865 |
+
|
13866 |
+
|
13867 |
+
|
13868 |
+
|
13869 |
+
|
13870 |
+
|
13871 |
+
|
13872 |
+
|
13873 |
+
|
13874 |
+
|
13875 |
+
|
13876 |
+
|
13877 |
+
|
13878 |
+
|
13879 |
+
|
13880 |
+
|
13881 |
+
|
13882 |
+
|
13883 |
+
|
13884 |
+
|
13885 |
+
|
13886 |
+
|
13887 |
+
|
13888 |
+
|
13889 |
+
|
13890 |
+
|
13891 |
+
|
13892 |
+
|
13893 |
+
|
13894 |
+
|
13895 |
+
|
13896 |
+
|
13897 |
+
|
13898 |
+
|
13899 |
+
|
13900 |
+
|
13901 |
+
|
13902 |
+
|
13903 |
+
|
13904 |
+
|
13905 |
+
|
13906 |
+
|
13907 |
+
|
13908 |
+
|
13909 |
+
|
13910 |
+
|
13911 |
+
|
13912 |
+
|
13913 |
+
|
13914 |
+
|
13915 |
+
|
13916 |
+
|
13917 |
+
|
13918 |
+
|
13919 |
+
|
13920 |
+
|
13921 |
+
|
13922 |
+
|
13923 |
+
|
13924 |
+
|
13925 |
+
|
13926 |
+
|
13927 |
+
|
13928 |
+
|
13929 |
+
|
13930 |
+
|
13931 |
+
|
13932 |
+
|
13933 |
+
|
13934 |
+
|
13935 |
+
|
13936 |
+
|
13937 |
+
|
13938 |
+
|
13939 |
+
|
13940 |
+
|
13941 |
+
|
13942 |
+
|
13943 |
+
|
13944 |
+
|
13945 |
+
|
13946 |
+
|
13947 |
+
|
13948 |
+
|
13949 |
+
|
13950 |
+
|
13951 |
+
|
13952 |
+
|
13953 |
+
|
13954 |
+
Training...: 91%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████��██████████████████████ | 56011/61250 [8:17:19<47:34, 1.84it/s]
|
13955 |
+
|
13956 |
+
|
13957 |
+
|
13958 |
+
|
13959 |
+
|
13960 |
+
|
13961 |
+
|
13962 |
+
|
13963 |
+
|
13964 |
+
|
13965 |
+
|
13966 |
+
|
13967 |
+
|
13968 |
+
|
13969 |
+
|
13970 |
+
|
13971 |
+
|
13972 |
+
|
13973 |
+
|
13974 |
+
|
13975 |
+
|
13976 |
+
|
13977 |
+
|
13978 |
+
|
13979 |
+
|
13980 |
+
|
13981 |
+
|
13982 |
+
|
13983 |
+
|
13984 |
+
|
13985 |
+
|
13986 |
+
|
13987 |
+
|
13988 |
+
|
13989 |
+
|
13990 |
+
|
13991 |
+
|
13992 |
+
|
13993 |
+
|
13994 |
+
|
13995 |
+
|
13996 |
+
|
13997 |
+
|
13998 |
+
|
13999 |
+
|
14000 |
+
|
14001 |
+
|
14002 |
+
|
14003 |
+
|
14004 |
+
|
14005 |
+
|
14006 |
+
|
14007 |
+
|
14008 |
+
|
14009 |
+
|
14010 |
+
|
14011 |
+
|
14012 |
+
|
14013 |
+
|
14014 |
+
|
14015 |
+
|
14016 |
+
|
14017 |
+
|
14018 |
+
|
14019 |
+
|
14020 |
+
|
14021 |
+
|
14022 |
+
|
14023 |
+
|
14024 |
+
|
14025 |
+
|
14026 |
+
|
14027 |
+
|
14028 |
+
|
14029 |
+
|
14030 |
+
|
14031 |
+
|
14032 |
+
|
14033 |
+
|
14034 |
+
|
14035 |
+
|
14036 |
+
|
14037 |
+
|
14038 |
+
|
14039 |
+
|
14040 |
+
|
14041 |
+
|
14042 |
+
|
14043 |
+
|
14044 |
+
|
14045 |
+
|
14046 |
+
|
14047 |
+
|
14048 |
+
|
14049 |
+
|
14050 |
+
|
14051 |
+
|
14052 |
+
|
14053 |
+
|
14054 |
+
|
14055 |
+
|
14056 |
+
|
14057 |
+
|
14058 |
+
|
14059 |
+
|
14060 |
+
|
14061 |
+
|
14062 |
+
|
14063 |
+
|
14064 |
+
|
14065 |
+
|
14066 |
+
|
14067 |
+
|
14068 |
+
Training...: 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 56500/61250 [8:21:05<38:23, 2.06it/s]
|
14069 |
+
|
14070 |
+
|
14071 |
+
|
14072 |
+
|
14073 |
+
|
14074 |
+
|
14075 |
+
|
14076 |
+
|
14077 |
+
|
14078 |
+
|
14079 |
+
|
14080 |
+
|
14081 |
+
|
14082 |
+
|
14083 |
+
|
14084 |
+
|
14085 |
+
|
14086 |
+
|
14087 |
+
|
14088 |
+
|
14089 |
+
|
14090 |
+
|
14091 |
+
|
14092 |
+
|
14093 |
+
|
14094 |
+
|
14095 |
+
|
14096 |
+
|
14097 |
+
|
14098 |
+
|
14099 |
+
|
14100 |
+
|
14101 |
+
|
14102 |
+
|
14103 |
+
|
14104 |
+
|
14105 |
+
|
14106 |
+
|
14107 |
+
|
14108 |
+
|
14109 |
+
|
14110 |
+
|
14111 |
+
|
14112 |
+
|
14113 |
+
|
14114 |
+
|
14115 |
+
|
14116 |
+
|
14117 |
+
|
14118 |
+
|
14119 |
+
|
14120 |
+
|
14121 |
+
|
14122 |
+
|
14123 |
+
|
14124 |
+
|
14125 |
+
|
14126 |
+
|
14127 |
+
|
14128 |
+
|
14129 |
+
|
14130 |
+
|
14131 |
+
|
14132 |
+
|
14133 |
+
|
14134 |
+
|
14135 |
+
|
14136 |
+
|
14137 |
+
|
14138 |
+
|
14139 |
+
|
14140 |
+
|
14141 |
+
|
14142 |
+
|
14143 |
+
|
14144 |
+
|
14145 |
+
|
14146 |
+
|
14147 |
+
|
14148 |
+
|
14149 |
+
|
14150 |
+
|
14151 |
+
|
14152 |
+
|
14153 |
+
|
14154 |
+
|
14155 |
+
|
14156 |
+
|
14157 |
+
|
14158 |
+
|
14159 |
+
|
14160 |
+
|
14161 |
+
|
14162 |
+
|
14163 |
+
|
14164 |
+
|
14165 |
+
|
14166 |
+
|
14167 |
+
|
14168 |
+
|
14169 |
+
|
14170 |
+
|
14171 |
+
|
14172 |
+
|
14173 |
+
|
14174 |
+
|
14175 |
+
|
14176 |
+
|
14177 |
+
|
14178 |
+
|
14179 |
+
|
14180 |
+
|
14181 |
+
|
14182 |
+
|
14183 |
+
Training...: 93%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 57000/61250 [8:25:09<34:22, 2.06it/s]
|
14184 |
+
|
14185 |
+
|
14186 |
+
|
14187 |
+
|
14188 |
+
|
14189 |
+
|
14190 |
+
|
14191 |
+
|
14192 |
+
|
14193 |
+
|
14194 |
+
|
14195 |
+
|
14196 |
+
|
14197 |
+
|
14198 |
+
|
14199 |
+
|
14200 |
+
|
14201 |
+
|
14202 |
+
|
14203 |
+
|
14204 |
+
|
14205 |
+
|
14206 |
+
|
14207 |
+
|
14208 |
+
|
14209 |
+
|
14210 |
+
|
14211 |
+
|
14212 |
+
|
14213 |
+
|
14214 |
+
|
14215 |
+
|
14216 |
+
|
14217 |
+
|
14218 |
+
|
14219 |
+
|
14220 |
+
|
14221 |
+
|
14222 |
+
|
14223 |
+
|
14224 |
+
|
14225 |
+
|
14226 |
+
|
14227 |
+
|
14228 |
+
|
14229 |
+
|
14230 |
+
|
14231 |
+
|
14232 |
+
|
14233 |
+
|
14234 |
+
|
14235 |
+
|
14236 |
+
|
14237 |
+
|
14238 |
+
|
14239 |
+
|
14240 |
+
|
14241 |
+
|
14242 |
+
|
14243 |
+
|
14244 |
+
|
14245 |
+
|
14246 |
+
|
14247 |
+
|
14248 |
+
|
14249 |
+
|
14250 |
+
|
14251 |
+
|
14252 |
+
|
14253 |
+
|
14254 |
+
|
14255 |
+
|
14256 |
+
|
14257 |
+
|
14258 |
+
|
14259 |
+
|
14260 |
+
|
14261 |
+
|
14262 |
+
|
14263 |
+
|
14264 |
+
|
14265 |
+
|
14266 |
+
|
14267 |
+
|
14268 |
+
|
14269 |
+
|
14270 |
+
|
14271 |
+
|
14272 |
+
|
14273 |
+
|
14274 |
+
|
14275 |
+
|
14276 |
+
|
14277 |
+
|
14278 |
+
|
14279 |
+
|
14280 |
+
|
14281 |
+
|
14282 |
+
|
14283 |
+
|
14284 |
+
|
14285 |
+
|
14286 |
+
|
14287 |
+
|
14288 |
+
|
14289 |
+
|
14290 |
+
|
14291 |
+
|
14292 |
+
|
14293 |
+
|
14294 |
+
|
14295 |
+
|
14296 |
+
|
14297 |
+
Training...: 94%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 57500/61250 [8:29:12<30:19, 2.06it/s]
|
14298 |
+
|
14299 |
+
|
14300 |
+
|
14301 |
+
|
14302 |
+
|
14303 |
+
|
14304 |
+
|
14305 |
+
|
14306 |
+
|
14307 |
+
|
14308 |
+
|
14309 |
+
|
14310 |
+
|
14311 |
+
|
14312 |
+
|
14313 |
+
|
14314 |
+
|
14315 |
+
|
14316 |
+
|
14317 |
+
|
14318 |
+
|
14319 |
+
|
14320 |
+
|
14321 |
+
|
14322 |
+
|
14323 |
+
|
14324 |
+
|
14325 |
+
|
14326 |
+
|
14327 |
+
|
14328 |
+
|
14329 |
+
|
14330 |
+
|
14331 |
+
|
14332 |
+
|
14333 |
+
|
14334 |
+
|
14335 |
+
|
14336 |
+
|
14337 |
+
|
14338 |
+
|
14339 |
+
|
14340 |
+
|
14341 |
+
|
14342 |
+
|
14343 |
+
|
14344 |
+
|
14345 |
+
|
14346 |
+
|
14347 |
+
|
14348 |
+
|
14349 |
+
|
14350 |
+
|
14351 |
+
|
14352 |
+
|
14353 |
+
|
14354 |
+
|
14355 |
+
|
14356 |
+
|
14357 |
+
|
14358 |
+
|
14359 |
+
|
14360 |
+
|
14361 |
+
|
14362 |
+
|
14363 |
+
|
14364 |
+
|
14365 |
+
|
14366 |
+
|
14367 |
+
|
14368 |
+
|
14369 |
+
|
14370 |
+
|
14371 |
+
|
14372 |
+
|
14373 |
+
|
14374 |
+
|
14375 |
+
|
14376 |
+
|
14377 |
+
|
14378 |
+
|
14379 |
+
|
14380 |
+
|
14381 |
+
|
14382 |
+
|
14383 |
+
|
14384 |
+
|
14385 |
+
|
14386 |
+
|
14387 |
+
|
14388 |
+
|
14389 |
+
|
14390 |
+
|
14391 |
+
|
14392 |
+
|
14393 |
+
|
14394 |
+
|
14395 |
+
|
14396 |
+
|
14397 |
+
|
14398 |
+
|
14399 |
+
|
14400 |
+
|
14401 |
+
|
14402 |
+
|
14403 |
+
|
14404 |
+
|
14405 |
+
|
14406 |
+
|
14407 |
+
|
14408 |
+
|
14409 |
+
|
14410 |
+
|
14411 |
+
Training...: 95%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 58000/61250 [8:33:15<26:16, 2.06it/s]
|
14412 |
+
|
14413 |
+
|
14414 |
+
|
14415 |
+
|
14416 |
+
|
14417 |
+
|
14418 |
+
|
14419 |
+
|
14420 |
+
|
14421 |
+
|
14422 |
+
|
14423 |
+
|
14424 |
+
|
14425 |
+
|
14426 |
+
|
14427 |
+
|
14428 |
+
|
14429 |
+
|
14430 |
+
|
14431 |
+
|
14432 |
+
|
14433 |
+
|
14434 |
+
|
14435 |
+
|
14436 |
+
|
14437 |
+
|
14438 |
+
|
14439 |
+
|
14440 |
+
|
14441 |
+
|
14442 |
+
|
14443 |
+
|
14444 |
+
|
14445 |
+
|
14446 |
+
|
14447 |
+
|
14448 |
+
|
14449 |
+
|
14450 |
+
|
14451 |
+
|
14452 |
+
|
14453 |
+
|
14454 |
+
|
14455 |
+
|
14456 |
+
|
14457 |
+
|
14458 |
+
|
14459 |
+
|
14460 |
+
|
14461 |
+
|
14462 |
+
|
14463 |
+
|
14464 |
+
|
14465 |
+
|
14466 |
+
|
14467 |
+
|
14468 |
+
|
14469 |
+
|
14470 |
+
|
14471 |
+
|
14472 |
+
|
14473 |
+
|
14474 |
+
|
14475 |
+
|
14476 |
+
|
14477 |
+
|
14478 |
+
|
14479 |
+
|
14480 |
+
|
14481 |
+
|
14482 |
+
|
14483 |
+
|
14484 |
+
|
14485 |
+
|
14486 |
+
|
14487 |
+
|
14488 |
+
|
14489 |
+
|
14490 |
+
|
14491 |
+
|
14492 |
+
|
14493 |
+
|
14494 |
+
|
14495 |
+
|
14496 |
+
|
14497 |
+
|
14498 |
+
|
14499 |
+
|
14500 |
+
|
14501 |
+
|
14502 |
+
|
14503 |
+
|
14504 |
+
|
14505 |
+
|
14506 |
+
|
14507 |
+
|
14508 |
+
|
14509 |
+
|
14510 |
+
|
14511 |
+
|
14512 |
+
|
14513 |
+
|
14514 |
+
|
14515 |
+
|
14516 |
+
|
14517 |
+
|
14518 |
+
|
14519 |
+
|
14520 |
+
|
14521 |
+
|
14522 |
+
|
14523 |
+
|
14524 |
+
|
14525 |
+
Training...: 96%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 58500/61250 [8:37:18<22:13, 2.06it/s]
|
14526 |
+
|
14527 |
+
|
14528 |
+
|
14529 |
+
|
14530 |
+
|
14531 |
+
|
14532 |
+
|
14533 |
+
|
14534 |
+
|
14535 |
+
|
14536 |
+
|
14537 |
+
|
14538 |
+
|
14539 |
+
|
14540 |
+
|
14541 |
+
|
14542 |
+
|
14543 |
+
|
14544 |
+
|
14545 |
+
|
14546 |
+
|
14547 |
+
|
14548 |
+
|
14549 |
+
|
14550 |
+
|
14551 |
+
|
14552 |
+
|
14553 |
+
|
14554 |
+
|
14555 |
+
|
14556 |
+
|
14557 |
+
|
14558 |
+
|
14559 |
+
|
14560 |
+
|
14561 |
+
|
14562 |
+
|
14563 |
+
|
14564 |
+
|
14565 |
+
|
14566 |
+
|
14567 |
+
|
14568 |
+
|
14569 |
+
|
14570 |
+
|
14571 |
+
|
14572 |
+
|
14573 |
+
|
14574 |
+
|
14575 |
+
|
14576 |
+
|
14577 |
+
|
14578 |
+
|
14579 |
+
|
14580 |
+
|
14581 |
+
|
14582 |
+
|
14583 |
+
|
14584 |
+
|
14585 |
+
|
14586 |
+
|
14587 |
+
|
14588 |
+
|
14589 |
+
|
14590 |
+
|
14591 |
+
|
14592 |
+
|
14593 |
+
|
14594 |
+
|
14595 |
+
|
14596 |
+
|
14597 |
+
|
14598 |
+
|
14599 |
+
|
14600 |
+
|
14601 |
+
|
14602 |
+
|
14603 |
+
|
14604 |
+
|
14605 |
+
|
14606 |
+
|
14607 |
+
|
14608 |
+
|
14609 |
+
|
14610 |
+
|
14611 |
+
|
14612 |
+
|
14613 |
+
|
14614 |
+
|
14615 |
+
|
14616 |
+
|
14617 |
+
|
14618 |
+
|
14619 |
+
|
14620 |
+
|
14621 |
+
|
14622 |
+
|
14623 |
+
|
14624 |
+
|
14625 |
+
|
14626 |
+
|
14627 |
+
|
14628 |
+
|
14629 |
+
|
14630 |
+
|
14631 |
+
|
14632 |
+
|
14633 |
+
|
14634 |
+
|
14635 |
+
|
14636 |
+
|
14637 |
+
|
14638 |
+
|
14639 |
+
Training...: 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 59000/61250 [8:41:21<18:10, 2.06it/s]
|
14640 |
+
|
14641 |
+
|
14642 |
+
|
14643 |
+
|
14644 |
+
|
14645 |
+
|
14646 |
+
|
14647 |
+
|
14648 |
+
|
14649 |
+
|
14650 |
+
|
14651 |
+
|
14652 |
+
|
14653 |
+
|
14654 |
+
|
14655 |
+
|
14656 |
+
|
14657 |
+
|
14658 |
+
|
14659 |
+
|
14660 |
+
|
14661 |
+
|
14662 |
+
|
14663 |
+
|
14664 |
+
|
14665 |
+
|
14666 |
+
|
14667 |
+
|
14668 |
+
|
14669 |
+
|
14670 |
+
|
14671 |
+
|
14672 |
+
|
14673 |
+
|
14674 |
+
|
14675 |
+
|
14676 |
+
|
14677 |
+
|
14678 |
+
|
14679 |
+
|
14680 |
+
|
14681 |
+
|
14682 |
+
|
14683 |
+
|
14684 |
+
|
14685 |
+
|
14686 |
+
|
14687 |
+
|
14688 |
+
|
14689 |
+
|
14690 |
+
|
14691 |
+
|
14692 |
+
|
14693 |
+
|
14694 |
+
|
14695 |
+
|
14696 |
+
|
14697 |
+
|
14698 |
+
|
14699 |
+
|
14700 |
+
|
14701 |
+
|
14702 |
+
|
14703 |
+
|
14704 |
+
|
14705 |
+
|
14706 |
+
|
14707 |
+
|
14708 |
+
|
14709 |
+
|
14710 |
+
|
14711 |
+
|
14712 |
+
|
14713 |
+
|
14714 |
+
|
14715 |
+
|
14716 |
+
|
14717 |
+
|
14718 |
+
|
14719 |
+
|
14720 |
+
|
14721 |
+
|
14722 |
+
|
14723 |
+
|
14724 |
+
|
14725 |
+
|
14726 |
+
|
14727 |
+
|
14728 |
+
|
14729 |
+
|
14730 |
+
|
14731 |
+
|
14732 |
+
|
14733 |
+
|
14734 |
+
|
14735 |
+
|
14736 |
+
|
14737 |
+
|
14738 |
+
|
14739 |
+
|
14740 |
+
|
14741 |
+
|
14742 |
+
|
14743 |
+
|
14744 |
+
|
14745 |
+
|
14746 |
+
|
14747 |
+
|
14748 |
+
|
14749 |
+
|
14750 |
+
|
14751 |
+
|
14752 |
+
|
14753 |
+
|
14754 |
+
Training...: 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋ | 59525/61250 [8:45:43<02:52, 10.00it/s]
|
14755 |
+
|
14756 |
+
|
14757 |
+
|
14758 |
+
|
14759 |
+
|
14760 |
+
|
14761 |
+
|
14762 |
+
|
14763 |
+
|
14764 |
+
|
14765 |
+
|
14766 |
+
|
14767 |
+
|
14768 |
+
|
14769 |
+
|
14770 |
+
|
14771 |
+
|
14772 |
+
|
14773 |
+
|
14774 |
+
|
14775 |
+
|
14776 |
+
|
14777 |
+
|
14778 |
+
|
14779 |
+
|
14780 |
+
|
14781 |
+
|
14782 |
+
|
14783 |
+
|
14784 |
+
|
14785 |
+
|
14786 |
+
|
14787 |
+
|
14788 |
+
|
14789 |
+
|
14790 |
+
|
14791 |
+
|
14792 |
+
|
14793 |
+
|
14794 |
+
|
14795 |
+
|
14796 |
+
|
14797 |
+
|
14798 |
+
|
14799 |
+
|
14800 |
+
|
14801 |
+
|
14802 |
+
|
14803 |
+
|
14804 |
+
|
14805 |
+
|
14806 |
+
|
14807 |
+
|
14808 |
+
|
14809 |
+
|
14810 |
+
|
14811 |
+
|
14812 |
+
|
14813 |
+
|
14814 |
+
|
14815 |
+
|
14816 |
+
|
14817 |
+
|
14818 |
+
|
14819 |
+
|
14820 |
+
|
14821 |
+
|
14822 |
+
|
14823 |
+
|
14824 |
+
|
14825 |
+
|
14826 |
+
|
14827 |
+
|
14828 |
+
|
14829 |
+
|
14830 |
+
|
14831 |
+
|
14832 |
+
|
14833 |
+
|
14834 |
+
|
14835 |
+
|
14836 |
+
|
14837 |
+
|
14838 |
+
|
14839 |
+
|
14840 |
+
|
14841 |
+
|
14842 |
+
|
14843 |
+
|
14844 |
+
|
14845 |
+
|
14846 |
+
|
14847 |
+
|
14848 |
+
|
14849 |
+
|
14850 |
+
|
14851 |
+
|
14852 |
+
|
14853 |
+
|
14854 |
+
|
14855 |
+
|
14856 |
+
|
14857 |
+
|
14858 |
+
|
14859 |
+
|
14860 |
+
|
14861 |
+
|
14862 |
+
|
14863 |
+
|
14864 |
+
|
14865 |
+
|
14866 |
+
|
14867 |
+
|
14868 |
+
Training...: 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 60000/61250 [8:49:28<10:06, 2.06it/s]
|
14869 |
+
Training...: 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 60000/61250 [8:49:44<10:06, 2.06it/s]
|
14870 |
+
|
14871 |
+
|
14872 |
+
|
14873 |
+
|
14874 |
+
|
14875 |
+
|
14876 |
+
|
14877 |
+
|
14878 |
+
|
14879 |
+
|
14880 |
+
|
14881 |
+
|
14882 |
+
|
14883 |
+
|
14884 |
+
|
14885 |
+
|
14886 |
+
|
14887 |
+
|
14888 |
+
|
14889 |
+
|
14890 |
+
|
14891 |
+
|
14892 |
+
|
14893 |
+
|
14894 |
+
|
14895 |
+
|
14896 |
+
|
14897 |
+
|
14898 |
+
|
14899 |
+
|
14900 |
+
|
14901 |
+
|
14902 |
+
|
14903 |
+
|
14904 |
+
|
14905 |
+
|
14906 |
+
|
14907 |
+
|
14908 |
+
|
14909 |
+
|
14910 |
+
|
14911 |
+
|
14912 |
+
|
14913 |
+
|
14914 |
+
|
14915 |
+
|
14916 |
+
|
14917 |
+
|
14918 |
+
|
14919 |
+
|
14920 |
+
|
14921 |
+
|
14922 |
+
|
14923 |
+
|
14924 |
+
|
14925 |
+
|
14926 |
+
|
14927 |
+
|
14928 |
+
|
14929 |
+
|
14930 |
+
|
14931 |
+
|
14932 |
+
|
14933 |
+
|
14934 |
+
|
14935 |
+
|
14936 |
+
|
14937 |
+
|
14938 |
+
|
14939 |
+
|
14940 |
+
|
14941 |
+
|
14942 |
+
|
14943 |
+
|
14944 |
+
|
14945 |
+
|
14946 |
+
|
14947 |
+
|
14948 |
+
|
14949 |
+
|
14950 |
+
|
14951 |
+
|
14952 |
+
|
14953 |
+
|
14954 |
+
|
14955 |
+
|
14956 |
+
|
14957 |
+
|
14958 |
+
|
14959 |
+
|
14960 |
+
|
14961 |
+
|
14962 |
+
|
14963 |
+
|
14964 |
+
|
14965 |
+
|
14966 |
+
|
14967 |
+
|
14968 |
+
|
14969 |
+
|
14970 |
+
|
14971 |
+
|
14972 |
+
|
14973 |
+
|
14974 |
+
|
14975 |
+
|
14976 |
+
|
14977 |
+
|
14978 |
+
|
14979 |
+
|
14980 |
+
|
14981 |
+
|
14982 |
+
|
14983 |
+
|
14984 |
+
|
14985 |
+
|
14986 |
+
|
14987 |
+
|
14988 |
+
|
14989 |
+
|
14990 |
+
|
14991 |
+
|
14992 |
+
|
14993 |
+
|
14994 |
+
|
14995 |
+
|
14996 |
+
|
14997 |
+
|
14998 |
+
|
14999 |
+
|
15000 |
+
|
15001 |
+
|
15002 |
+
|
15003 |
+
|
15004 |
+
|
15005 |
+
|
15006 |
+
|
15007 |
+
|
15008 |
+
|
15009 |
+
|
15010 |
+
|
15011 |
+
|
15012 |
+
|
15013 |
+
|
15014 |
+
|
15015 |
+
|
15016 |
+
|
15017 |
+
|
15018 |
+
|
15019 |
+
|
15020 |
+
|
15021 |
+
|
15022 |
+
|
15023 |
+
|
15024 |
+
|
15025 |
+
|
15026 |
+
|
15027 |
+
|
15028 |
+
|
15029 |
+
|
15030 |
+
|
15031 |
+
|
15032 |
+
|
15033 |
+
|
15034 |
+
|
15035 |
+
|
15036 |
+
|
15037 |
+
|
15038 |
+
|
15039 |
+
|
15040 |
+
|
15041 |
+
|
15042 |
+
|
15043 |
+
|
15044 |
+
|
15045 |
+
|
15046 |
+
|
15047 |
+
|
15048 |
+
|
15049 |
+
|
15050 |
+
|
15051 |
+
|
15052 |
+
|
15053 |
+
|
15054 |
+
|
15055 |
+
|
15056 |
+
|
15057 |
+
|
15058 |
+
|
15059 |
+
|
15060 |
+
|
15061 |
+
|
15062 |
+
|
15063 |
+
|
15064 |
+
|
15065 |
+
|
15066 |
+
|
15067 |
+
|
15068 |
+
|
15069 |
+
|
15070 |
+
|
15071 |
+
|
15072 |
+
|
15073 |
+
Training...: 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊ | 60521/61250 [8:56:56<01:41, 7.15it/s]
|
15074 |
+
|
15075 |
+
|
15076 |
+
|
15077 |
+
|
15078 |
+
|
15079 |
+
|
15080 |
+
|
15081 |
+
|
15082 |
+
|
15083 |
+
|
15084 |
+
|
15085 |
+
|
15086 |
+
|
15087 |
+
|
15088 |
+
|
15089 |
+
|
15090 |
+
|
15091 |
+
|
15092 |
+
|
15093 |
+
|
15094 |
+
|
15095 |
+
|
15096 |
+
|
15097 |
+
|
15098 |
+
|
15099 |
+
|
15100 |
+
|
15101 |
+
|
15102 |
+
|
15103 |
+
|
15104 |
+
|
15105 |
+
|
15106 |
+
|
15107 |
+
|
15108 |
+
|
15109 |
+
|
15110 |
+
|
15111 |
+
|
15112 |
+
|
15113 |
+
|
15114 |
+
|
15115 |
+
|
15116 |
+
|
15117 |
+
|
15118 |
+
|
15119 |
+
|
15120 |
+
|
15121 |
+
|
15122 |
+
|
15123 |
+
|
15124 |
+
|
15125 |
+
|
15126 |
+
|
15127 |
+
|
15128 |
+
|
15129 |
+
|
15130 |
+
|
15131 |
+
|
15132 |
+
|
15133 |
+
|
15134 |
+
|
15135 |
+
|
15136 |
+
|
15137 |
+
|
15138 |
+
|
15139 |
+
|
15140 |
+
|
15141 |
+
|
15142 |
+
|
15143 |
+
|
15144 |
+
|
15145 |
+
|
15146 |
+
|
15147 |
+
|
15148 |
+
|
15149 |
+
|
15150 |
+
|
15151 |
+
|
15152 |
+
|
15153 |
+
|
15154 |
+
|
15155 |
+
|
15156 |
+
|
15157 |
+
|
15158 |
+
|
15159 |
+
|
15160 |
+
|
15161 |
+
|
15162 |
+
|
15163 |
+
|
15164 |
+
|
15165 |
+
|
15166 |
+
|
15167 |
+
|
15168 |
+
|
15169 |
+
|
15170 |
+
|
15171 |
+
|
15172 |
+
|
15173 |
+
|
15174 |
+
|
15175 |
+
|
15176 |
+
|
15177 |
+
|
15178 |
+
|
15179 |
+
|
15180 |
+
|
15181 |
+
|
15182 |
+
|
15183 |
+
|
15184 |
+
|
15185 |
+
|
15186 |
+
|
15187 |
+
Training...: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏| 61000/61250 [9:00:42<02:01, 2.06it/s]
|
15188 |
+
|
15189 |
+
|
15190 |
+
|
15191 |
+
|
15192 |
+
|
15193 |
+
|
15194 |
+
|
15195 |
+
|
15196 |
+
|
15197 |
+
|
15198 |
+
|
15199 |
+
|
15200 |
+
|
15201 |
+
|
15202 |
+
|
15203 |
+
|
15204 |
+
|
15205 |
+
|
15206 |
+
|
15207 |
+
|
15208 |
+
|
15209 |
+
|
15210 |
+
|
15211 |
+
|
15212 |
+
|
15213 |
+
|
15214 |
+
|
15215 |
+
|
15216 |
+
|
15217 |
+
|
15218 |
+
|
15219 |
+
|
15220 |
+
|
15221 |
+
|
15222 |
+
|
15223 |
+
|
15224 |
+
|
15225 |
+
|
15226 |
+
|
15227 |
+
|
15228 |
+
|
15229 |
+
|
15230 |
+
|
15231 |
+
|
15232 |
+
|
15233 |
+
|
15234 |
+
|
15235 |
+
|
15236 |
+
|
15237 |
+
|
15238 |
+
|
15239 |
+
|
15240 |
+
|
15241 |
+
Step... (110000 | Loss: 1.404970645904541, Acc: 0.7204243540763855): 33%|████████████████████████████████████████████▎ | 1/3 [9:03:51<18:07:42, 32631.03s/it]
|
15242 |
+
|
15243 |
+
|
15244 |
+
|
15245 |
+
|
15246 |
+
|
15247 |
+
|
15248 |
+
|
15249 |
+
|
15250 |
+
|
15251 |
+
|
15252 |
+
|
15253 |
+
|
15254 |
+
|
15255 |
+
|
15256 |
+
|
15257 |
+
|
15258 |
+
|
15259 |
+
|
15260 |
+
|
15261 |
+
|
15262 |
+
|
15263 |
+
|
15264 |
+
|
15265 |
+
|
15266 |
+
|
15267 |
+
|
15268 |
+
|
15269 |
+
|
15270 |
+
|
15271 |
+
|
15272 |
+
|
15273 |
+
|
15274 |
+
|
15275 |
+
|
15276 |
+
|
15277 |
+
|
15278 |
+
|
15279 |
+
|
15280 |
+
|
15281 |
+
|
15282 |
+
|
15283 |
+
|
15284 |
+
|
15285 |
+
|
15286 |
+
|
15287 |
+
|
15288 |
+
|
15289 |
+
|
15290 |
+
|
15291 |
+
|
15292 |
+
|
15293 |
+
|
15294 |
+
|
15295 |
+
Training...: 0%|▊ | 251/61250 [02:02<88:28:04, 5.22s/it]
|
15296 |
+
|
15297 |
+
|
15298 |
+
|
15299 |
+
|
15300 |
+
|
15301 |
+
|
15302 |
+
|
15303 |
+
|
15304 |
+
|
15305 |
+
|
15306 |
+
|
15307 |
+
|
15308 |
+
|
15309 |
+
|
15310 |
+
|
15311 |
+
|
15312 |
+
|
15313 |
+
|
15314 |
+
|
15315 |
+
|
15316 |
+
|
15317 |
+
|
15318 |
+
|
15319 |
+
|
15320 |
+
|
15321 |
+
|
15322 |
+
|
15323 |
+
|
15324 |
+
|
15325 |
+
|
15326 |
+
|
15327 |
+
|
15328 |
+
|
15329 |
+
|
15330 |
+
|
15331 |
+
|
15332 |
+
|
15333 |
+
|
15334 |
+
|
15335 |
+
|
15336 |
+
|
15337 |
+
|
15338 |
+
|
15339 |
+
|
15340 |
+
|
15341 |
+
|
15342 |
+
|
15343 |
+
|
15344 |
+
|
15345 |
+
|
15346 |
+
|
15347 |
+
|
15348 |
+
|
15349 |
+
|
15350 |
+
|
15351 |
+
|
15352 |
+
|
15353 |
+
|
15354 |
+
|
15355 |
+
|
15356 |
+
|
15357 |
+
|
15358 |
+
|
15359 |
+
|
15360 |
+
|
15361 |
+
|
15362 |
+
|
15363 |
+
|
15364 |
+
|
15365 |
+
|
15366 |
+
|
15367 |
+
|
15368 |
+
|
15369 |
+
|
15370 |
+
|
15371 |
+
|
15372 |
+
|
15373 |
+
|
15374 |
+
|
15375 |
+
|
15376 |
+
|
15377 |
+
|
15378 |
+
|
15379 |
+
|
15380 |
+
|
15381 |
+
|
15382 |
+
|
15383 |
+
|
15384 |
+
|
15385 |
+
|
15386 |
+
|
15387 |
+
|
15388 |
+
|
15389 |
+
|
15390 |
+
|
15391 |
+
|
15392 |
+
|
15393 |
+
|
15394 |
+
|
15395 |
+
|
15396 |
+
|
15397 |
+
|
15398 |
+
|
15399 |
+
|
15400 |
+
|
15401 |
+
|
15402 |
+
|
15403 |
+
|
15404 |
+
|
15405 |
+
|
15406 |
+
|
15407 |
+
|
15408 |
+
|
15409 |
+
Training...: 1%|██▎ | 750/61250 [05:48<8:09:04, 2.06it/s]
|
15410 |
+
|
15411 |
+
|
15412 |
+
|
15413 |
+
|
15414 |
+
|
15415 |
+
|
15416 |
+
|
15417 |
+
|
15418 |
+
|
15419 |
+
|
15420 |
+
|
15421 |
+
|
15422 |
+
|
15423 |
+
|
15424 |
+
|
15425 |
+
|
15426 |
+
|
15427 |
+
|
15428 |
+
|
15429 |
+
|
15430 |
+
|
15431 |
+
|
15432 |
+
|
15433 |
+
|
15434 |
+
|
15435 |
+
|
15436 |
+
|
15437 |
+
|
15438 |
+
|
15439 |
+
|
15440 |
+
|
15441 |
+
|
15442 |
+
|
15443 |
+
|
15444 |
+
|
15445 |
+
|
15446 |
+
|
15447 |
+
|
15448 |
+
|
15449 |
+
|
15450 |
+
|
15451 |
+
|
15452 |
+
|
15453 |
+
|
15454 |
+
|
15455 |
+
|
15456 |
+
|
15457 |
+
|
15458 |
+
|
15459 |
+
|
15460 |
+
|
15461 |
+
|
15462 |
+
|
15463 |
+
|
15464 |
+
|
15465 |
+
|
15466 |
+
|
15467 |
+
|
15468 |
+
|
15469 |
+
|
15470 |
+
|
15471 |
+
|
15472 |
+
|
15473 |
+
|
15474 |
+
|
15475 |
+
|
15476 |
+
|
15477 |
+
|
15478 |
+
|
15479 |
+
|
15480 |
+
|
15481 |
+
|
15482 |
+
|
15483 |
+
|
15484 |
+
|
15485 |
+
|
15486 |
+
|
15487 |
+
|
15488 |
+
|
15489 |
+
|
15490 |
+
|
15491 |
+
|
15492 |
+
|
15493 |
+
|
15494 |
+
|
15495 |
+
|
15496 |
+
|
15497 |
+
|
15498 |
+
|
15499 |
+
|
15500 |
+
|
15501 |
+
|
15502 |
+
|
15503 |
+
|
15504 |
+
|
15505 |
+
|
15506 |
+
|
15507 |
+
|
15508 |
+
|
15509 |
+
|
15510 |
+
|
15511 |
+
|
15512 |
+
|
15513 |
+
|
15514 |
+
|
15515 |
+
|
15516 |
+
|
15517 |
+
|
15518 |
+
|
15519 |
+
|
15520 |
+
|
15521 |
+
|
15522 |
+
|
15523 |
+
Training...: 2%|███▊ | 1250/61250 [09:51<8:05:02, 2.06it/s]
|
15524 |
+
|
15525 |
+
|
15526 |
+
|
15527 |
+
|
15528 |
+
|
15529 |
+
|
15530 |
+
|
15531 |
+
|
15532 |
+
|
15533 |
+
|
15534 |
+
|
15535 |
+
|
15536 |
+
|
15537 |
+
|
15538 |
+
|
15539 |
+
|
15540 |
+
|
15541 |
+
|
15542 |
+
|
15543 |
+
|
15544 |
+
|
15545 |
+
|
15546 |
+
|
15547 |
+
|
15548 |
+
|
15549 |
+
|
15550 |
+
|
15551 |
+
|
15552 |
+
|
15553 |
+
|
15554 |
+
|
15555 |
+
|
15556 |
+
|
15557 |
+
|
15558 |
+
|
15559 |
+
|
15560 |
+
|
15561 |
+
|
15562 |
+
|
15563 |
+
|
15564 |
+
|
15565 |
+
|
15566 |
+
|
15567 |
+
|
15568 |
+
|
15569 |
+
|
15570 |
+
|
15571 |
+
|
15572 |
+
|
15573 |
+
|
15574 |
+
|
15575 |
+
|
15576 |
+
|
15577 |
+
|
15578 |
+
|
15579 |
+
|
15580 |
+
|
15581 |
+
|
15582 |
+
|
15583 |
+
|
15584 |
+
|
15585 |
+
|
15586 |
+
|
15587 |
+
|
15588 |
+
|
15589 |
+
|
15590 |
+
|
15591 |
+
|
15592 |
+
|
15593 |
+
|
15594 |
+
|
15595 |
+
|
15596 |
+
|
15597 |
+
|
15598 |
+
|
15599 |
+
|
15600 |
+
|
15601 |
+
|
15602 |
+
|
15603 |
+
|
15604 |
+
|
15605 |
+
|
15606 |
+
|
15607 |
+
|
15608 |
+
|
15609 |
+
|
15610 |
+
|
15611 |
+
|
15612 |
+
|
15613 |
+
|
15614 |
+
|
15615 |
+
|
15616 |
+
|
15617 |
+
|
15618 |
+
|
15619 |
+
|
15620 |
+
|
15621 |
+
|
15622 |
+
|
15623 |
+
|
15624 |
+
|
15625 |
+
|
15626 |
+
|
15627 |
+
|
15628 |
+
|
15629 |
+
|
15630 |
+
|
15631 |
+
|
15632 |
+
|
15633 |
+
|
15634 |
+
|
15635 |
+
|
15636 |
+
|
15637 |
+
Training...: 3%|█████▎ | 1750/61250 [13:55<8:01:25, 2.06it/s]
|
15638 |
+
|
15639 |
+
|
15640 |
+
|
15641 |
+
|
15642 |
+
|
15643 |
+
|
15644 |
+
|
15645 |
+
|
15646 |
+
|
15647 |
+
|
15648 |
+
|
15649 |
+
|
15650 |
+
|
15651 |
+
|
15652 |
+
|
15653 |
+
|
15654 |
+
|
15655 |
+
|
15656 |
+
|
15657 |
+
|
15658 |
+
|
15659 |
+
|
15660 |
+
|
15661 |
+
|
15662 |
+
|
15663 |
+
|
15664 |
+
|
15665 |
+
|
15666 |
+
|
15667 |
+
|
15668 |
+
|
15669 |
+
|
15670 |
+
|
15671 |
+
|
15672 |
+
|
15673 |
+
|
15674 |
+
|
15675 |
+
|
15676 |
+
|
15677 |
+
|
15678 |
+
|
15679 |
+
|
15680 |
+
|
15681 |
+
|
15682 |
+
|
15683 |
+
|
15684 |
+
|
15685 |
+
|
15686 |
+
|
15687 |
+
|
15688 |
+
|
15689 |
+
|
15690 |
+
|
15691 |
+
|
15692 |
+
|
15693 |
+
|
15694 |
+
|
15695 |
+
|
15696 |
+
|
15697 |
+
|
15698 |
+
|
15699 |
+
|
15700 |
+
|
15701 |
+
|
15702 |
+
|
15703 |
+
|
15704 |
+
|
15705 |
+
|
15706 |
+
|
15707 |
+
|
15708 |
+
|
15709 |
+
|
15710 |
+
|
15711 |
+
|
15712 |
+
|
15713 |
+
|
15714 |
+
|
15715 |
+
|
15716 |
+
|
15717 |
+
|
15718 |
+
|
15719 |
+
|
15720 |
+
|
15721 |
+
|
15722 |
+
|
15723 |
+
|
15724 |
+
|
15725 |
+
|
15726 |
+
|
15727 |
+
|
15728 |
+
|
15729 |
+
|
15730 |
+
|
15731 |
+
|
15732 |
+
|
15733 |
+
|
15734 |
+
|
15735 |
+
|
15736 |
+
|
15737 |
+
|
15738 |
+
|
15739 |
+
|
15740 |
+
|
15741 |
+
|
15742 |
+
|
15743 |
+
|
15744 |
+
|
15745 |
+
|
15746 |
+
|
15747 |
+
|
15748 |
+
|
15749 |
+
|
15750 |
+
|
15751 |
+
|
15752 |
+
Training...: 4%|██████▉ | 2277/61250 [18:16<1:27:14, 11.27it/s]
|
15753 |
+
|
15754 |
+
|
15755 |
+
|
15756 |
+
|
15757 |
+
|
15758 |
+
|
15759 |
+
|
15760 |
+
|
15761 |
+
|
15762 |
+
|
15763 |
+
|
15764 |
+
|
15765 |
+
|
15766 |
+
|
15767 |
+
|
15768 |
+
|
15769 |
+
|
15770 |
+
|
15771 |
+
|
15772 |
+
|
15773 |
+
|
15774 |
+
|
15775 |
+
|
15776 |
+
|
15777 |
+
|
15778 |
+
|
15779 |
+
|
15780 |
+
|
15781 |
+
|
15782 |
+
|
15783 |
+
|
15784 |
+
|
15785 |
+
|
15786 |
+
|
15787 |
+
|
15788 |
+
|
15789 |
+
|
15790 |
+
|
15791 |
+
|
15792 |
+
|
15793 |
+
|
15794 |
+
|
15795 |
+
|
15796 |
+
|
15797 |
+
|
15798 |
+
|
15799 |
+
|
15800 |
+
|
15801 |
+
|
15802 |
+
|
15803 |
+
|
15804 |
+
|
15805 |
+
|
15806 |
+
|
15807 |
+
|
15808 |
+
|
15809 |
+
|
15810 |
+
|
15811 |
+
|
15812 |
+
|
15813 |
+
|
15814 |
+
|
15815 |
+
|
15816 |
+
|
15817 |
+
|
15818 |
+
|
15819 |
+
|
15820 |
+
|
15821 |
+
|
15822 |
+
|
15823 |
+
|
15824 |
+
|
15825 |
+
|
15826 |
+
|
15827 |
+
|
15828 |
+
|
15829 |
+
|
15830 |
+
|
15831 |
+
|
15832 |
+
|
15833 |
+
|
15834 |
+
|
15835 |
+
|
15836 |
+
|
15837 |
+
|
15838 |
+
|
15839 |
+
|
15840 |
+
|
15841 |
+
|
15842 |
+
|
15843 |
+
|
15844 |
+
|
15845 |
+
|
15846 |
+
|
15847 |
+
|
15848 |
+
|
15849 |
+
|
15850 |
+
|
15851 |
+
|
15852 |
+
|
15853 |
+
|
15854 |
+
|
15855 |
+
|
15856 |
+
|
15857 |
+
|
15858 |
+
|
15859 |
+
|
15860 |
+
|
15861 |
+
|
15862 |
+
|
15863 |
+
|
15864 |
+
|
15865 |
+
Training...: 4%|████████▍ | 2750/61250 [22:01<7:52:56, 2.06it/s]
|
15866 |
+
|
15867 |
+
|
15868 |
+
|
15869 |
+
|
15870 |
+
|
15871 |
+
|
15872 |
+
|
15873 |
+
|
15874 |
+
|
15875 |
+
|
15876 |
+
|
15877 |
+
|
15878 |
+
|
15879 |
+
|
15880 |
+
|
15881 |
+
|
15882 |
+
|
15883 |
+
|
15884 |
+
|
15885 |
+
|
15886 |
+
|
15887 |
+
|
15888 |
+
|
15889 |
+
|
15890 |
+
|
15891 |
+
|
15892 |
+
|
15893 |
+
|
15894 |
+
|
15895 |
+
|
15896 |
+
|
15897 |
+
|
15898 |
+
|
15899 |
+
|
15900 |
+
|
15901 |
+
|
15902 |
+
|
15903 |
+
|
15904 |
+
|
15905 |
+
|
15906 |
+
|
15907 |
+
|
15908 |
+
|
15909 |
+
|
15910 |
+
|
15911 |
+
|
15912 |
+
|
15913 |
+
|
15914 |
+
|
15915 |
+
|
15916 |
+
|
15917 |
+
|
15918 |
+
|
15919 |
+
|
15920 |
+
|
15921 |
+
|
15922 |
+
|
15923 |
+
|
15924 |
+
|
15925 |
+
|
15926 |
+
|
15927 |
+
|
15928 |
+
|
15929 |
+
|
15930 |
+
|
15931 |
+
|
15932 |
+
|
15933 |
+
|
15934 |
+
|
15935 |
+
|
15936 |
+
|
15937 |
+
|
15938 |
+
|
15939 |
+
|
15940 |
+
|
15941 |
+
|
15942 |
+
|
15943 |
+
|
15944 |
+
|
15945 |
+
|
15946 |
+
|
15947 |
+
|
15948 |
+
|
15949 |
+
|
15950 |
+
|
15951 |
+
|
15952 |
+
|
15953 |
+
|
15954 |
+
|
15955 |
+
|
15956 |
+
|
15957 |
+
|
15958 |
+
|
15959 |
+
|
15960 |
+
|
15961 |
+
|
15962 |
+
|
15963 |
+
|
15964 |
+
|
15965 |
+
|
15966 |
+
|
15967 |
+
|
15968 |
+
|
15969 |
+
|
15970 |
+
|
15971 |
+
|
15972 |
+
|
15973 |
+
|
15974 |
+
|
15975 |
+
|
15976 |
+
|
15977 |
+
|
15978 |
+
|
15979 |
+
|
15980 |
+
Training...: 5%|█████████▉ | 3250/61250 [26:04<7:48:48, 2.06it/s]
|
15981 |
+
|
15982 |
+
|
15983 |
+
|
15984 |
+
|
15985 |
+
|
15986 |
+
|
15987 |
+
|
15988 |
+
|
15989 |
+
|
15990 |
+
|
15991 |
+
|
15992 |
+
|
15993 |
+
|
15994 |
+
|
15995 |
+
|
15996 |
+
|
15997 |
+
|
15998 |
+
|
15999 |
+
|
16000 |
+
|
16001 |
+
|
16002 |
+
|
16003 |
+
|
16004 |
+
|
16005 |
+
|
16006 |
+
|
16007 |
+
|
16008 |
+
|
16009 |
+
|
16010 |
+
|
16011 |
+
|
16012 |
+
|
16013 |
+
|
16014 |
+
|
16015 |
+
|
16016 |
+
|
16017 |
+
|
16018 |
+
|
16019 |
+
|
16020 |
+
|
16021 |
+
|
16022 |
+
|
16023 |
+
|
16024 |
+
|
16025 |
+
|
16026 |
+
|
16027 |
+
|
16028 |
+
|
16029 |
+
|
16030 |
+
|
16031 |
+
|
16032 |
+
|
16033 |
+
|
16034 |
+
|
16035 |
+
|
16036 |
+
|
16037 |
+
|
16038 |
+
|
16039 |
+
|
16040 |
+
|
16041 |
+
|
16042 |
+
|
16043 |
+
|
16044 |
+
|
16045 |
+
|
16046 |
+
|
16047 |
+
|
16048 |
+
|
16049 |
+
|
16050 |
+
|
16051 |
+
|
16052 |
+
|
16053 |
+
|
16054 |
+
|
16055 |
+
|
16056 |
+
|
16057 |
+
|
16058 |
+
|
16059 |
+
|
16060 |
+
|
16061 |
+
|
16062 |
+
|
16063 |
+
|
16064 |
+
|
16065 |
+
|
16066 |
+
|
16067 |
+
|
16068 |
+
|
16069 |
+
|
16070 |
+
|
16071 |
+
|
16072 |
+
|
16073 |
+
|
16074 |
+
|
16075 |
+
|
16076 |
+
|
16077 |
+
|
16078 |
+
|
16079 |
+
|
16080 |
+
|
16081 |
+
|
16082 |
+
|
16083 |
+
|
16084 |
+
|
16085 |
+
|
16086 |
+
|
16087 |
+
|
16088 |
+
|
16089 |
+
|
16090 |
+
|
16091 |
+
|
16092 |
+
|
16093 |
+
|
16094 |
+
|
16095 |
+
Training...: 6%|███████████▌ | 3750/61250 [30:24<7:44:42, 2.06it/s]
|
16096 |
+
Evaluating ...: 0%|▌ | 2/625 [00:00<01:07, 9.25it/s]
|
16097 |
+
|
16098 |
+
|
16099 |
+
|
16100 |
+
|
16101 |
+
|
16102 |
+
|
16103 |
+
|
16104 |
+
|
16105 |
+
|
16106 |
+
|
16107 |
+
|
16108 |
+
|
16109 |
+
|
16110 |
+
|
16111 |
+
|
16112 |
+
|
16113 |
+
|
16114 |
+
|
16115 |
+
|
16116 |
+
|
16117 |
+
|
16118 |
+
|
16119 |
+
|
16120 |
+
|
16121 |
+
|
16122 |
+
|
16123 |
+
|
16124 |
+
|
16125 |
+
|
16126 |
+
|
16127 |
+
|
16128 |
+
|
16129 |
+
|
16130 |
+
|
16131 |
+
|
16132 |
+
|
16133 |
+
|
16134 |
+
|
16135 |
+
|
16136 |
+
|
16137 |
+
|
16138 |
+
|
16139 |
+
|
16140 |
+
|
16141 |
+
|
16142 |
+
|
16143 |
+
|
16144 |
+
|
16145 |
+
|
16146 |
+
|
16147 |
+
|
16148 |
+
|
16149 |
+
|
16150 |
+
|
16151 |
+
|
16152 |
+
|
16153 |
+
|
16154 |
+
|
16155 |
+
|
16156 |
+
|
16157 |
+
|
16158 |
+
|
16159 |
+
|
16160 |
+
|
16161 |
+
|
16162 |
+
|
16163 |
+
|
16164 |
+
|
16165 |
+
|
16166 |
+
|
16167 |
+
|
16168 |
+
|
16169 |
+
|
16170 |
+
|
16171 |
+
|
16172 |
+
|
16173 |
+
|
16174 |
+
|
16175 |
+
|
16176 |
+
|
16177 |
+
|
16178 |
+
|
16179 |
+
|
16180 |
+
|
16181 |
+
|
16182 |
+
|
16183 |
+
|
16184 |
+
|
16185 |
+
|
16186 |
+
|
16187 |
+
|
16188 |
+
|
16189 |
+
|
16190 |
+
|
16191 |
+
|
16192 |
+
|
16193 |
+
|
16194 |
+
|
16195 |
+
|
16196 |
+
|
16197 |
+
|
16198 |
+
|
16199 |
+
|
16200 |
+
|
16201 |
+
|
16202 |
+
|
16203 |
+
|
16204 |
+
|
16205 |
+
|
16206 |
+
|
16207 |
+
|
16208 |
+
|
16209 |
+
|
16210 |
+
|
16211 |
+
|
16212 |
+
|
16213 |
+
|
16214 |
+
|
16215 |
+
|
16216 |
+
|
16217 |
+
|
16218 |
+
|
16219 |
+
|
16220 |
+
|
16221 |
+
|
16222 |
+
|
16223 |
+
|
16224 |
+
|
16225 |
+
|
16226 |
+
|
16227 |
+
|
16228 |
+
|
16229 |
+
|
16230 |
+
|
16231 |
+
|
16232 |
+
|
16233 |
+
|
16234 |
+
|
16235 |
+
|
16236 |
+
|
16237 |
+
|
16238 |
+
|
16239 |
+
|
16240 |
+
|
16241 |
+
|
16242 |
+
|
16243 |
+
|
16244 |
+
|
16245 |
+
|
16246 |
+
|
16247 |
+
|
16248 |
+
|
16249 |
+
|
16250 |
+
|
16251 |
+
|
16252 |
+
|
16253 |
+
|
16254 |
+
|
16255 |
+
|
16256 |
+
|
16257 |
+
|
16258 |
+
|
16259 |
+
|
16260 |
+
|
16261 |
+
|
16262 |
+
|
16263 |
+
|
16264 |
+
|
16265 |
+
|
16266 |
+
|
16267 |
+
|
16268 |
+
|
16269 |
+
|
16270 |
+
|
16271 |
+
|
16272 |
+
|
16273 |
+
|
16274 |
+
|
16275 |
+
|
16276 |
+
|
16277 |
+
|
16278 |
+
|
16279 |
+
|
16280 |
+
|
16281 |
+
|
16282 |
+
|
16283 |
+
|
16284 |
+
|
16285 |
+
|
16286 |
+
|
16287 |
+
|
16288 |
+
|
16289 |
+
|
16290 |
+
|
16291 |
+
|
16292 |
+
|
16293 |
+
|
16294 |
+
|
16295 |
+
|
16296 |
+
|
16297 |
+
|
16298 |
+
|
16299 |
+
Training...: 7%|█████████████ | 4250/61250 [37:18<7:40:45, 2.06it/s]
|
16300 |
+
|
16301 |
+
|
16302 |
+
|
16303 |
+
|
16304 |
+
|
16305 |
+
|
16306 |
+
|
16307 |
+
|
16308 |
+
|
16309 |
+
|
16310 |
+
|
16311 |
+
|
16312 |
+
|
16313 |
+
|
16314 |
+
|
16315 |
+
|
16316 |
+
|
16317 |
+
|
16318 |
+
|
16319 |
+
|
16320 |
+
|
16321 |
+
|
16322 |
+
|
16323 |
+
|
16324 |
+
|
16325 |
+
|
16326 |
+
|
16327 |
+
|
16328 |
+
|
16329 |
+
|
16330 |
+
|
16331 |
+
|
16332 |
+
|
16333 |
+
|
16334 |
+
|
16335 |
+
|
16336 |
+
|
16337 |
+
|
16338 |
+
|
16339 |
+
|
16340 |
+
|
16341 |
+
|
16342 |
+
|
16343 |
+
|
16344 |
+
|
16345 |
+
|
16346 |
+
|
16347 |
+
|
16348 |
+
|
16349 |
+
|
16350 |
+
|
16351 |
+
|
16352 |
+
|
16353 |
+
|
16354 |
+
|
16355 |
+
|
16356 |
+
|
16357 |
+
|
16358 |
+
|
16359 |
+
|
16360 |
+
|
16361 |
+
|
16362 |
+
|
16363 |
+
|
16364 |
+
|
16365 |
+
|
16366 |
+
|
16367 |
+
|
16368 |
+
|
16369 |
+
|
16370 |
+
|
16371 |
+
|
16372 |
+
|
16373 |
+
|
16374 |
+
|
16375 |
+
|
16376 |
+
|
16377 |
+
|
16378 |
+
|
16379 |
+
|
16380 |
+
|
16381 |
+
|
16382 |
+
|
16383 |
+
|
16384 |
+
|
16385 |
+
|
16386 |
+
|
16387 |
+
|
16388 |
+
|
16389 |
+
|
16390 |
+
|
16391 |
+
|
16392 |
+
|
16393 |
+
|
16394 |
+
|
16395 |
+
|
16396 |
+
|
16397 |
+
|
16398 |
+
|
16399 |
+
|
16400 |
+
|
16401 |
+
|
16402 |
+
|
16403 |
+
|
16404 |
+
|
16405 |
+
|
16406 |
+
|
16407 |
+
|
16408 |
+
|
16409 |
+
|
16410 |
+
|
16411 |
+
|
16412 |
+
|
16413 |
+
|
16414 |
+
Training...: 8%|██████████████▌ | 4750/61250 [41:21<7:36:38, 2.06it/s]
|
16415 |
+
|
16416 |
+
|
16417 |
+
|
16418 |
+
|
16419 |
+
|
16420 |
+
|
16421 |
+
|
16422 |
+
|
16423 |
+
|
16424 |
+
|
16425 |
+
|
16426 |
+
|
16427 |
+
|
16428 |
+
|
16429 |
+
|
16430 |
+
|
16431 |
+
|
16432 |
+
|
16433 |
+
|
16434 |
+
|
16435 |
+
|
16436 |
+
|
16437 |
+
|
16438 |
+
|
16439 |
+
|
16440 |
+
|
16441 |
+
|
16442 |
+
|
16443 |
+
|
16444 |
+
|
16445 |
+
|
16446 |
+
|
16447 |
+
|
16448 |
+
|
16449 |
+
|
16450 |
+
|
16451 |
+
|
16452 |
+
|
16453 |
+
|
16454 |
+
|
16455 |
+
|
16456 |
+
|
16457 |
+
|
16458 |
+
|
16459 |
+
|
16460 |
+
|
16461 |
+
|
16462 |
+
|
16463 |
+
|
16464 |
+
|
16465 |
+
|
16466 |
+
|
16467 |
+
|
16468 |
+
|
16469 |
+
|
16470 |
+
|
16471 |
+
|
16472 |
+
|
16473 |
+
|
16474 |
+
|
16475 |
+
|
16476 |
+
|
16477 |
+
|
16478 |
+
|
16479 |
+
|
16480 |
+
|
16481 |
+
|
16482 |
+
|
16483 |
+
|
16484 |
+
|
16485 |
+
|
16486 |
+
|
16487 |
+
|
16488 |
+
|
16489 |
+
|
16490 |
+
|
16491 |
+
|
16492 |
+
|
16493 |
+
|
16494 |
+
|
16495 |
+
|
16496 |
+
|
16497 |
+
|
16498 |
+
|
16499 |
+
|
16500 |
+
|
16501 |
+
|
16502 |
+
|
16503 |
+
|
16504 |
+
|
16505 |
+
|
16506 |
+
|
16507 |
+
|
16508 |
+
|
16509 |
+
|
16510 |
+
|
16511 |
+
|
16512 |
+
|
16513 |
+
|
16514 |
+
|
16515 |
+
|
16516 |
+
|
16517 |
+
|
16518 |
+
|
16519 |
+
|
16520 |
+
|
16521 |
+
|
16522 |
+
|
16523 |
+
|
16524 |
+
|
16525 |
+
|
16526 |
+
|
16527 |
+
|
16528 |
+
|
16529 |
+
|
16530 |
+
Training...: 9%|████████████████ | 5257/61250 [45:41<17:51:54, 1.15s/it]
|
16531 |
+
|
16532 |
+
|
16533 |
+
|
16534 |
+
|
16535 |
+
|
16536 |
+
|
16537 |
+
|
16538 |
+
|
16539 |
+
|
16540 |
+
|
16541 |
+
|
16542 |
+
|
16543 |
+
|
16544 |
+
|
16545 |
+
|
16546 |
+
|
16547 |
+
|
16548 |
+
|
16549 |
+
|
16550 |
+
|
16551 |
+
|
16552 |
+
|
16553 |
+
|
16554 |
+
|
16555 |
+
|
16556 |
+
|
16557 |
+
|
16558 |
+
|
16559 |
+
|
16560 |
+
|
16561 |
+
|
16562 |
+
|
16563 |
+
|
16564 |
+
|
16565 |
+
|
16566 |
+
|
16567 |
+
|
16568 |
+
|
16569 |
+
|
16570 |
+
|
16571 |
+
|
16572 |
+
|
16573 |
+
|
16574 |
+
|
16575 |
+
|
16576 |
+
|
16577 |
+
|
16578 |
+
|
16579 |
+
|
16580 |
+
|
16581 |
+
|
16582 |
+
|
16583 |
+
|
16584 |
+
|
16585 |
+
|
16586 |
+
|
16587 |
+
|
16588 |
+
|
16589 |
+
|
16590 |
+
|
16591 |
+
|
16592 |
+
|
16593 |
+
|
16594 |
+
|
16595 |
+
|
16596 |
+
|
16597 |
+
|
16598 |
+
|
16599 |
+
|
16600 |
+
|
16601 |
+
|
16602 |
+
|
16603 |
+
|
16604 |
+
|
16605 |
+
|
16606 |
+
|
16607 |
+
|
16608 |
+
|
16609 |
+
|
16610 |
+
|
16611 |
+
|
16612 |
+
|
16613 |
+
|
16614 |
+
|
16615 |
+
|
16616 |
+
|
16617 |
+
|
16618 |
+
|
16619 |
+
|
16620 |
+
|
16621 |
+
|
16622 |
+
|
16623 |
+
|
16624 |
+
|
16625 |
+
|
16626 |
+
|
16627 |
+
|
16628 |
+
|
16629 |
+
|
16630 |
+
|
16631 |
+
|
16632 |
+
|
16633 |
+
|
16634 |
+
|
16635 |
+
|
16636 |
+
|
16637 |
+
|
16638 |
+
|
16639 |
+
|
16640 |
+
|
16641 |
+
|
16642 |
+
|
16643 |
+
|
16644 |
+
|
16645 |
+
Training...: 9%|█████████████████▌ | 5757/61250 [49:44<17:42:34, 1.15s/it]
|
16646 |
+
|
16647 |
+
|
16648 |
+
|
16649 |
+
|
16650 |
+
|
16651 |
+
|
16652 |
+
|
16653 |
+
|
16654 |
+
|
16655 |
+
|
16656 |
+
|
16657 |
+
|
16658 |
+
|
16659 |
+
|
16660 |
+
|
16661 |
+
|
16662 |
+
|
16663 |
+
|
16664 |
+
|
16665 |
+
|
16666 |
+
|
16667 |
+
|
16668 |
+
|
16669 |
+
|
16670 |
+
|
16671 |
+
|
16672 |
+
|
16673 |
+
|
16674 |
+
|
16675 |
+
|
16676 |
+
|
16677 |
+
|
16678 |
+
|
16679 |
+
|
16680 |
+
|
16681 |
+
|
16682 |
+
|
16683 |
+
|
16684 |
+
|
16685 |
+
|
16686 |
+
|
16687 |
+
|
16688 |
+
|
16689 |
+
|
16690 |
+
|
16691 |
+
|
16692 |
+
|
16693 |
+
|
16694 |
+
|
16695 |
+
|
16696 |
+
|
16697 |
+
|
16698 |
+
|
16699 |
+
|
16700 |
+
|
16701 |
+
|
16702 |
+
|
16703 |
+
|
16704 |
+
|
16705 |
+
|
16706 |
+
|
16707 |
+
|
16708 |
+
|
16709 |
+
|
16710 |
+
|
16711 |
+
|
16712 |
+
|
16713 |
+
|
16714 |
+
|
16715 |
+
|
16716 |
+
|
16717 |
+
|
16718 |
+
|
16719 |
+
|
16720 |
+
|
16721 |
+
|
16722 |
+
|
16723 |
+
|
16724 |
+
|
16725 |
+
|
16726 |
+
|
16727 |
+
|
16728 |
+
|
16729 |
+
|
16730 |
+
|
16731 |
+
|
16732 |
+
|
16733 |
+
|
16734 |
+
|
16735 |
+
|
16736 |
+
|
16737 |
+
|
16738 |
+
|
16739 |
+
|
16740 |
+
|
16741 |
+
|
16742 |
+
|
16743 |
+
|
16744 |
+
|
16745 |
+
|
16746 |
+
|
16747 |
+
|
16748 |
+
|
16749 |
+
|
16750 |
+
|
16751 |
+
|
16752 |
+
|
16753 |
+
|
16754 |
+
|
16755 |
+
|
16756 |
+
|
16757 |
+
|
16758 |
+
|
16759 |
+
|
16760 |
+
Training...: 10%|███████████████████ | 6255/61250 [53:47<26:57:17, 1.76s/it]
|
16761 |
+
|
16762 |
+
|
16763 |
+
|
16764 |
+
|
16765 |
+
|
16766 |
+
|
16767 |
+
|
16768 |
+
|
16769 |
+
|
16770 |
+
|
16771 |
+
|
16772 |
+
|
16773 |
+
|
16774 |
+
|
16775 |
+
|
16776 |
+
|
16777 |
+
|
16778 |
+
|
16779 |
+
|
16780 |
+
|
16781 |
+
|
16782 |
+
|
16783 |
+
|
16784 |
+
|
16785 |
+
|
16786 |
+
|
16787 |
+
|
16788 |
+
|
16789 |
+
|
16790 |
+
|
16791 |
+
|
16792 |
+
|
16793 |
+
|
16794 |
+
|
16795 |
+
|
16796 |
+
|
16797 |
+
|
16798 |
+
|
16799 |
+
|
16800 |
+
|
16801 |
+
|
16802 |
+
|
16803 |
+
|
16804 |
+
|
16805 |
+
|
16806 |
+
|
16807 |
+
|
16808 |
+
|
16809 |
+
|
16810 |
+
|
16811 |
+
|
16812 |
+
|
16813 |
+
|
16814 |
+
|
16815 |
+
|
16816 |
+
|
16817 |
+
|
16818 |
+
|
16819 |
+
|
16820 |
+
|
16821 |
+
|
16822 |
+
|
16823 |
+
|
16824 |
+
|
16825 |
+
|
16826 |
+
|
16827 |
+
|
16828 |
+
|
16829 |
+
|
16830 |
+
|
16831 |
+
|
16832 |
+
|
16833 |
+
|
16834 |
+
|
16835 |
+
|
16836 |
+
|
16837 |
+
|
16838 |
+
|
16839 |
+
|
16840 |
+
|
16841 |
+
|
16842 |
+
|
16843 |
+
|
16844 |
+
|
16845 |
+
|
16846 |
+
|
16847 |
+
|
16848 |
+
|
16849 |
+
|
16850 |
+
|
16851 |
+
|
16852 |
+
|
16853 |
+
|
16854 |
+
|
16855 |
+
|
16856 |
+
|
16857 |
+
|
16858 |
+
|
16859 |
+
|
16860 |
+
|
16861 |
+
|
16862 |
+
|
16863 |
+
|
16864 |
+
|
16865 |
+
|
16866 |
+
|
16867 |
+
|
16868 |
+
|
16869 |
+
|
16870 |
+
|
16871 |
+
|
16872 |
+
|
16873 |
+
|
16874 |
+
|
16875 |
+
Training...: 11%|████████████████████▌ | 6755/61250 [57:51<26:42:00, 1.76s/it]
|
16876 |
+
|
16877 |
+
|
16878 |
+
|
16879 |
+
|
16880 |
+
|
16881 |
+
|
16882 |
+
|
16883 |
+
|
16884 |
+
|
16885 |
+
|
16886 |
+
|
16887 |
+
|
16888 |
+
|
16889 |
+
|
16890 |
+
|
16891 |
+
|
16892 |
+
|
16893 |
+
|
16894 |
+
|
16895 |
+
|
16896 |
+
|
16897 |
+
|
16898 |
+
|
16899 |
+
|
16900 |
+
|
16901 |
+
|
16902 |
+
|
16903 |
+
|
16904 |
+
|
16905 |
+
|
16906 |
+
|
16907 |
+
|
16908 |
+
|
16909 |
+
|
16910 |
+
|
16911 |
+
|
16912 |
+
|
16913 |
+
|
16914 |
+
|
16915 |
+
|
16916 |
+
|
16917 |
+
|
16918 |
+
|
16919 |
+
|
16920 |
+
|
16921 |
+
|
16922 |
+
|
16923 |
+
|
16924 |
+
|
16925 |
+
|
16926 |
+
|
16927 |
+
|
16928 |
+
|
16929 |
+
|
16930 |
+
|
16931 |
+
|
16932 |
+
|
16933 |
+
|
16934 |
+
|
16935 |
+
|
16936 |
+
|
16937 |
+
|
16938 |
+
|
16939 |
+
|
16940 |
+
|
16941 |
+
|
16942 |
+
|
16943 |
+
|
16944 |
+
|
16945 |
+
|
16946 |
+
|
16947 |
+
|
16948 |
+
|
16949 |
+
|
16950 |
+
|
16951 |
+
|
16952 |
+
|
16953 |
+
|
16954 |
+
|
16955 |
+
|
16956 |
+
|
16957 |
+
|
16958 |
+
|
16959 |
+
|
16960 |
+
|
16961 |
+
|
16962 |
+
|
16963 |
+
|
16964 |
+
|
16965 |
+
|
16966 |
+
|
16967 |
+
|
16968 |
+
|
16969 |
+
|
16970 |
+
|
16971 |
+
|
16972 |
+
|
16973 |
+
|
16974 |
+
|
16975 |
+
|
16976 |
+
|
16977 |
+
|
16978 |
+
|
16979 |
+
|
16980 |
+
|
16981 |
+
|
16982 |
+
|
16983 |
+
|
16984 |
+
|
16985 |
+
|
16986 |
+
|
16987 |
+
|
16988 |
+
|
16989 |
+
|
16990 |
+
Training...: 12%|█████████████████████▉ | 7253/61250 [1:01:54<43:14:07, 2.88s/it]
|
16991 |
+
|
16992 |
+
|
16993 |
+
|
16994 |
+
|
16995 |
+
|
16996 |
+
|
16997 |
+
|
16998 |
+
|
16999 |
+
|
17000 |
+
|
17001 |
+
|
17002 |
+
|
17003 |
+
|
17004 |
+
|
17005 |
+
|
17006 |
+
|
17007 |
+
|
17008 |
+
|
17009 |
+
|
17010 |
+
|
17011 |
+
|
17012 |
+
|
17013 |
+
|
17014 |
+
|
17015 |
+
|
17016 |
+
|
17017 |
+
|
17018 |
+
|
17019 |
+
|
17020 |
+
|
17021 |
+
|
17022 |
+
|
17023 |
+
|
17024 |
+
|
17025 |
+
|
17026 |
+
|
17027 |
+
|
17028 |
+
|
17029 |
+
|
17030 |
+
|
17031 |
+
|
17032 |
+
|
17033 |
+
|
17034 |
+
|
17035 |
+
|
17036 |
+
|
17037 |
+
|
17038 |
+
|
17039 |
+
|
17040 |
+
|
17041 |
+
|
17042 |
+
|
17043 |
+
|
17044 |
+
|
17045 |
+
|
17046 |
+
|
17047 |
+
|
17048 |
+
|
17049 |
+
|
17050 |
+
|
17051 |
+
|
17052 |
+
|
17053 |
+
|
17054 |
+
|
17055 |
+
|
17056 |
+
|
17057 |
+
|
17058 |
+
|
17059 |
+
|
17060 |
+
|
17061 |
+
|
17062 |
+
|
17063 |
+
|
17064 |
+
|
17065 |
+
|
17066 |
+
|
17067 |
+
|
17068 |
+
|
17069 |
+
|
17070 |
+
|
17071 |
+
|
17072 |
+
|
17073 |
+
|
17074 |
+
|
17075 |
+
|
17076 |
+
|
17077 |
+
|
17078 |
+
|
17079 |
+
|
17080 |
+
|
17081 |
+
|
17082 |
+
|
17083 |
+
|
17084 |
+
|
17085 |
+
|
17086 |
+
|
17087 |
+
|
17088 |
+
|
17089 |
+
|
17090 |
+
|
17091 |
+
|
17092 |
+
|
17093 |
+
|
17094 |
+
|
17095 |
+
|
17096 |
+
|
17097 |
+
|
17098 |
+
|
17099 |
+
|
17100 |
+
|
17101 |
+
|
17102 |
+
|
17103 |
+
|
17104 |
+
|
17105 |
+
Training...: 13%|███████████████████████▍ | 7753/61250 [1:05:57<42:53:26, 2.89s/it]
|
17106 |
+
|
17107 |
+
|
17108 |
+
|
17109 |
+
|
17110 |
+
|
17111 |
+
|
17112 |
+
|
17113 |
+
|
17114 |
+
|
17115 |
+
|
17116 |
+
|
17117 |
+
|
17118 |
+
|
17119 |
+
|
17120 |
+
|
17121 |
+
|
17122 |
+
|
17123 |
+
|
17124 |
+
|
17125 |
+
|
17126 |
+
|
17127 |
+
|
17128 |
+
|
17129 |
+
|
17130 |
+
|
17131 |
+
|
17132 |
+
|
17133 |
+
|
17134 |
+
|
17135 |
+
|
17136 |
+
|
17137 |
+
|
17138 |
+
|
17139 |
+
|
17140 |
+
|
17141 |
+
|
17142 |
+
|
17143 |
+
|
17144 |
+
|
17145 |
+
|
17146 |
+
|
17147 |
+
|
17148 |
+
|
17149 |
+
|
17150 |
+
|
17151 |
+
|
17152 |
+
|
17153 |
+
|
17154 |
+
|
17155 |
+
|
17156 |
+
|
17157 |
+
|
17158 |
+
|
17159 |
+
|
17160 |
+
|
17161 |
+
|
17162 |
+
|
17163 |
+
|
17164 |
+
|
17165 |
+
|
17166 |
+
|
17167 |
+
|
17168 |
+
|
17169 |
+
|
17170 |
+
|
17171 |
+
|
17172 |
+
|
17173 |
+
|
17174 |
+
|
17175 |
+
|
17176 |
+
|
17177 |
+
|
17178 |
+
|
17179 |
+
|
17180 |
+
|
17181 |
+
|
17182 |
+
|
17183 |
+
|
17184 |
+
|
17185 |
+
|
17186 |
+
|
17187 |
+
|
17188 |
+
|
17189 |
+
|
17190 |
+
|
17191 |
+
|
17192 |
+
|
17193 |
+
|
17194 |
+
|
17195 |
+
|
17196 |
+
|
17197 |
+
|
17198 |
+
|
17199 |
+
|
17200 |
+
|
17201 |
+
|
17202 |
+
|
17203 |
+
|
17204 |
+
|
17205 |
+
|
17206 |
+
|
17207 |
+
|
17208 |
+
|
17209 |
+
|
17210 |
+
|
17211 |
+
|
17212 |
+
|
17213 |
+
|
17214 |
+
|
17215 |
+
|
17216 |
+
|
17217 |
+
|
17218 |
+
|
17219 |
+
|
17220 |
+
Training...: 13%|████████████████████████▉ | 8253/61250 [1:10:00<42:28:41, 2.89s/it]
|
17221 |
+
|
17222 |
+
|
17223 |
+
|
17224 |
+
|
17225 |
+
|
17226 |
+
|
17227 |
+
|
17228 |
+
|
17229 |
+
|
17230 |
+
|
17231 |
+
|
17232 |
+
|
17233 |
+
|
17234 |
+
|
17235 |
+
|
17236 |
+
|
17237 |
+
|
17238 |
+
|
17239 |
+
|
17240 |
+
|
17241 |
+
|
17242 |
+
|
17243 |
+
|
17244 |
+
|
17245 |
+
|
17246 |
+
|
17247 |
+
|
17248 |
+
|
17249 |
+
|
17250 |
+
|
17251 |
+
|
17252 |
+
|
17253 |
+
|
17254 |
+
|
17255 |
+
|
17256 |
+
|
17257 |
+
|
17258 |
+
|
17259 |
+
|
17260 |
+
|
17261 |
+
|
17262 |
+
|
17263 |
+
|
17264 |
+
|
17265 |
+
|
17266 |
+
|
17267 |
+
|
17268 |
+
|
17269 |
+
|
17270 |
+
|
17271 |
+
|
17272 |
+
|
17273 |
+
|
17274 |
+
|
17275 |
+
|
17276 |
+
|
17277 |
+
|
17278 |
+
|
17279 |
+
|
17280 |
+
|
17281 |
+
|
17282 |
+
|
17283 |
+
|
17284 |
+
|
17285 |
+
|
17286 |
+
|
17287 |
+
|
17288 |
+
|
17289 |
+
|
17290 |
+
|
17291 |
+
|
17292 |
+
|
17293 |
+
|
17294 |
+
|
17295 |
+
|
17296 |
+
|
17297 |
+
|
17298 |
+
|
17299 |
+
|
17300 |
+
|
17301 |
+
|
17302 |
+
|
17303 |
+
|
17304 |
+
|
17305 |
+
|
17306 |
+
|
17307 |
+
|
17308 |
+
|
17309 |
+
|
17310 |
+
|
17311 |
+
|
17312 |
+
|
17313 |
+
|
17314 |
+
|
17315 |
+
|
17316 |
+
|
17317 |
+
|
17318 |
+
|
17319 |
+
|
17320 |
+
|
17321 |
+
|
17322 |
+
|
17323 |
+
|
17324 |
+
|
17325 |
+
|
17326 |
+
|
17327 |
+
|
17328 |
+
|
17329 |
+
|
17330 |
+
|
17331 |
+
|
17332 |
+
|
17333 |
+
|
17334 |
+
Training...: 14%|██████████████████████████▌ | 8750/61250 [1:13:46<7:04:26, 2.06it/s]
|
17335 |
+
Training...: 14%|██████████████████████████▌ | 8750/61250 [1:14:03<7:04:26, 2.06it/s]
|
17336 |
+
|
17337 |
+
|
17338 |
+
|
17339 |
+
|
17340 |
+
|
17341 |
+
|
17342 |
+
|
17343 |
+
|
17344 |
+
|
17345 |
+
|
17346 |
+
|
17347 |
+
|
17348 |
+
|
17349 |
+
|
17350 |
+
|
17351 |
+
|
17352 |
+
|
17353 |
+
|
17354 |
+
|
17355 |
+
|
17356 |
+
|
17357 |
+
|
17358 |
+
|
17359 |
+
|
17360 |
+
|
17361 |
+
|
17362 |
+
|
17363 |
+
|
17364 |
+
|
17365 |
+
|
17366 |
+
|
17367 |
+
|
17368 |
+
|
17369 |
+
|
17370 |
+
|
17371 |
+
|
17372 |
+
|
17373 |
+
|
17374 |
+
|
17375 |
+
|
17376 |
+
|
17377 |
+
|
17378 |
+
|
17379 |
+
|
17380 |
+
|
17381 |
+
|
17382 |
+
|
17383 |
+
|
17384 |
+
|
17385 |
+
|
17386 |
+
|
17387 |
+
|
17388 |
+
|
17389 |
+
|
17390 |
+
|
17391 |
+
|
17392 |
+
|
17393 |
+
|
17394 |
+
|
17395 |
+
|
17396 |
+
|
17397 |
+
|
17398 |
+
|
17399 |
+
|
17400 |
+
|
17401 |
+
|
17402 |
+
|
17403 |
+
|
17404 |
+
|
17405 |
+
|
17406 |
+
|
17407 |
+
|
17408 |
+
|
17409 |
+
|
17410 |
+
|
17411 |
+
|
17412 |
+
|
17413 |
+
|
17414 |
+
|
17415 |
+
|
17416 |
+
|
17417 |
+
|
17418 |
+
|
17419 |
+
|
17420 |
+
|
17421 |
+
|
17422 |
+
|
17423 |
+
|
17424 |
+
|
17425 |
+
|
17426 |
+
|
17427 |
+
|
17428 |
+
|
17429 |
+
|
17430 |
+
|
17431 |
+
|
17432 |
+
|
17433 |
+
|
17434 |
+
|
17435 |
+
|
17436 |
+
|
17437 |
+
|
17438 |
+
|
17439 |
+
|
17440 |
+
|
17441 |
+
|
17442 |
+
|
17443 |
+
|
17444 |
+
|
17445 |
+
|
17446 |
+
|
17447 |
+
|
17448 |
+
|
17449 |
+
|
17450 |
+
|
17451 |
+
|
17452 |
+
|
17453 |
+
|
17454 |
+
|
17455 |
+
|
17456 |
+
|
17457 |
+
|
17458 |
+
|
17459 |
+
|
17460 |
+
|
17461 |
+
|
17462 |
+
|
17463 |
+
|
17464 |
+
|
17465 |
+
|
17466 |
+
|
17467 |
+
|
17468 |
+
|
17469 |
+
|
17470 |
+
|
17471 |
+
|
17472 |
+
|
17473 |
+
|
17474 |
+
|
17475 |
+
|
17476 |
+
|
17477 |
+
|
17478 |
+
|
17479 |
+
|
17480 |
+
|
17481 |
+
|
17482 |
+
|
17483 |
+
|
17484 |
+
|
17485 |
+
|
17486 |
+
|
17487 |
+
|
17488 |
+
|
17489 |
+
|
17490 |
+
|
17491 |
+
|
17492 |
+
|
17493 |
+
|
17494 |
+
|
17495 |
+
|
17496 |
+
|
17497 |
+
|
17498 |
+
|
17499 |
+
|
17500 |
+
|
17501 |
+
|
17502 |
+
|
17503 |
+
|
17504 |
+
|
17505 |
+
|
17506 |
+
|
17507 |
+
|
17508 |
+
|
17509 |
+
|
17510 |
+
|
17511 |
+
|
17512 |
+
|
17513 |
+
|
17514 |
+
|
17515 |
+
|
17516 |
+
|
17517 |
+
|
17518 |
+
|
17519 |
+
|
17520 |
+
|
17521 |
+
|
17522 |
+
|
17523 |
+
|
17524 |
+
|
17525 |
+
|
17526 |
+
|
17527 |
+
|
17528 |
+
|
17529 |
+
|
17530 |
+
|
17531 |
+
|
17532 |
+
|
17533 |
+
|
17534 |
+
|
17535 |
+
|
17536 |
+
|
17537 |
+
Training...: 15%|████████████████████████████ | 9250/61250 [1:20:57<7:00:16, 2.06it/s]
|
17538 |
+
|
17539 |
+
|
17540 |
+
|
17541 |
+
|
17542 |
+
|
17543 |
+
|
17544 |
+
|
17545 |
+
|
17546 |
+
|
17547 |
+
|
17548 |
+
|
17549 |
+
|
17550 |
+
|
17551 |
+
|
17552 |
+
|
17553 |
+
|
17554 |
+
|
17555 |
+
|
17556 |
+
|
17557 |
+
|
17558 |
+
|
17559 |
+
|
17560 |
+
|
17561 |
+
|
17562 |
+
|
17563 |
+
|
17564 |
+
|
17565 |
+
|
17566 |
+
|
17567 |
+
|
17568 |
+
|
17569 |
+
|
17570 |
+
|
17571 |
+
|
17572 |
+
|
17573 |
+
|
17574 |
+
|
17575 |
+
|
17576 |
+
|
17577 |
+
|
17578 |
+
|
17579 |
+
|
17580 |
+
|
17581 |
+
|
17582 |
+
|
17583 |
+
|
17584 |
+
|
17585 |
+
|
17586 |
+
|
17587 |
+
|
17588 |
+
|
17589 |
+
|
17590 |
+
|
17591 |
+
|
17592 |
+
|
17593 |
+
|
17594 |
+
|
17595 |
+
|
17596 |
+
|
17597 |
+
|
17598 |
+
|
17599 |
+
|
17600 |
+
|
17601 |
+
|
17602 |
+
|
17603 |
+
|
17604 |
+
|
17605 |
+
|
17606 |
+
|
17607 |
+
|
17608 |
+
|
17609 |
+
|
17610 |
+
|
17611 |
+
|
17612 |
+
|
17613 |
+
|
17614 |
+
|
17615 |
+
|
17616 |
+
|
17617 |
+
|
17618 |
+
|
17619 |
+
|
17620 |
+
|
17621 |
+
|
17622 |
+
|
17623 |
+
|
17624 |
+
|
17625 |
+
|
17626 |
+
|
17627 |
+
|
17628 |
+
|
17629 |
+
|
17630 |
+
|
17631 |
+
|
17632 |
+
|
17633 |
+
|
17634 |
+
|
17635 |
+
|
17636 |
+
|
17637 |
+
|
17638 |
+
|
17639 |
+
|
17640 |
+
|
17641 |
+
|
17642 |
+
|
17643 |
+
|
17644 |
+
|
17645 |
+
|
17646 |
+
|
17647 |
+
|
17648 |
+
|
17649 |
+
|
17650 |
+
|
17651 |
+
Training...: 16%|█████████████████████████████▌ | 9750/61250 [1:25:00<6:56:21, 2.06it/s]
|
17652 |
+
|
17653 |
+
|
17654 |
+
|
17655 |
+
|
17656 |
+
|
17657 |
+
|
17658 |
+
|
17659 |
+
|
17660 |
+
|
17661 |
+
|
17662 |
+
|
17663 |
+
|
17664 |
+
|
17665 |
+
|
17666 |
+
|
17667 |
+
|
17668 |
+
|
17669 |
+
|
17670 |
+
|
17671 |
+
|
17672 |
+
|
17673 |
+
|
17674 |
+
|
17675 |
+
|
17676 |
+
|
17677 |
+
|
17678 |
+
|
17679 |
+
|
17680 |
+
|
17681 |
+
|
17682 |
+
|
17683 |
+
|
17684 |
+
|
17685 |
+
|
17686 |
+
|
17687 |
+
|
17688 |
+
|
17689 |
+
|
17690 |
+
|
17691 |
+
|
17692 |
+
|
17693 |
+
|
17694 |
+
|
17695 |
+
|
17696 |
+
|
17697 |
+
|
17698 |
+
|
17699 |
+
|
17700 |
+
|
17701 |
+
|
17702 |
+
|
17703 |
+
|
17704 |
+
|
17705 |
+
|
17706 |
+
|
17707 |
+
|
17708 |
+
|
17709 |
+
|
17710 |
+
|
17711 |
+
|
17712 |
+
|
17713 |
+
|
17714 |
+
|
17715 |
+
|
17716 |
+
|
17717 |
+
|
17718 |
+
|
17719 |
+
|
17720 |
+
|
17721 |
+
|
17722 |
+
|
17723 |
+
|
17724 |
+
|
17725 |
+
|
17726 |
+
|
17727 |
+
|
17728 |
+
|
17729 |
+
|
17730 |
+
|
17731 |
+
|
17732 |
+
|
17733 |
+
|
17734 |
+
|
17735 |
+
|
17736 |
+
|
17737 |
+
|
17738 |
+
|
17739 |
+
|
17740 |
+
|
17741 |
+
|
17742 |
+
|
17743 |
+
|
17744 |
+
|
17745 |
+
|
17746 |
+
|
17747 |
+
|
17748 |
+
|
17749 |
+
|
17750 |
+
|
17751 |
+
|
17752 |
+
|
17753 |
+
|
17754 |
+
|
17755 |
+
|
17756 |
+
|
17757 |
+
|
17758 |
+
|
17759 |
+
|
17760 |
+
|
17761 |
+
|
17762 |
+
|
17763 |
+
|
17764 |
+
|
17765 |
+
|
17766 |
+
Training...: 17%|██████████████████████████████▉ | 10250/61250 [1:29:04<6:52:18, 2.06it/s]
|
17767 |
+
|
17768 |
+
|
17769 |
+
|
17770 |
+
|
17771 |
+
|
17772 |
+
|
17773 |
+
|
17774 |
+
|
17775 |
+
|
17776 |
+
|
17777 |
+
|
17778 |
+
|
17779 |
+
|
17780 |
+
|
17781 |
+
|
17782 |
+
|
17783 |
+
|
17784 |
+
|
17785 |
+
|
17786 |
+
|
17787 |
+
|
17788 |
+
|
17789 |
+
|
17790 |
+
|
17791 |
+
|
17792 |
+
|
17793 |
+
|
17794 |
+
|
17795 |
+
|
17796 |
+
|
17797 |
+
|
17798 |
+
|
17799 |
+
|
17800 |
+
|
17801 |
+
|
17802 |
+
|
17803 |
+
|
17804 |
+
|
17805 |
+
|
17806 |
+
|
17807 |
+
|
17808 |
+
|
17809 |
+
|
17810 |
+
|
17811 |
+
|
17812 |
+
|
17813 |
+
|
17814 |
+
|
17815 |
+
|
17816 |
+
|
17817 |
+
|
17818 |
+
|
17819 |
+
|
17820 |
+
|
17821 |
+
|
17822 |
+
|
17823 |
+
|
17824 |
+
|
17825 |
+
|
17826 |
+
|
17827 |
+
|
17828 |
+
|
17829 |
+
|
17830 |
+
|
17831 |
+
|
17832 |
+
|
17833 |
+
|
17834 |
+
|
17835 |
+
|
17836 |
+
|
17837 |
+
|
17838 |
+
|
17839 |
+
|
17840 |
+
|
17841 |
+
|
17842 |
+
|
17843 |
+
|
17844 |
+
|
17845 |
+
|
17846 |
+
|
17847 |
+
|
17848 |
+
|
17849 |
+
|
17850 |
+
|
17851 |
+
|
17852 |
+
|
17853 |
+
|
17854 |
+
|
17855 |
+
|
17856 |
+
|
17857 |
+
|
17858 |
+
|
17859 |
+
|
17860 |
+
|
17861 |
+
|
17862 |
+
|
17863 |
+
|
17864 |
+
|
17865 |
+
|
17866 |
+
|
17867 |
+
|
17868 |
+
|
17869 |
+
|
17870 |
+
|
17871 |
+
|
17872 |
+
|
17873 |
+
|
17874 |
+
|
17875 |
+
|
17876 |
+
|
17877 |
+
|
17878 |
+
|
17879 |
+
|
17880 |
+
Step... (70000 | Loss: 1.39120352268219, Acc: 0.7222487330436707): 33%|████████████████████████████████████████████▋ | 1/3 [10:38:19<21:16:38, 38299.27s/it]
|
17881 |
+
Traceback (most recent call last):
|
17882 |
+
File "run_muril_bb_flax_mlm.py", line 709, in <module>
|
17883 |
+
write_train_metric(summary_writer, train_metrics, train_time, cur_step)
|
17884 |
+
File "run_muril_bb_flax_mlm.py", line 269, in write_train_metric
|
17885 |
+
train_metrics = get_metrics(train_metrics)
|
17886 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/training/common_utils.py", line 52, in get_metrics
|
17887 |
+
metrics_np = jax.device_get(device_metrics)
|
17888 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/api.py", line 2682, in device_get
|
17889 |
+
return tree_map(_device_get, x)
|
17890 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/tree_util.py", line 178, in tree_map
|
17891 |
+
return treedef.unflatten(f(*xs) for xs in zip(*all_leaves))
|
17892 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/tree_util.py", line 178, in <genexpr>
|
17893 |
+
return treedef.unflatten(f(*xs) for xs in zip(*all_leaves))
|
17894 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/api.py", line 2646, in _device_get
|
17895 |
+
return copy()
|
17896 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 1428, in copy
|
17897 |
+
return np.asarray(self)
|
17898 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 1496, in __array__
|
17899 |
+
return np.asarray(self._value, dtype=dtype)
|
17900 |
+
KeyboardInterrupt
|
wandb/run-20211107_204808-2ns3lo5n/files/wandb-summary.json
CHANGED
@@ -1 +1 @@
|
|
1 |
-
{"training_step":
|
|
|
1 |
+
{"training_step": 71500, "learning_rate": 3.054421904380433e-05, "train_loss": 1.3594125509262085, "_runtime": 38088, "_timestamp": 1636356176, "_step": 158, "eval_step": 70000, "eval_accuracy": 0.7222487330436707, "eval_loss": 1.39120352268219, "_wandb": {"runtime": 38320}}
|
wandb/run-20211107_204808-2ns3lo5n/logs/debug-internal.log
CHANGED
The diff for this file is too large to render.
See raw diff
|
|
wandb/run-20211107_204808-2ns3lo5n/logs/debug.log
CHANGED
@@ -23,3 +23,125 @@ config: {}
|
|
23 |
2021-11-07 20:48:11,628 INFO MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov07_20-47-12_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 50000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te-from-50k', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
|
24 |
2021-11-07 20:48:11,630 INFO MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
|
25 |
2021-11-07 20:48:11,631 INFO MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
23 |
2021-11-07 20:48:11,628 INFO MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov07_20-47-12_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 50000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te-from-50k', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
|
24 |
2021-11-07 20:48:11,630 INFO MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
|
25 |
2021-11-07 20:48:11,631 INFO MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}
|
26 |
+
2021-11-08 07:26:47,472 INFO MainThread:20231 [wandb_run.py:_atexit_cleanup():1663] got exitcode: 255
|
27 |
+
2021-11-08 07:26:47,472 INFO MainThread:20231 [wandb_run.py:_restore():1635] restore
|
28 |
+
2021-11-08 07:26:50,100 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
29 |
+
wandb_count: 1
|
30 |
+
}
|
31 |
+
pusher_stats {
|
32 |
+
uploaded_bytes: 1758
|
33 |
+
total_bytes: 1758
|
34 |
+
}
|
35 |
+
|
36 |
+
2021-11-08 07:26:50,366 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
37 |
+
wandb_count: 1
|
38 |
+
}
|
39 |
+
pusher_stats {
|
40 |
+
uploaded_bytes: 1758
|
41 |
+
total_bytes: 1758
|
42 |
+
}
|
43 |
+
|
44 |
+
2021-11-08 07:26:50,710 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
45 |
+
wandb_count: 5
|
46 |
+
}
|
47 |
+
pusher_stats {
|
48 |
+
uploaded_bytes: 1758
|
49 |
+
total_bytes: 95883
|
50 |
+
}
|
51 |
+
|
52 |
+
2021-11-08 07:26:50,812 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
53 |
+
wandb_count: 5
|
54 |
+
}
|
55 |
+
pusher_stats {
|
56 |
+
uploaded_bytes: 1758
|
57 |
+
total_bytes: 95883
|
58 |
+
}
|
59 |
+
|
60 |
+
2021-11-08 07:26:50,914 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
61 |
+
wandb_count: 5
|
62 |
+
}
|
63 |
+
pusher_stats {
|
64 |
+
uploaded_bytes: 95883
|
65 |
+
total_bytes: 95883
|
66 |
+
}
|
67 |
+
|
68 |
+
2021-11-08 07:26:51,016 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
69 |
+
wandb_count: 5
|
70 |
+
}
|
71 |
+
pusher_stats {
|
72 |
+
uploaded_bytes: 95883
|
73 |
+
total_bytes: 95883
|
74 |
+
}
|
75 |
+
|
76 |
+
2021-11-08 07:26:51,117 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
77 |
+
wandb_count: 5
|
78 |
+
}
|
79 |
+
pusher_stats {
|
80 |
+
uploaded_bytes: 95883
|
81 |
+
total_bytes: 95883
|
82 |
+
}
|
83 |
+
|
84 |
+
2021-11-08 07:26:51,219 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
85 |
+
wandb_count: 5
|
86 |
+
}
|
87 |
+
pusher_stats {
|
88 |
+
uploaded_bytes: 95883
|
89 |
+
total_bytes: 95883
|
90 |
+
}
|
91 |
+
|
92 |
+
2021-11-08 07:26:51,321 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
93 |
+
wandb_count: 5
|
94 |
+
}
|
95 |
+
pusher_stats {
|
96 |
+
uploaded_bytes: 95883
|
97 |
+
total_bytes: 95883
|
98 |
+
}
|
99 |
+
|
100 |
+
2021-11-08 07:26:51,423 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
101 |
+
wandb_count: 5
|
102 |
+
}
|
103 |
+
pusher_stats {
|
104 |
+
uploaded_bytes: 95883
|
105 |
+
total_bytes: 95883
|
106 |
+
}
|
107 |
+
|
108 |
+
2021-11-08 07:26:51,525 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
109 |
+
wandb_count: 5
|
110 |
+
}
|
111 |
+
pusher_stats {
|
112 |
+
uploaded_bytes: 95883
|
113 |
+
total_bytes: 95883
|
114 |
+
}
|
115 |
+
|
116 |
+
2021-11-08 07:26:51,626 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
117 |
+
wandb_count: 5
|
118 |
+
}
|
119 |
+
pusher_stats {
|
120 |
+
uploaded_bytes: 95883
|
121 |
+
total_bytes: 95883
|
122 |
+
}
|
123 |
+
|
124 |
+
2021-11-08 07:26:51,946 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
125 |
+
wandb_count: 5
|
126 |
+
}
|
127 |
+
pusher_stats {
|
128 |
+
uploaded_bytes: 95883
|
129 |
+
total_bytes: 95883
|
130 |
+
}
|
131 |
+
|
132 |
+
2021-11-08 07:26:52,296 INFO MainThread:20231 [wandb_run.py:_wait_for_finish():1793] got exit ret: done: true
|
133 |
+
exit_result {
|
134 |
+
}
|
135 |
+
file_counts {
|
136 |
+
wandb_count: 5
|
137 |
+
}
|
138 |
+
pusher_stats {
|
139 |
+
uploaded_bytes: 95883
|
140 |
+
total_bytes: 95883
|
141 |
+
}
|
142 |
+
local_info {
|
143 |
+
}
|
144 |
+
|
145 |
+
2021-11-08 07:26:53,747 INFO MainThread:20231 [wandb_run.py:_append_history():2011] rendering history
|
146 |
+
2021-11-08 07:26:53,748 INFO MainThread:20231 [wandb_run.py:_append_summary():1966] rendering summary
|
147 |
+
2021-11-08 07:26:53,748 INFO MainThread:20231 [wandb_run.py:_append_files():2061] logging synced files
|
wandb/run-20211107_204808-2ns3lo5n/run-2ns3lo5n.wandb
CHANGED
Binary files a/wandb/run-20211107_204808-2ns3lo5n/run-2ns3lo5n.wandb and b/wandb/run-20211107_204808-2ns3lo5n/run-2ns3lo5n.wandb differ
|
|
wandb/run-20211108_073656-hyqam6zi/files/config.yaml
ADDED
@@ -0,0 +1,347 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
wandb_version: 1
|
2 |
+
|
3 |
+
__cached__setup_devices:
|
4 |
+
desc: null
|
5 |
+
value: cpu
|
6 |
+
_n_gpu:
|
7 |
+
desc: null
|
8 |
+
value: 0
|
9 |
+
_wandb:
|
10 |
+
desc: null
|
11 |
+
value:
|
12 |
+
cli_version: 0.12.6
|
13 |
+
framework: huggingface
|
14 |
+
huggingface_version: 4.13.0.dev0
|
15 |
+
is_jupyter_run: false
|
16 |
+
is_kaggle_kernel: true
|
17 |
+
python_version: 3.8.10
|
18 |
+
start_time: 1636357016
|
19 |
+
t:
|
20 |
+
1:
|
21 |
+
- 1
|
22 |
+
- 2
|
23 |
+
- 3
|
24 |
+
- 11
|
25 |
+
- 12
|
26 |
+
2:
|
27 |
+
- 1
|
28 |
+
- 2
|
29 |
+
- 3
|
30 |
+
- 11
|
31 |
+
- 12
|
32 |
+
3:
|
33 |
+
- 13
|
34 |
+
4: 3.8.10
|
35 |
+
5: 0.12.6
|
36 |
+
6: 4.13.0.dev0
|
37 |
+
8:
|
38 |
+
- 2
|
39 |
+
- 5
|
40 |
+
adafactor:
|
41 |
+
desc: null
|
42 |
+
value: false
|
43 |
+
adam_beta1:
|
44 |
+
desc: null
|
45 |
+
value: 0.9
|
46 |
+
adam_beta2:
|
47 |
+
desc: null
|
48 |
+
value: 0.999
|
49 |
+
adam_epsilon:
|
50 |
+
desc: null
|
51 |
+
value: 1.0e-08
|
52 |
+
cache_dir:
|
53 |
+
desc: null
|
54 |
+
value: null
|
55 |
+
config_name:
|
56 |
+
desc: null
|
57 |
+
value: null
|
58 |
+
dataloader_drop_last:
|
59 |
+
desc: null
|
60 |
+
value: false
|
61 |
+
dataloader_num_workers:
|
62 |
+
desc: null
|
63 |
+
value: 0
|
64 |
+
dataloader_pin_memory:
|
65 |
+
desc: null
|
66 |
+
value: true
|
67 |
+
dataset_config_name:
|
68 |
+
desc: null
|
69 |
+
value: null
|
70 |
+
dataset_name:
|
71 |
+
desc: null
|
72 |
+
value: null
|
73 |
+
ddp_find_unused_parameters:
|
74 |
+
desc: null
|
75 |
+
value: null
|
76 |
+
debug:
|
77 |
+
desc: null
|
78 |
+
value: []
|
79 |
+
deepspeed:
|
80 |
+
desc: null
|
81 |
+
value: null
|
82 |
+
disable_tqdm:
|
83 |
+
desc: null
|
84 |
+
value: false
|
85 |
+
do_eval:
|
86 |
+
desc: null
|
87 |
+
value: true
|
88 |
+
do_predict:
|
89 |
+
desc: null
|
90 |
+
value: false
|
91 |
+
do_train:
|
92 |
+
desc: null
|
93 |
+
value: true
|
94 |
+
dtype:
|
95 |
+
desc: null
|
96 |
+
value: bfloat16
|
97 |
+
eval_accumulation_steps:
|
98 |
+
desc: null
|
99 |
+
value: null
|
100 |
+
eval_steps:
|
101 |
+
desc: null
|
102 |
+
value: 5000
|
103 |
+
evaluation_strategy:
|
104 |
+
desc: null
|
105 |
+
value: IntervalStrategy.NO
|
106 |
+
fp16:
|
107 |
+
desc: null
|
108 |
+
value: false
|
109 |
+
fp16_backend:
|
110 |
+
desc: null
|
111 |
+
value: auto
|
112 |
+
fp16_full_eval:
|
113 |
+
desc: null
|
114 |
+
value: false
|
115 |
+
fp16_opt_level:
|
116 |
+
desc: null
|
117 |
+
value: O1
|
118 |
+
gradient_accumulation_steps:
|
119 |
+
desc: null
|
120 |
+
value: 1
|
121 |
+
gradient_checkpointing:
|
122 |
+
desc: null
|
123 |
+
value: false
|
124 |
+
greater_is_better:
|
125 |
+
desc: null
|
126 |
+
value: null
|
127 |
+
group_by_length:
|
128 |
+
desc: null
|
129 |
+
value: false
|
130 |
+
hub_model_id:
|
131 |
+
desc: null
|
132 |
+
value: null
|
133 |
+
hub_strategy:
|
134 |
+
desc: null
|
135 |
+
value: HubStrategy.EVERY_SAVE
|
136 |
+
hub_token:
|
137 |
+
desc: null
|
138 |
+
value: null
|
139 |
+
ignore_data_skip:
|
140 |
+
desc: null
|
141 |
+
value: false
|
142 |
+
label_names:
|
143 |
+
desc: null
|
144 |
+
value: null
|
145 |
+
label_smoothing_factor:
|
146 |
+
desc: null
|
147 |
+
value: 0.0
|
148 |
+
learning_rate:
|
149 |
+
desc: null
|
150 |
+
value: 5.0e-05
|
151 |
+
length_column_name:
|
152 |
+
desc: null
|
153 |
+
value: length
|
154 |
+
line_by_line:
|
155 |
+
desc: null
|
156 |
+
value: false
|
157 |
+
load_best_model_at_end:
|
158 |
+
desc: null
|
159 |
+
value: false
|
160 |
+
local_rank:
|
161 |
+
desc: null
|
162 |
+
value: -1
|
163 |
+
log_level:
|
164 |
+
desc: null
|
165 |
+
value: -1
|
166 |
+
log_level_replica:
|
167 |
+
desc: null
|
168 |
+
value: -1
|
169 |
+
log_on_each_node:
|
170 |
+
desc: null
|
171 |
+
value: true
|
172 |
+
logging_dir:
|
173 |
+
desc: null
|
174 |
+
value: ./runs/Nov08_07-36-00_t1v-n-ba7aa1a5-w-0
|
175 |
+
logging_first_step:
|
176 |
+
desc: null
|
177 |
+
value: false
|
178 |
+
logging_nan_inf_filter:
|
179 |
+
desc: null
|
180 |
+
value: true
|
181 |
+
logging_steps:
|
182 |
+
desc: null
|
183 |
+
value: 1000
|
184 |
+
logging_strategy:
|
185 |
+
desc: null
|
186 |
+
value: IntervalStrategy.STEPS
|
187 |
+
lr_scheduler_type:
|
188 |
+
desc: null
|
189 |
+
value: SchedulerType.LINEAR
|
190 |
+
max_grad_norm:
|
191 |
+
desc: null
|
192 |
+
value: 1.0
|
193 |
+
max_seq_length:
|
194 |
+
desc: null
|
195 |
+
value: 4096
|
196 |
+
max_steps:
|
197 |
+
desc: null
|
198 |
+
value: -1
|
199 |
+
metric_for_best_model:
|
200 |
+
desc: null
|
201 |
+
value: null
|
202 |
+
mlm_probability:
|
203 |
+
desc: null
|
204 |
+
value: 0.15
|
205 |
+
model_name_or_path:
|
206 |
+
desc: null
|
207 |
+
value: muril-bigbird
|
208 |
+
model_type:
|
209 |
+
desc: null
|
210 |
+
value: null
|
211 |
+
mp_parameters:
|
212 |
+
desc: null
|
213 |
+
value: ''
|
214 |
+
no_cuda:
|
215 |
+
desc: null
|
216 |
+
value: false
|
217 |
+
num_train_epochs:
|
218 |
+
desc: null
|
219 |
+
value: 3.0
|
220 |
+
output_dir:
|
221 |
+
desc: null
|
222 |
+
value: ./
|
223 |
+
overwrite_cache:
|
224 |
+
desc: null
|
225 |
+
value: false
|
226 |
+
overwrite_output_dir:
|
227 |
+
desc: null
|
228 |
+
value: true
|
229 |
+
pad_to_max_length:
|
230 |
+
desc: null
|
231 |
+
value: true
|
232 |
+
past_index:
|
233 |
+
desc: null
|
234 |
+
value: -1
|
235 |
+
per_device_eval_batch_size:
|
236 |
+
desc: null
|
237 |
+
value: 2
|
238 |
+
per_device_train_batch_size:
|
239 |
+
desc: null
|
240 |
+
value: 1
|
241 |
+
per_gpu_eval_batch_size:
|
242 |
+
desc: null
|
243 |
+
value: null
|
244 |
+
per_gpu_train_batch_size:
|
245 |
+
desc: null
|
246 |
+
value: null
|
247 |
+
prediction_loss_only:
|
248 |
+
desc: null
|
249 |
+
value: false
|
250 |
+
preprocessing_num_workers:
|
251 |
+
desc: null
|
252 |
+
value: null
|
253 |
+
pretokenized:
|
254 |
+
desc: null
|
255 |
+
value: true
|
256 |
+
push_to_hub:
|
257 |
+
desc: null
|
258 |
+
value: true
|
259 |
+
push_to_hub_model_id:
|
260 |
+
desc: null
|
261 |
+
value: null
|
262 |
+
push_to_hub_organization:
|
263 |
+
desc: null
|
264 |
+
value: null
|
265 |
+
push_to_hub_token:
|
266 |
+
desc: null
|
267 |
+
value: null
|
268 |
+
remove_unused_columns:
|
269 |
+
desc: null
|
270 |
+
value: true
|
271 |
+
report_to:
|
272 |
+
desc: null
|
273 |
+
value:
|
274 |
+
- tensorboard
|
275 |
+
- wandb
|
276 |
+
resume_from_checkpoint:
|
277 |
+
desc: null
|
278 |
+
value: checkpoint_100000
|
279 |
+
run_name:
|
280 |
+
desc: null
|
281 |
+
value: en-ta-hi-te-from-100k
|
282 |
+
save_on_each_node:
|
283 |
+
desc: null
|
284 |
+
value: false
|
285 |
+
save_steps:
|
286 |
+
desc: null
|
287 |
+
value: 25000
|
288 |
+
save_strategy:
|
289 |
+
desc: null
|
290 |
+
value: IntervalStrategy.STEPS
|
291 |
+
save_total_limit:
|
292 |
+
desc: null
|
293 |
+
value: 1
|
294 |
+
seed:
|
295 |
+
desc: null
|
296 |
+
value: 18
|
297 |
+
sharded_ddp:
|
298 |
+
desc: null
|
299 |
+
value: []
|
300 |
+
skip_memory_metrics:
|
301 |
+
desc: null
|
302 |
+
value: true
|
303 |
+
streaming:
|
304 |
+
desc: null
|
305 |
+
value: false
|
306 |
+
tokenizer_name:
|
307 |
+
desc: null
|
308 |
+
value: muril-bigbird
|
309 |
+
tpu_metrics_debug:
|
310 |
+
desc: null
|
311 |
+
value: false
|
312 |
+
tpu_num_cores:
|
313 |
+
desc: null
|
314 |
+
value: null
|
315 |
+
train_file:
|
316 |
+
desc: null
|
317 |
+
value: /dev/shm/data-files
|
318 |
+
train_ref_file:
|
319 |
+
desc: null
|
320 |
+
value: null
|
321 |
+
use_fast_tokenizer:
|
322 |
+
desc: null
|
323 |
+
value: true
|
324 |
+
use_legacy_prediction_loop:
|
325 |
+
desc: null
|
326 |
+
value: false
|
327 |
+
validation_file:
|
328 |
+
desc: null
|
329 |
+
value: .
|
330 |
+
validation_ref_file:
|
331 |
+
desc: null
|
332 |
+
value: null
|
333 |
+
validation_split_percentage:
|
334 |
+
desc: null
|
335 |
+
value: 5
|
336 |
+
warmup_ratio:
|
337 |
+
desc: null
|
338 |
+
value: 0.0
|
339 |
+
warmup_steps:
|
340 |
+
desc: null
|
341 |
+
value: 0
|
342 |
+
weight_decay:
|
343 |
+
desc: null
|
344 |
+
value: 0.01
|
345 |
+
xpu_backend:
|
346 |
+
desc: null
|
347 |
+
value: null
|
wandb/run-20211108_073656-hyqam6zi/files/output.log
ADDED
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
[07:37:15] - INFO - absl - A polynomial schedule was set with a non-positive `transition_steps` value; this results in a constant schedule with value `init_value`.
|
2 |
+
[07:37:15] - INFO - absl - Restoring checkpoint from checkpoint_100000
|
3 |
+
tcmalloc: large alloc 2886115328 bytes == 0x95a0a000 @ 0x7fe75a4b6680 0x7fe75a4d7824 0x5bc8a7 0x50a83b 0x50c08b 0x5f68ef 0x57107c 0x56a0ba 0x5f6343 0x56bddd 0x56a0ba 0x68d5b7 0x67cd01 0x67cd7f 0x67ce21 0x67ef47 0x6b7242 0x6b75cd 0x7fe75a2cd0b3 0x5fb18e
|
4 |
+
Traceback (most recent call last):
|
5 |
+
File "run_muril_bb_flax_mlm.py", line 612, in <module>
|
6 |
+
state = restore_checkpoint(training_args.resume_from_checkpoint, state)
|
7 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/training/checkpoints.py", line 274, in restore_checkpoint
|
8 |
+
results = list(results)
|
9 |
+
File "/usr/lib/python3.8/concurrent/futures/_base.py", line 619, in result_iterator
|
10 |
+
yield fs.pop().result()
|
11 |
+
File "/usr/lib/python3.8/concurrent/futures/_base.py", line 439, in result
|
12 |
+
self._condition.wait(timeout)
|
13 |
+
File "/usr/lib/python3.8/threading.py", line 302, in wait
|
14 |
+
waiter.acquire()
|
15 |
+
KeyboardInterrupt
|
wandb/run-20211108_073656-hyqam6zi/files/requirements.txt
ADDED
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
absl-py==0.15.0
|
2 |
+
aiohttp==3.8.0
|
3 |
+
aiosignal==1.2.0
|
4 |
+
astunparse==1.6.3
|
5 |
+
async-timeout==4.0.0
|
6 |
+
attrs==21.2.0
|
7 |
+
cachetools==4.2.4
|
8 |
+
certifi==2021.10.8
|
9 |
+
charset-normalizer==2.0.7
|
10 |
+
chex==0.0.8
|
11 |
+
click==8.0.3
|
12 |
+
configparser==5.1.0
|
13 |
+
cycler==0.11.0
|
14 |
+
datasets==1.15.1
|
15 |
+
dill==0.3.4
|
16 |
+
dm-tree==0.1.6
|
17 |
+
docker-pycreds==0.4.0
|
18 |
+
filelock==3.3.2
|
19 |
+
flatbuffers==2.0
|
20 |
+
flax==0.3.6
|
21 |
+
frozenlist==1.2.0
|
22 |
+
fsspec==2021.11.0
|
23 |
+
gast==0.4.0
|
24 |
+
gitdb==4.0.9
|
25 |
+
gitpython==3.1.24
|
26 |
+
google-auth-oauthlib==0.4.6
|
27 |
+
google-auth==2.3.3
|
28 |
+
google-pasta==0.2.0
|
29 |
+
grpcio==1.41.1
|
30 |
+
h5py==3.5.0
|
31 |
+
huggingface-hub==0.1.1
|
32 |
+
idna==3.3
|
33 |
+
jax==0.2.24
|
34 |
+
jaxlib==0.1.73
|
35 |
+
joblib==1.1.0
|
36 |
+
kaggle==1.5.12
|
37 |
+
keras-preprocessing==1.1.2
|
38 |
+
keras==2.7.0
|
39 |
+
kiwisolver==1.3.2
|
40 |
+
libclang==12.0.0
|
41 |
+
libtpu-nightly==0.1.dev20211018
|
42 |
+
markdown==3.3.4
|
43 |
+
matplotlib==3.4.3
|
44 |
+
msgpack==1.0.2
|
45 |
+
multidict==5.2.0
|
46 |
+
multiprocess==0.70.12.2
|
47 |
+
numpy==1.21.4
|
48 |
+
oauthlib==3.1.1
|
49 |
+
opt-einsum==3.3.0
|
50 |
+
optax==0.0.9
|
51 |
+
packaging==21.2
|
52 |
+
pandas==1.3.4
|
53 |
+
pathtools==0.1.2
|
54 |
+
pillow==8.4.0
|
55 |
+
pip==21.3.1
|
56 |
+
pkg-resources==0.0.0
|
57 |
+
promise==2.3
|
58 |
+
protobuf==3.19.1
|
59 |
+
psutil==5.8.0
|
60 |
+
pyarrow==6.0.0
|
61 |
+
pyasn1-modules==0.2.8
|
62 |
+
pyasn1==0.4.8
|
63 |
+
pyparsing==2.4.7
|
64 |
+
python-dateutil==2.8.2
|
65 |
+
python-slugify==5.0.2
|
66 |
+
pytz==2021.3
|
67 |
+
pyyaml==6.0
|
68 |
+
regex==2021.11.2
|
69 |
+
requests-oauthlib==1.3.0
|
70 |
+
requests==2.26.0
|
71 |
+
rsa==4.7.2
|
72 |
+
sacremoses==0.0.46
|
73 |
+
scipy==1.7.1
|
74 |
+
sentry-sdk==1.4.3
|
75 |
+
setuptools==44.0.0
|
76 |
+
shortuuid==1.0.1
|
77 |
+
six==1.16.0
|
78 |
+
smmap==5.0.0
|
79 |
+
subprocess32==3.5.4
|
80 |
+
tensorboard-data-server==0.6.1
|
81 |
+
tensorboard-plugin-wit==1.8.0
|
82 |
+
tensorboard==2.7.0
|
83 |
+
tensorflow-estimator==2.7.0
|
84 |
+
tensorflow-io-gcs-filesystem==0.21.0
|
85 |
+
tensorflow==2.7.0
|
86 |
+
termcolor==1.1.0
|
87 |
+
text-unidecode==1.3
|
88 |
+
tokenizers==0.10.3
|
89 |
+
toolz==0.11.1
|
90 |
+
torch==1.10.0
|
91 |
+
tqdm==4.62.3
|
92 |
+
transformers==4.13.0.dev0
|
93 |
+
typing-extensions==3.10.0.2
|
94 |
+
urllib3==1.26.7
|
95 |
+
wandb==0.12.6
|
96 |
+
werkzeug==2.0.2
|
97 |
+
wheel==0.37.0
|
98 |
+
wrapt==1.13.3
|
99 |
+
xxhash==2.0.2
|
100 |
+
yarl==1.7.2
|
101 |
+
yaspin==2.1.0
|
wandb/run-20211108_073656-hyqam6zi/files/wandb-metadata.json
ADDED
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
|
3 |
+
"python": "3.8.10",
|
4 |
+
"heartbeatAt": "2021-11-08T07:36:59.308384",
|
5 |
+
"startedAt": "2021-11-08T07:36:56.467866",
|
6 |
+
"docker": null,
|
7 |
+
"cpu_count": 96,
|
8 |
+
"cuda": null,
|
9 |
+
"args": [
|
10 |
+
"--model_name_or_path",
|
11 |
+
"muril-bigbird",
|
12 |
+
"--tokenizer_name",
|
13 |
+
"muril-bigbird",
|
14 |
+
"--dtype",
|
15 |
+
"bfloat16",
|
16 |
+
"--pretokenized",
|
17 |
+
"--resume_from_checkpoint",
|
18 |
+
"checkpoint_100000",
|
19 |
+
"--train_file",
|
20 |
+
"/dev/shm/data-files",
|
21 |
+
"--validation_file",
|
22 |
+
".",
|
23 |
+
"--max_seq_length",
|
24 |
+
"4096",
|
25 |
+
"--pad_to_max_length",
|
26 |
+
"--output_dir",
|
27 |
+
"./",
|
28 |
+
"--overwrite_output_dir",
|
29 |
+
"--do_train",
|
30 |
+
"--do_eval",
|
31 |
+
"--eval_steps",
|
32 |
+
"5000",
|
33 |
+
"--per_device_train_batch_size",
|
34 |
+
"1",
|
35 |
+
"--per_device_eval_batch_size",
|
36 |
+
"2",
|
37 |
+
"--learning_rate",
|
38 |
+
"5e-5",
|
39 |
+
"--weight_decay",
|
40 |
+
"0.01",
|
41 |
+
"--num_train_epochs",
|
42 |
+
"3",
|
43 |
+
"--warmup_steps",
|
44 |
+
"0",
|
45 |
+
"--logging_steps",
|
46 |
+
"1000",
|
47 |
+
"--save_steps",
|
48 |
+
"25000",
|
49 |
+
"--save_total_limit",
|
50 |
+
"1",
|
51 |
+
"--seed",
|
52 |
+
"18",
|
53 |
+
"--push_to_hub",
|
54 |
+
"--run_name",
|
55 |
+
"en-ta-hi-te-from-100k"
|
56 |
+
],
|
57 |
+
"state": "running",
|
58 |
+
"program": "run_muril_bb_flax_mlm.py",
|
59 |
+
"codePath": "run_muril_bb_flax_mlm.py",
|
60 |
+
"git": {
|
61 |
+
"remote": "https://huggingface.co/nbroad/muril-bigbird-base-1mil",
|
62 |
+
"commit": "5fca06e3f8deb06ee5f938513b5f093fc3b66a62"
|
63 |
+
},
|
64 |
+
"email": "nicholas@nmbroad.com",
|
65 |
+
"root": "/home/nicholasbroad/muril-bigbird-base-1mil",
|
66 |
+
"host": "t1v-n-ba7aa1a5-w-0",
|
67 |
+
"username": "nicholasbroad",
|
68 |
+
"executable": "/home/nicholasbroad/venv/bin/python"
|
69 |
+
}
|
wandb/run-20211108_073656-hyqam6zi/files/wandb-summary.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"_wandb": {"runtime": 25}}
|
wandb/run-20211108_073656-hyqam6zi/logs/debug-internal.log
ADDED
@@ -0,0 +1,145 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2021-11-08 07:36:57,125 INFO MainThread:30896 [internal.py:wandb_internal():87] W&B internal server running at pid: 30896, started at: 2021-11-08 07:36:57.125057
|
2 |
+
2021-11-08 07:36:57,127 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: check_version
|
3 |
+
2021-11-08 07:36:57,127 INFO WriterThread:30896 [datastore.py:open_for_write():77] open: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/run-hyqam6zi.wandb
|
4 |
+
2021-11-08 07:36:57,128 DEBUG SenderThread:30896 [sender.py:send():238] send: header
|
5 |
+
2021-11-08 07:36:57,128 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: check_version
|
6 |
+
2021-11-08 07:36:57,165 DEBUG SenderThread:30896 [sender.py:send():238] send: run
|
7 |
+
2021-11-08 07:36:57,327 INFO SenderThread:30896 [dir_watcher.py:__init__():169] watching files in: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files
|
8 |
+
2021-11-08 07:36:57,327 INFO SenderThread:30896 [sender.py:_start_run_threads():825] run started: hyqam6zi with start time 1636357016
|
9 |
+
2021-11-08 07:36:57,327 DEBUG SenderThread:30896 [sender.py:send():238] send: summary
|
10 |
+
2021-11-08 07:36:57,327 INFO SenderThread:30896 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
|
11 |
+
2021-11-08 07:36:57,328 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: run_start
|
12 |
+
2021-11-08 07:36:58,329 INFO Thread-8 :30896 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/wandb-summary.json
|
13 |
+
2021-11-08 07:36:59,308 DEBUG HandlerThread:30896 [meta.py:__init__():40] meta init
|
14 |
+
2021-11-08 07:36:59,308 DEBUG HandlerThread:30896 [meta.py:__init__():54] meta init done
|
15 |
+
2021-11-08 07:36:59,308 DEBUG HandlerThread:30896 [meta.py:probe():214] probe
|
16 |
+
2021-11-08 07:36:59,309 DEBUG HandlerThread:30896 [meta.py:_setup_git():204] setup git
|
17 |
+
2021-11-08 07:36:59,338 DEBUG HandlerThread:30896 [meta.py:_setup_git():211] setup git done
|
18 |
+
2021-11-08 07:36:59,338 DEBUG HandlerThread:30896 [meta.py:_save_pip():58] save pip
|
19 |
+
2021-11-08 07:36:59,339 DEBUG HandlerThread:30896 [meta.py:_save_pip():72] save pip done
|
20 |
+
2021-11-08 07:36:59,339 DEBUG HandlerThread:30896 [meta.py:probe():252] probe done
|
21 |
+
2021-11-08 07:36:59,342 DEBUG SenderThread:30896 [sender.py:send():238] send: files
|
22 |
+
2021-11-08 07:36:59,342 INFO SenderThread:30896 [sender.py:_save_file():960] saving file wandb-metadata.json with policy now
|
23 |
+
2021-11-08 07:36:59,347 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: stop_status
|
24 |
+
2021-11-08 07:36:59,347 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: stop_status
|
25 |
+
2021-11-08 07:36:59,480 DEBUG SenderThread:30896 [sender.py:send():238] send: config
|
26 |
+
2021-11-08 07:36:59,480 DEBUG SenderThread:30896 [sender.py:send():238] send: config
|
27 |
+
2021-11-08 07:36:59,480 DEBUG SenderThread:30896 [sender.py:send():238] send: config
|
28 |
+
2021-11-08 07:36:59,806 INFO Thread-11 :30896 [upload_job.py:push():137] Uploaded file /tmp/tmptc9slxn9wandb/2u6gi7c9-wandb-metadata.json
|
29 |
+
2021-11-08 07:37:00,336 INFO Thread-8 :30896 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/requirements.txt
|
30 |
+
2021-11-08 07:37:00,337 INFO Thread-8 :30896 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/wandb-metadata.json
|
31 |
+
2021-11-08 07:37:00,337 INFO Thread-8 :30896 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/output.log
|
32 |
+
2021-11-08 07:37:14,958 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: stop_status
|
33 |
+
2021-11-08 07:37:14,958 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: stop_status
|
34 |
+
2021-11-08 07:37:19,976 WARNING MainThread:30896 [internal.py:wandb_internal():146] Internal process interrupt: 1
|
35 |
+
2021-11-08 07:37:20,344 INFO Thread-8 :30896 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/output.log
|
36 |
+
2021-11-08 07:37:22,344 INFO Thread-8 :30896 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/output.log
|
37 |
+
2021-11-08 07:37:22,876 DEBUG SenderThread:30896 [sender.py:send():238] send: telemetry
|
38 |
+
2021-11-08 07:37:22,876 DEBUG SenderThread:30896 [sender.py:send():238] send: exit
|
39 |
+
2021-11-08 07:37:22,877 INFO SenderThread:30896 [sender.py:send_exit():370] handling exit code: 255
|
40 |
+
2021-11-08 07:37:22,877 INFO SenderThread:30896 [sender.py:send_exit():372] handling runtime: 25
|
41 |
+
2021-11-08 07:37:22,877 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
|
42 |
+
2021-11-08 07:37:22,878 INFO SenderThread:30896 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
|
43 |
+
2021-11-08 07:37:22,878 INFO SenderThread:30896 [sender.py:send_exit():382] send defer
|
44 |
+
2021-11-08 07:37:22,878 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
|
45 |
+
2021-11-08 07:37:22,879 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
|
46 |
+
2021-11-08 07:37:22,879 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 0
|
47 |
+
2021-11-08 07:37:22,879 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
|
48 |
+
2021-11-08 07:37:22,879 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 0
|
49 |
+
2021-11-08 07:37:22,879 INFO SenderThread:30896 [sender.py:transition_state():395] send defer: 1
|
50 |
+
2021-11-08 07:37:22,879 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
|
51 |
+
2021-11-08 07:37:22,879 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 1
|
52 |
+
2021-11-08 07:37:22,976 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
|
53 |
+
2021-11-08 07:37:22,976 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 1
|
54 |
+
2021-11-08 07:37:22,976 INFO SenderThread:30896 [sender.py:transition_state():395] send defer: 2
|
55 |
+
2021-11-08 07:37:22,976 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
|
56 |
+
2021-11-08 07:37:22,976 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 2
|
57 |
+
2021-11-08 07:37:22,976 DEBUG SenderThread:30896 [sender.py:send():238] send: stats
|
58 |
+
2021-11-08 07:37:22,976 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
|
59 |
+
2021-11-08 07:37:22,977 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 2
|
60 |
+
2021-11-08 07:37:22,977 INFO SenderThread:30896 [sender.py:transition_state():395] send defer: 3
|
61 |
+
2021-11-08 07:37:22,977 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
|
62 |
+
2021-11-08 07:37:22,977 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 3
|
63 |
+
2021-11-08 07:37:22,977 DEBUG SenderThread:30896 [sender.py:send():238] send: summary
|
64 |
+
2021-11-08 07:37:22,978 INFO SenderThread:30896 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
|
65 |
+
2021-11-08 07:37:22,978 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
|
66 |
+
2021-11-08 07:37:22,978 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 3
|
67 |
+
2021-11-08 07:37:22,978 INFO SenderThread:30896 [sender.py:transition_state():395] send defer: 4
|
68 |
+
2021-11-08 07:37:22,978 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
|
69 |
+
2021-11-08 07:37:22,978 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 4
|
70 |
+
2021-11-08 07:37:22,979 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
|
71 |
+
2021-11-08 07:37:22,979 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 4
|
72 |
+
2021-11-08 07:37:22,983 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
|
73 |
+
2021-11-08 07:37:23,162 INFO SenderThread:30896 [sender.py:transition_state():395] send defer: 5
|
74 |
+
2021-11-08 07:37:23,162 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
|
75 |
+
2021-11-08 07:37:23,163 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
|
76 |
+
2021-11-08 07:37:23,163 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 5
|
77 |
+
2021-11-08 07:37:23,163 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
|
78 |
+
2021-11-08 07:37:23,163 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 5
|
79 |
+
2021-11-08 07:37:23,163 INFO SenderThread:30896 [dir_watcher.py:finish():283] shutting down directory watcher
|
80 |
+
2021-11-08 07:37:23,264 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
|
81 |
+
2021-11-08 07:37:23,345 INFO Thread-8 :30896 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/wandb-summary.json
|
82 |
+
2021-11-08 07:37:23,345 INFO SenderThread:30896 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/config.yaml
|
83 |
+
2021-11-08 07:37:23,346 INFO SenderThread:30896 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/output.log
|
84 |
+
2021-11-08 07:37:23,346 INFO SenderThread:30896 [dir_watcher.py:finish():313] scan: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files
|
85 |
+
2021-11-08 07:37:23,346 INFO SenderThread:30896 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/requirements.txt requirements.txt
|
86 |
+
2021-11-08 07:37:23,346 INFO SenderThread:30896 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/output.log output.log
|
87 |
+
2021-11-08 07:37:23,346 INFO SenderThread:30896 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/wandb-metadata.json wandb-metadata.json
|
88 |
+
2021-11-08 07:37:23,346 INFO SenderThread:30896 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/config.yaml config.yaml
|
89 |
+
2021-11-08 07:37:23,347 INFO SenderThread:30896 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/wandb-summary.json wandb-summary.json
|
90 |
+
2021-11-08 07:37:23,352 INFO SenderThread:30896 [sender.py:transition_state():395] send defer: 6
|
91 |
+
2021-11-08 07:37:23,353 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
|
92 |
+
2021-11-08 07:37:23,356 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
|
93 |
+
2021-11-08 07:37:23,356 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 6
|
94 |
+
2021-11-08 07:37:23,359 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
|
95 |
+
2021-11-08 07:37:23,359 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 6
|
96 |
+
2021-11-08 07:37:23,359 INFO SenderThread:30896 [file_pusher.py:finish():177] shutting down file pusher
|
97 |
+
2021-11-08 07:37:23,454 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
|
98 |
+
2021-11-08 07:37:23,454 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
|
99 |
+
2021-11-08 07:37:23,556 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
|
100 |
+
2021-11-08 07:37:23,556 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
|
101 |
+
2021-11-08 07:37:23,658 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
|
102 |
+
2021-11-08 07:37:23,658 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
|
103 |
+
2021-11-08 07:37:23,759 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
|
104 |
+
2021-11-08 07:37:23,760 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
|
105 |
+
2021-11-08 07:37:23,853 INFO Thread-14 :30896 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/config.yaml
|
106 |
+
2021-11-08 07:37:23,861 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
|
107 |
+
2021-11-08 07:37:23,861 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
|
108 |
+
2021-11-08 07:37:23,867 INFO Thread-13 :30896 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/output.log
|
109 |
+
2021-11-08 07:37:23,908 INFO Thread-12 :30896 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/requirements.txt
|
110 |
+
2021-11-08 07:37:23,921 INFO Thread-15 :30896 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/files/wandb-summary.json
|
111 |
+
2021-11-08 07:37:23,963 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
|
112 |
+
2021-11-08 07:37:23,963 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
|
113 |
+
2021-11-08 07:37:24,064 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
|
114 |
+
2021-11-08 07:37:24,065 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
|
115 |
+
2021-11-08 07:37:24,122 INFO Thread-7 :30896 [sender.py:transition_state():395] send defer: 7
|
116 |
+
2021-11-08 07:37:24,122 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
|
117 |
+
2021-11-08 07:37:24,122 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 7
|
118 |
+
2021-11-08 07:37:24,122 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
|
119 |
+
2021-11-08 07:37:24,122 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 7
|
120 |
+
2021-11-08 07:37:24,166 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
|
121 |
+
2021-11-08 07:37:24,407 INFO SenderThread:30896 [sender.py:transition_state():395] send defer: 8
|
122 |
+
2021-11-08 07:37:24,408 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
|
123 |
+
2021-11-08 07:37:24,408 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
|
124 |
+
2021-11-08 07:37:24,408 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 8
|
125 |
+
2021-11-08 07:37:24,408 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
|
126 |
+
2021-11-08 07:37:24,409 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 8
|
127 |
+
2021-11-08 07:37:24,409 INFO SenderThread:30896 [sender.py:transition_state():395] send defer: 9
|
128 |
+
2021-11-08 07:37:24,409 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: defer
|
129 |
+
2021-11-08 07:37:24,409 INFO HandlerThread:30896 [handler.py:handle_request_defer():147] handle defer: 9
|
130 |
+
2021-11-08 07:37:24,409 DEBUG SenderThread:30896 [sender.py:send():238] send: final
|
131 |
+
2021-11-08 07:37:24,410 DEBUG SenderThread:30896 [sender.py:send():238] send: footer
|
132 |
+
2021-11-08 07:37:24,410 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: defer
|
133 |
+
2021-11-08 07:37:24,410 INFO SenderThread:30896 [sender.py:send_request_defer():391] handle sender defer: 9
|
134 |
+
2021-11-08 07:37:24,509 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: poll_exit
|
135 |
+
2021-11-08 07:37:24,509 DEBUG SenderThread:30896 [sender.py:send_request():252] send_request: poll_exit
|
136 |
+
2021-11-08 07:37:24,509 INFO SenderThread:30896 [file_pusher.py:join():182] waiting for file pusher
|
137 |
+
2021-11-08 07:37:24,762 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: get_summary
|
138 |
+
2021-11-08 07:37:24,762 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: sampled_history
|
139 |
+
2021-11-08 07:37:24,763 DEBUG HandlerThread:30896 [handler.py:handle_request():130] handle_request: shutdown
|
140 |
+
2021-11-08 07:37:24,763 INFO HandlerThread:30896 [handler.py:finish():731] shutting down handler
|
141 |
+
2021-11-08 07:37:25,410 INFO WriterThread:30896 [datastore.py:close():281] close: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/run-hyqam6zi.wandb
|
142 |
+
2021-11-08 07:37:25,761 INFO SenderThread:30896 [sender.py:finish():1091] shutting down sender
|
143 |
+
2021-11-08 07:37:25,761 INFO SenderThread:30896 [file_pusher.py:finish():177] shutting down file pusher
|
144 |
+
2021-11-08 07:37:25,761 INFO SenderThread:30896 [file_pusher.py:join():182] waiting for file pusher
|
145 |
+
2021-11-08 07:37:25,763 INFO MainThread:30896 [internal.py:handle_exit():77] Internal process exited
|
wandb/run-20211108_073656-hyqam6zi/logs/debug.log
ADDED
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2021-11-08 07:36:56,476 INFO MainThread:29588 [wandb_setup.py:_flush():71] setting env: {}
|
2 |
+
2021-11-08 07:36:56,477 INFO MainThread:29588 [wandb_setup.py:_flush():71] setting login settings: {}
|
3 |
+
2021-11-08 07:36:56,477 INFO MainThread:29588 [wandb_init.py:_log_setup():357] Logging user logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/logs/debug.log
|
4 |
+
2021-11-08 07:36:56,477 INFO MainThread:29588 [wandb_init.py:_log_setup():358] Logging internal logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_073656-hyqam6zi/logs/debug-internal.log
|
5 |
+
2021-11-08 07:36:56,477 INFO MainThread:29588 [wandb_init.py:init():390] calling init triggers
|
6 |
+
2021-11-08 07:36:56,477 INFO MainThread:29588 [wandb_init.py:init():395] wandb.init called with sweep_config: {}
|
7 |
+
config: {}
|
8 |
+
2021-11-08 07:36:56,477 INFO MainThread:29588 [wandb_init.py:init():435] starting backend
|
9 |
+
2021-11-08 07:36:56,477 INFO MainThread:29588 [backend.py:_multiprocessing_setup():95] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
|
10 |
+
2021-11-08 07:36:56,515 INFO MainThread:29588 [backend.py:ensure_launched():199] starting backend process...
|
11 |
+
2021-11-08 07:36:56,551 INFO MainThread:29588 [backend.py:ensure_launched():204] started backend process with pid: 30896
|
12 |
+
2021-11-08 07:36:56,553 INFO MainThread:29588 [wandb_init.py:init():444] backend started and connected
|
13 |
+
2021-11-08 07:36:56,555 INFO MainThread:29588 [wandb_init.py:init():503] updated telemetry
|
14 |
+
2021-11-08 07:36:56,638 INFO MainThread:29588 [wandb_init.py:init():533] communicating current version
|
15 |
+
2021-11-08 07:36:57,164 INFO MainThread:29588 [wandb_init.py:init():538] got version response
|
16 |
+
2021-11-08 07:36:57,164 INFO MainThread:29588 [wandb_init.py:init():548] communicating run to backend with 30 second timeout
|
17 |
+
2021-11-08 07:36:57,328 INFO MainThread:29588 [wandb_init.py:init():576] starting run threads in backend
|
18 |
+
2021-11-08 07:36:59,347 INFO MainThread:29588 [wandb_run.py:_console_start():1693] atexit reg
|
19 |
+
2021-11-08 07:36:59,348 INFO MainThread:29588 [wandb_run.py:_redirect():1567] redirect: SettingsConsole.REDIRECT
|
20 |
+
2021-11-08 07:36:59,348 INFO MainThread:29588 [wandb_run.py:_redirect():1572] Redirecting console.
|
21 |
+
2021-11-08 07:36:59,350 INFO MainThread:29588 [wandb_run.py:_redirect():1628] Redirects installed.
|
22 |
+
2021-11-08 07:36:59,350 INFO MainThread:29588 [wandb_init.py:init():603] run started, returning control to user process
|
23 |
+
2021-11-08 07:36:59,355 INFO MainThread:29588 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov08_07-36-00_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 1000, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 25000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te-from-100k', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'checkpoint_100000', 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
|
24 |
+
2021-11-08 07:36:59,357 INFO MainThread:29588 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
|
25 |
+
2021-11-08 07:36:59,358 INFO MainThread:29588 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}
|
26 |
+
2021-11-08 07:37:20,244 INFO MainThread:29588 [wandb_run.py:_atexit_cleanup():1663] got exitcode: 255
|
27 |
+
2021-11-08 07:37:20,245 INFO MainThread:29588 [wandb_run.py:_restore():1635] restore
|
28 |
+
2021-11-08 07:37:22,879 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
29 |
+
wandb_count: 1
|
30 |
+
}
|
31 |
+
pusher_stats {
|
32 |
+
uploaded_bytes: 1825
|
33 |
+
total_bytes: 1825
|
34 |
+
}
|
35 |
+
|
36 |
+
2021-11-08 07:37:23,163 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
37 |
+
wandb_count: 1
|
38 |
+
}
|
39 |
+
pusher_stats {
|
40 |
+
uploaded_bytes: 1825
|
41 |
+
total_bytes: 1825
|
42 |
+
}
|
43 |
+
|
44 |
+
2021-11-08 07:37:23,353 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
45 |
+
wandb_count: 4
|
46 |
+
}
|
47 |
+
pusher_stats {
|
48 |
+
uploaded_bytes: 1825
|
49 |
+
total_bytes: 9944
|
50 |
+
}
|
51 |
+
|
52 |
+
2021-11-08 07:37:23,455 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
53 |
+
wandb_count: 5
|
54 |
+
}
|
55 |
+
pusher_stats {
|
56 |
+
uploaded_bytes: 1825
|
57 |
+
total_bytes: 9971
|
58 |
+
}
|
59 |
+
|
60 |
+
2021-11-08 07:37:23,557 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
61 |
+
wandb_count: 5
|
62 |
+
}
|
63 |
+
pusher_stats {
|
64 |
+
uploaded_bytes: 9971
|
65 |
+
total_bytes: 9971
|
66 |
+
}
|
67 |
+
|
68 |
+
2021-11-08 07:37:23,659 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
69 |
+
wandb_count: 5
|
70 |
+
}
|
71 |
+
pusher_stats {
|
72 |
+
uploaded_bytes: 9971
|
73 |
+
total_bytes: 9971
|
74 |
+
}
|
75 |
+
|
76 |
+
2021-11-08 07:37:23,760 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
77 |
+
wandb_count: 5
|
78 |
+
}
|
79 |
+
pusher_stats {
|
80 |
+
uploaded_bytes: 9971
|
81 |
+
total_bytes: 9971
|
82 |
+
}
|
83 |
+
|
84 |
+
2021-11-08 07:37:23,862 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
85 |
+
wandb_count: 5
|
86 |
+
}
|
87 |
+
pusher_stats {
|
88 |
+
uploaded_bytes: 9971
|
89 |
+
total_bytes: 9971
|
90 |
+
}
|
91 |
+
|
92 |
+
2021-11-08 07:37:23,963 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
93 |
+
wandb_count: 5
|
94 |
+
}
|
95 |
+
pusher_stats {
|
96 |
+
uploaded_bytes: 9971
|
97 |
+
total_bytes: 9971
|
98 |
+
}
|
99 |
+
|
100 |
+
2021-11-08 07:37:24,065 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
101 |
+
wandb_count: 5
|
102 |
+
}
|
103 |
+
pusher_stats {
|
104 |
+
uploaded_bytes: 9971
|
105 |
+
total_bytes: 9971
|
106 |
+
}
|
107 |
+
|
108 |
+
2021-11-08 07:37:24,408 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
109 |
+
wandb_count: 5
|
110 |
+
}
|
111 |
+
pusher_stats {
|
112 |
+
uploaded_bytes: 9971
|
113 |
+
total_bytes: 9971
|
114 |
+
}
|
115 |
+
|
116 |
+
2021-11-08 07:37:24,761 INFO MainThread:29588 [wandb_run.py:_wait_for_finish():1793] got exit ret: done: true
|
117 |
+
exit_result {
|
118 |
+
}
|
119 |
+
file_counts {
|
120 |
+
wandb_count: 5
|
121 |
+
}
|
122 |
+
pusher_stats {
|
123 |
+
uploaded_bytes: 9971
|
124 |
+
total_bytes: 9971
|
125 |
+
}
|
126 |
+
local_info {
|
127 |
+
}
|
128 |
+
|
129 |
+
2021-11-08 07:37:26,128 INFO MainThread:29588 [wandb_run.py:_append_files():2061] logging synced files
|
wandb/run-20211108_073656-hyqam6zi/run-hyqam6zi.wandb
ADDED
Binary file (4.85 kB). View file
|
|
wandb/run-20211108_080537-1gq01aek/files/config.yaml
ADDED
@@ -0,0 +1,347 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
wandb_version: 1
|
2 |
+
|
3 |
+
__cached__setup_devices:
|
4 |
+
desc: null
|
5 |
+
value: cpu
|
6 |
+
_n_gpu:
|
7 |
+
desc: null
|
8 |
+
value: 0
|
9 |
+
_wandb:
|
10 |
+
desc: null
|
11 |
+
value:
|
12 |
+
cli_version: 0.12.6
|
13 |
+
framework: huggingface
|
14 |
+
huggingface_version: 4.13.0.dev0
|
15 |
+
is_jupyter_run: false
|
16 |
+
is_kaggle_kernel: true
|
17 |
+
python_version: 3.8.10
|
18 |
+
start_time: 1636358737
|
19 |
+
t:
|
20 |
+
1:
|
21 |
+
- 1
|
22 |
+
- 2
|
23 |
+
- 3
|
24 |
+
- 11
|
25 |
+
- 12
|
26 |
+
2:
|
27 |
+
- 1
|
28 |
+
- 2
|
29 |
+
- 3
|
30 |
+
- 11
|
31 |
+
- 12
|
32 |
+
3:
|
33 |
+
- 13
|
34 |
+
4: 3.8.10
|
35 |
+
5: 0.12.6
|
36 |
+
6: 4.13.0.dev0
|
37 |
+
8:
|
38 |
+
- 2
|
39 |
+
- 5
|
40 |
+
adafactor:
|
41 |
+
desc: null
|
42 |
+
value: false
|
43 |
+
adam_beta1:
|
44 |
+
desc: null
|
45 |
+
value: 0.9
|
46 |
+
adam_beta2:
|
47 |
+
desc: null
|
48 |
+
value: 0.999
|
49 |
+
adam_epsilon:
|
50 |
+
desc: null
|
51 |
+
value: 1.0e-08
|
52 |
+
cache_dir:
|
53 |
+
desc: null
|
54 |
+
value: null
|
55 |
+
config_name:
|
56 |
+
desc: null
|
57 |
+
value: null
|
58 |
+
dataloader_drop_last:
|
59 |
+
desc: null
|
60 |
+
value: false
|
61 |
+
dataloader_num_workers:
|
62 |
+
desc: null
|
63 |
+
value: 0
|
64 |
+
dataloader_pin_memory:
|
65 |
+
desc: null
|
66 |
+
value: true
|
67 |
+
dataset_config_name:
|
68 |
+
desc: null
|
69 |
+
value: null
|
70 |
+
dataset_name:
|
71 |
+
desc: null
|
72 |
+
value: null
|
73 |
+
ddp_find_unused_parameters:
|
74 |
+
desc: null
|
75 |
+
value: null
|
76 |
+
debug:
|
77 |
+
desc: null
|
78 |
+
value: []
|
79 |
+
deepspeed:
|
80 |
+
desc: null
|
81 |
+
value: null
|
82 |
+
disable_tqdm:
|
83 |
+
desc: null
|
84 |
+
value: false
|
85 |
+
do_eval:
|
86 |
+
desc: null
|
87 |
+
value: true
|
88 |
+
do_predict:
|
89 |
+
desc: null
|
90 |
+
value: false
|
91 |
+
do_train:
|
92 |
+
desc: null
|
93 |
+
value: true
|
94 |
+
dtype:
|
95 |
+
desc: null
|
96 |
+
value: bfloat16
|
97 |
+
eval_accumulation_steps:
|
98 |
+
desc: null
|
99 |
+
value: null
|
100 |
+
eval_steps:
|
101 |
+
desc: null
|
102 |
+
value: 5000
|
103 |
+
evaluation_strategy:
|
104 |
+
desc: null
|
105 |
+
value: IntervalStrategy.NO
|
106 |
+
fp16:
|
107 |
+
desc: null
|
108 |
+
value: false
|
109 |
+
fp16_backend:
|
110 |
+
desc: null
|
111 |
+
value: auto
|
112 |
+
fp16_full_eval:
|
113 |
+
desc: null
|
114 |
+
value: false
|
115 |
+
fp16_opt_level:
|
116 |
+
desc: null
|
117 |
+
value: O1
|
118 |
+
gradient_accumulation_steps:
|
119 |
+
desc: null
|
120 |
+
value: 1
|
121 |
+
gradient_checkpointing:
|
122 |
+
desc: null
|
123 |
+
value: false
|
124 |
+
greater_is_better:
|
125 |
+
desc: null
|
126 |
+
value: null
|
127 |
+
group_by_length:
|
128 |
+
desc: null
|
129 |
+
value: false
|
130 |
+
hub_model_id:
|
131 |
+
desc: null
|
132 |
+
value: null
|
133 |
+
hub_strategy:
|
134 |
+
desc: null
|
135 |
+
value: HubStrategy.EVERY_SAVE
|
136 |
+
hub_token:
|
137 |
+
desc: null
|
138 |
+
value: null
|
139 |
+
ignore_data_skip:
|
140 |
+
desc: null
|
141 |
+
value: false
|
142 |
+
label_names:
|
143 |
+
desc: null
|
144 |
+
value: null
|
145 |
+
label_smoothing_factor:
|
146 |
+
desc: null
|
147 |
+
value: 0.0
|
148 |
+
learning_rate:
|
149 |
+
desc: null
|
150 |
+
value: 5.0e-05
|
151 |
+
length_column_name:
|
152 |
+
desc: null
|
153 |
+
value: length
|
154 |
+
line_by_line:
|
155 |
+
desc: null
|
156 |
+
value: false
|
157 |
+
load_best_model_at_end:
|
158 |
+
desc: null
|
159 |
+
value: false
|
160 |
+
local_rank:
|
161 |
+
desc: null
|
162 |
+
value: -1
|
163 |
+
log_level:
|
164 |
+
desc: null
|
165 |
+
value: -1
|
166 |
+
log_level_replica:
|
167 |
+
desc: null
|
168 |
+
value: -1
|
169 |
+
log_on_each_node:
|
170 |
+
desc: null
|
171 |
+
value: true
|
172 |
+
logging_dir:
|
173 |
+
desc: null
|
174 |
+
value: ./runs/Nov08_08-04-38_t1v-n-ba7aa1a5-w-0
|
175 |
+
logging_first_step:
|
176 |
+
desc: null
|
177 |
+
value: false
|
178 |
+
logging_nan_inf_filter:
|
179 |
+
desc: null
|
180 |
+
value: true
|
181 |
+
logging_steps:
|
182 |
+
desc: null
|
183 |
+
value: 1000
|
184 |
+
logging_strategy:
|
185 |
+
desc: null
|
186 |
+
value: IntervalStrategy.STEPS
|
187 |
+
lr_scheduler_type:
|
188 |
+
desc: null
|
189 |
+
value: SchedulerType.LINEAR
|
190 |
+
max_grad_norm:
|
191 |
+
desc: null
|
192 |
+
value: 1.0
|
193 |
+
max_seq_length:
|
194 |
+
desc: null
|
195 |
+
value: 4096
|
196 |
+
max_steps:
|
197 |
+
desc: null
|
198 |
+
value: -1
|
199 |
+
metric_for_best_model:
|
200 |
+
desc: null
|
201 |
+
value: null
|
202 |
+
mlm_probability:
|
203 |
+
desc: null
|
204 |
+
value: 0.15
|
205 |
+
model_name_or_path:
|
206 |
+
desc: null
|
207 |
+
value: muril-bigbird
|
208 |
+
model_type:
|
209 |
+
desc: null
|
210 |
+
value: null
|
211 |
+
mp_parameters:
|
212 |
+
desc: null
|
213 |
+
value: ''
|
214 |
+
no_cuda:
|
215 |
+
desc: null
|
216 |
+
value: false
|
217 |
+
num_train_epochs:
|
218 |
+
desc: null
|
219 |
+
value: 3.0
|
220 |
+
output_dir:
|
221 |
+
desc: null
|
222 |
+
value: ./
|
223 |
+
overwrite_cache:
|
224 |
+
desc: null
|
225 |
+
value: false
|
226 |
+
overwrite_output_dir:
|
227 |
+
desc: null
|
228 |
+
value: true
|
229 |
+
pad_to_max_length:
|
230 |
+
desc: null
|
231 |
+
value: true
|
232 |
+
past_index:
|
233 |
+
desc: null
|
234 |
+
value: -1
|
235 |
+
per_device_eval_batch_size:
|
236 |
+
desc: null
|
237 |
+
value: 2
|
238 |
+
per_device_train_batch_size:
|
239 |
+
desc: null
|
240 |
+
value: 1
|
241 |
+
per_gpu_eval_batch_size:
|
242 |
+
desc: null
|
243 |
+
value: null
|
244 |
+
per_gpu_train_batch_size:
|
245 |
+
desc: null
|
246 |
+
value: null
|
247 |
+
prediction_loss_only:
|
248 |
+
desc: null
|
249 |
+
value: false
|
250 |
+
preprocessing_num_workers:
|
251 |
+
desc: null
|
252 |
+
value: null
|
253 |
+
pretokenized:
|
254 |
+
desc: null
|
255 |
+
value: true
|
256 |
+
push_to_hub:
|
257 |
+
desc: null
|
258 |
+
value: true
|
259 |
+
push_to_hub_model_id:
|
260 |
+
desc: null
|
261 |
+
value: null
|
262 |
+
push_to_hub_organization:
|
263 |
+
desc: null
|
264 |
+
value: null
|
265 |
+
push_to_hub_token:
|
266 |
+
desc: null
|
267 |
+
value: null
|
268 |
+
remove_unused_columns:
|
269 |
+
desc: null
|
270 |
+
value: true
|
271 |
+
report_to:
|
272 |
+
desc: null
|
273 |
+
value:
|
274 |
+
- tensorboard
|
275 |
+
- wandb
|
276 |
+
resume_from_checkpoint:
|
277 |
+
desc: null
|
278 |
+
value: checkpoint_100000
|
279 |
+
run_name:
|
280 |
+
desc: null
|
281 |
+
value: en-ta-hi-te-from-100k
|
282 |
+
save_on_each_node:
|
283 |
+
desc: null
|
284 |
+
value: false
|
285 |
+
save_steps:
|
286 |
+
desc: null
|
287 |
+
value: 25000
|
288 |
+
save_strategy:
|
289 |
+
desc: null
|
290 |
+
value: IntervalStrategy.STEPS
|
291 |
+
save_total_limit:
|
292 |
+
desc: null
|
293 |
+
value: 1
|
294 |
+
seed:
|
295 |
+
desc: null
|
296 |
+
value: 18
|
297 |
+
sharded_ddp:
|
298 |
+
desc: null
|
299 |
+
value: []
|
300 |
+
skip_memory_metrics:
|
301 |
+
desc: null
|
302 |
+
value: true
|
303 |
+
streaming:
|
304 |
+
desc: null
|
305 |
+
value: false
|
306 |
+
tokenizer_name:
|
307 |
+
desc: null
|
308 |
+
value: muril-bigbird
|
309 |
+
tpu_metrics_debug:
|
310 |
+
desc: null
|
311 |
+
value: false
|
312 |
+
tpu_num_cores:
|
313 |
+
desc: null
|
314 |
+
value: null
|
315 |
+
train_file:
|
316 |
+
desc: null
|
317 |
+
value: /dev/shm/data-files
|
318 |
+
train_ref_file:
|
319 |
+
desc: null
|
320 |
+
value: null
|
321 |
+
use_fast_tokenizer:
|
322 |
+
desc: null
|
323 |
+
value: true
|
324 |
+
use_legacy_prediction_loop:
|
325 |
+
desc: null
|
326 |
+
value: false
|
327 |
+
validation_file:
|
328 |
+
desc: null
|
329 |
+
value: .
|
330 |
+
validation_ref_file:
|
331 |
+
desc: null
|
332 |
+
value: null
|
333 |
+
validation_split_percentage:
|
334 |
+
desc: null
|
335 |
+
value: 5
|
336 |
+
warmup_ratio:
|
337 |
+
desc: null
|
338 |
+
value: 0.0
|
339 |
+
warmup_steps:
|
340 |
+
desc: null
|
341 |
+
value: 0
|
342 |
+
weight_decay:
|
343 |
+
desc: null
|
344 |
+
value: 0.01
|
345 |
+
xpu_backend:
|
346 |
+
desc: null
|
347 |
+
value: null
|
wandb/run-20211108_080537-1gq01aek/files/output.log
ADDED
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
|
2 |
+
[08:06:04] - INFO - absl - A polynomial schedule was set with a non-positive `transition_steps` value; this results in a constant schedule with value `init_value`.
|
3 |
+
[08:06:04] - INFO - absl - Restoring checkpoint from checkpoint_100000
|
4 |
+
tcmalloc: large alloc 2886115328 bytes == 0x943cc000 @ 0x7f99f07a3680 0x7f99f07c4824 0x5bc8a7 0x50a83b 0x50c08b 0x5f68ef 0x57107c 0x56a0ba 0x5f6343 0x56bddd 0x56a0ba 0x68d5b7 0x67cd01 0x67cd7f 0x67ce21 0x67ef47 0x6b7242 0x6b75cd 0x7f99f05ba0b3 0x5fb18e
|
5 |
+
Traceback (most recent call last):
|
6 |
+
File "run_muril_bb_flax_mlm.py", line 612, in <module>
|
7 |
+
state = restore_checkpoint(training_args.resume_from_checkpoint, state)
|
8 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/training/checkpoints.py", line 274, in restore_checkpoint
|
9 |
+
results = list(results)
|
10 |
+
File "/usr/lib/python3.8/concurrent/futures/_base.py", line 619, in result_iterator
|
11 |
+
yield fs.pop().result()
|
12 |
+
File "/usr/lib/python3.8/concurrent/futures/_base.py", line 439, in result
|
13 |
+
self._condition.wait(timeout)
|
14 |
+
File "/usr/lib/python3.8/threading.py", line 302, in wait
|
15 |
+
waiter.acquire()
|
16 |
+
KeyboardInterrupt
|
wandb/run-20211108_080537-1gq01aek/files/requirements.txt
ADDED
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
absl-py==0.15.0
|
2 |
+
aiohttp==3.8.0
|
3 |
+
aiosignal==1.2.0
|
4 |
+
astunparse==1.6.3
|
5 |
+
async-timeout==4.0.0
|
6 |
+
attrs==21.2.0
|
7 |
+
cachetools==4.2.4
|
8 |
+
certifi==2021.10.8
|
9 |
+
charset-normalizer==2.0.7
|
10 |
+
chex==0.0.8
|
11 |
+
click==8.0.3
|
12 |
+
configparser==5.1.0
|
13 |
+
cycler==0.11.0
|
14 |
+
datasets==1.15.1
|
15 |
+
dill==0.3.4
|
16 |
+
dm-tree==0.1.6
|
17 |
+
docker-pycreds==0.4.0
|
18 |
+
filelock==3.3.2
|
19 |
+
flatbuffers==2.0
|
20 |
+
flax==0.3.6
|
21 |
+
frozenlist==1.2.0
|
22 |
+
fsspec==2021.11.0
|
23 |
+
gast==0.4.0
|
24 |
+
gitdb==4.0.9
|
25 |
+
gitpython==3.1.24
|
26 |
+
google-auth-oauthlib==0.4.6
|
27 |
+
google-auth==2.3.3
|
28 |
+
google-pasta==0.2.0
|
29 |
+
grpcio==1.41.1
|
30 |
+
h5py==3.5.0
|
31 |
+
huggingface-hub==0.1.1
|
32 |
+
idna==3.3
|
33 |
+
jax==0.2.24
|
34 |
+
jaxlib==0.1.73
|
35 |
+
joblib==1.1.0
|
36 |
+
kaggle==1.5.12
|
37 |
+
keras-preprocessing==1.1.2
|
38 |
+
keras==2.7.0
|
39 |
+
kiwisolver==1.3.2
|
40 |
+
libclang==12.0.0
|
41 |
+
libtpu-nightly==0.1.dev20211018
|
42 |
+
markdown==3.3.4
|
43 |
+
matplotlib==3.4.3
|
44 |
+
msgpack==1.0.2
|
45 |
+
multidict==5.2.0
|
46 |
+
multiprocess==0.70.12.2
|
47 |
+
numpy==1.21.4
|
48 |
+
oauthlib==3.1.1
|
49 |
+
opt-einsum==3.3.0
|
50 |
+
optax==0.0.9
|
51 |
+
packaging==21.2
|
52 |
+
pandas==1.3.4
|
53 |
+
pathtools==0.1.2
|
54 |
+
pillow==8.4.0
|
55 |
+
pip==21.3.1
|
56 |
+
pkg-resources==0.0.0
|
57 |
+
promise==2.3
|
58 |
+
protobuf==3.19.1
|
59 |
+
psutil==5.8.0
|
60 |
+
pyarrow==6.0.0
|
61 |
+
pyasn1-modules==0.2.8
|
62 |
+
pyasn1==0.4.8
|
63 |
+
pyparsing==2.4.7
|
64 |
+
python-dateutil==2.8.2
|
65 |
+
python-slugify==5.0.2
|
66 |
+
pytz==2021.3
|
67 |
+
pyyaml==6.0
|
68 |
+
regex==2021.11.2
|
69 |
+
requests-oauthlib==1.3.0
|
70 |
+
requests==2.26.0
|
71 |
+
rsa==4.7.2
|
72 |
+
sacremoses==0.0.46
|
73 |
+
scipy==1.7.1
|
74 |
+
sentry-sdk==1.4.3
|
75 |
+
setuptools==44.0.0
|
76 |
+
shortuuid==1.0.1
|
77 |
+
six==1.16.0
|
78 |
+
smmap==5.0.0
|
79 |
+
subprocess32==3.5.4
|
80 |
+
tensorboard-data-server==0.6.1
|
81 |
+
tensorboard-plugin-wit==1.8.0
|
82 |
+
tensorboard==2.7.0
|
83 |
+
tensorflow-estimator==2.7.0
|
84 |
+
tensorflow-io-gcs-filesystem==0.21.0
|
85 |
+
tensorflow==2.7.0
|
86 |
+
termcolor==1.1.0
|
87 |
+
text-unidecode==1.3
|
88 |
+
tokenizers==0.10.3
|
89 |
+
toolz==0.11.1
|
90 |
+
torch==1.10.0
|
91 |
+
tqdm==4.62.3
|
92 |
+
transformers==4.13.0.dev0
|
93 |
+
typing-extensions==3.10.0.2
|
94 |
+
urllib3==1.26.7
|
95 |
+
wandb==0.12.6
|
96 |
+
werkzeug==2.0.2
|
97 |
+
wheel==0.37.0
|
98 |
+
wrapt==1.13.3
|
99 |
+
xxhash==2.0.2
|
100 |
+
yarl==1.7.2
|
101 |
+
yaspin==2.1.0
|
wandb/run-20211108_080537-1gq01aek/files/wandb-metadata.json
ADDED
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
|
3 |
+
"python": "3.8.10",
|
4 |
+
"heartbeatAt": "2021-11-08T08:05:40.298252",
|
5 |
+
"startedAt": "2021-11-08T08:05:37.281308",
|
6 |
+
"docker": null,
|
7 |
+
"cpu_count": 96,
|
8 |
+
"cuda": null,
|
9 |
+
"args": [
|
10 |
+
"--model_name_or_path",
|
11 |
+
"muril-bigbird",
|
12 |
+
"--tokenizer_name",
|
13 |
+
"muril-bigbird",
|
14 |
+
"--dtype",
|
15 |
+
"bfloat16",
|
16 |
+
"--pretokenized",
|
17 |
+
"--resume_from_checkpoint",
|
18 |
+
"checkpoint_100000",
|
19 |
+
"--train_file",
|
20 |
+
"/dev/shm/data-files",
|
21 |
+
"--validation_file",
|
22 |
+
".",
|
23 |
+
"--max_seq_length",
|
24 |
+
"4096",
|
25 |
+
"--pad_to_max_length",
|
26 |
+
"--output_dir",
|
27 |
+
"./",
|
28 |
+
"--overwrite_output_dir",
|
29 |
+
"--do_train",
|
30 |
+
"--do_eval",
|
31 |
+
"--eval_steps",
|
32 |
+
"5000",
|
33 |
+
"--per_device_train_batch_size",
|
34 |
+
"1",
|
35 |
+
"--per_device_eval_batch_size",
|
36 |
+
"2",
|
37 |
+
"--learning_rate",
|
38 |
+
"5e-5",
|
39 |
+
"--weight_decay",
|
40 |
+
"0.01",
|
41 |
+
"--num_train_epochs",
|
42 |
+
"3",
|
43 |
+
"--warmup_steps",
|
44 |
+
"0",
|
45 |
+
"--logging_steps",
|
46 |
+
"1000",
|
47 |
+
"--save_steps",
|
48 |
+
"25000",
|
49 |
+
"--save_total_limit",
|
50 |
+
"1",
|
51 |
+
"--seed",
|
52 |
+
"18",
|
53 |
+
"--push_to_hub",
|
54 |
+
"--run_name",
|
55 |
+
"en-ta-hi-te-from-100k"
|
56 |
+
],
|
57 |
+
"state": "running",
|
58 |
+
"program": "run_muril_bb_flax_mlm.py",
|
59 |
+
"codePath": "run_muril_bb_flax_mlm.py",
|
60 |
+
"git": {
|
61 |
+
"remote": "https://huggingface.co/nbroad/muril-bigbird-base-1mil",
|
62 |
+
"commit": "5fca06e3f8deb06ee5f938513b5f093fc3b66a62"
|
63 |
+
},
|
64 |
+
"email": "nicholas@nmbroad.com",
|
65 |
+
"root": "/home/nicholasbroad/muril-bigbird-base-1mil",
|
66 |
+
"host": "t1v-n-ba7aa1a5-w-0",
|
67 |
+
"username": "nicholasbroad",
|
68 |
+
"executable": "/home/nicholasbroad/venv/bin/python"
|
69 |
+
}
|
wandb/run-20211108_080537-1gq01aek/files/wandb-summary.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"_wandb": {"runtime": 86}}
|
wandb/run-20211108_080537-1gq01aek/logs/debug-internal.log
ADDED
@@ -0,0 +1,159 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2021-11-08 08:05:37,994 INFO MainThread:5221 [internal.py:wandb_internal():87] W&B internal server running at pid: 5221, started at: 2021-11-08 08:05:37.993832
|
2 |
+
2021-11-08 08:05:37,995 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: check_version
|
3 |
+
2021-11-08 08:05:37,995 INFO WriterThread:5221 [datastore.py:open_for_write():77] open: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/run-1gq01aek.wandb
|
4 |
+
2021-11-08 08:05:37,997 DEBUG SenderThread:5221 [sender.py:send():238] send: header
|
5 |
+
2021-11-08 08:05:37,997 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: check_version
|
6 |
+
2021-11-08 08:05:38,035 DEBUG SenderThread:5221 [sender.py:send():238] send: run
|
7 |
+
2021-11-08 08:05:38,200 INFO SenderThread:5221 [dir_watcher.py:__init__():169] watching files in: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files
|
8 |
+
2021-11-08 08:05:38,200 INFO SenderThread:5221 [sender.py:_start_run_threads():825] run started: 1gq01aek with start time 1636358737
|
9 |
+
2021-11-08 08:05:38,200 DEBUG SenderThread:5221 [sender.py:send():238] send: summary
|
10 |
+
2021-11-08 08:05:38,200 INFO SenderThread:5221 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
|
11 |
+
2021-11-08 08:05:38,201 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: run_start
|
12 |
+
2021-11-08 08:05:39,203 INFO Thread-8 :5221 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/wandb-summary.json
|
13 |
+
2021-11-08 08:05:40,298 DEBUG HandlerThread:5221 [meta.py:__init__():40] meta init
|
14 |
+
2021-11-08 08:05:40,298 DEBUG HandlerThread:5221 [meta.py:__init__():54] meta init done
|
15 |
+
2021-11-08 08:05:40,298 DEBUG HandlerThread:5221 [meta.py:probe():214] probe
|
16 |
+
2021-11-08 08:05:40,299 DEBUG HandlerThread:5221 [meta.py:_setup_git():204] setup git
|
17 |
+
2021-11-08 08:05:40,327 DEBUG HandlerThread:5221 [meta.py:_setup_git():211] setup git done
|
18 |
+
2021-11-08 08:05:40,328 DEBUG HandlerThread:5221 [meta.py:_save_pip():58] save pip
|
19 |
+
2021-11-08 08:05:40,328 DEBUG HandlerThread:5221 [meta.py:_save_pip():72] save pip done
|
20 |
+
2021-11-08 08:05:40,328 DEBUG HandlerThread:5221 [meta.py:probe():252] probe done
|
21 |
+
2021-11-08 08:05:40,331 DEBUG SenderThread:5221 [sender.py:send():238] send: files
|
22 |
+
2021-11-08 08:05:40,331 INFO SenderThread:5221 [sender.py:_save_file():960] saving file wandb-metadata.json with policy now
|
23 |
+
2021-11-08 08:05:40,337 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: stop_status
|
24 |
+
2021-11-08 08:05:40,337 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: stop_status
|
25 |
+
2021-11-08 08:05:40,465 DEBUG SenderThread:5221 [sender.py:send():238] send: config
|
26 |
+
2021-11-08 08:05:40,465 DEBUG SenderThread:5221 [sender.py:send():238] send: config
|
27 |
+
2021-11-08 08:05:40,465 DEBUG SenderThread:5221 [sender.py:send():238] send: config
|
28 |
+
2021-11-08 08:05:40,880 INFO Thread-11 :5221 [upload_job.py:push():137] Uploaded file /tmp/tmpzyu3dzxewandb/1hif9l1i-wandb-metadata.json
|
29 |
+
2021-11-08 08:05:41,208 INFO Thread-8 :5221 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/wandb-metadata.json
|
30 |
+
2021-11-08 08:05:41,208 INFO Thread-8 :5221 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/output.log
|
31 |
+
2021-11-08 08:05:41,208 INFO Thread-8 :5221 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/requirements.txt
|
32 |
+
2021-11-08 08:05:55,467 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: stop_status
|
33 |
+
2021-11-08 08:05:55,468 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: stop_status
|
34 |
+
2021-11-08 08:06:07,218 INFO Thread-8 :5221 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/output.log
|
35 |
+
2021-11-08 08:06:08,375 DEBUG SenderThread:5221 [sender.py:send():238] send: stats
|
36 |
+
2021-11-08 08:06:09,219 INFO Thread-8 :5221 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/output.log
|
37 |
+
2021-11-08 08:06:09,484 INFO Thread-8 :5221 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/config.yaml
|
38 |
+
2021-11-08 08:06:10,601 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: stop_status
|
39 |
+
2021-11-08 08:06:11,015 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: stop_status
|
40 |
+
2021-11-08 08:06:26,152 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: stop_status
|
41 |
+
2021-11-08 08:06:26,152 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: stop_status
|
42 |
+
2021-11-08 08:06:38,443 DEBUG SenderThread:5221 [sender.py:send():238] send: stats
|
43 |
+
2021-11-08 08:06:41,289 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: stop_status
|
44 |
+
2021-11-08 08:06:41,289 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: stop_status
|
45 |
+
2021-11-08 08:06:56,424 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: stop_status
|
46 |
+
2021-11-08 08:07:01,107 WARNING MainThread:5221 [internal.py:wandb_internal():146] Internal process interrupt: 1
|
47 |
+
2021-11-08 08:07:01,624 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: stop_status
|
48 |
+
2021-11-08 08:07:04,878 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
|
49 |
+
2021-11-08 08:07:04,878 DEBUG SenderThread:5221 [sender.py:send():238] send: telemetry
|
50 |
+
2021-11-08 08:07:04,878 DEBUG SenderThread:5221 [sender.py:send():238] send: exit
|
51 |
+
2021-11-08 08:07:04,878 INFO SenderThread:5221 [sender.py:send_exit():370] handling exit code: 255
|
52 |
+
2021-11-08 08:07:04,879 INFO SenderThread:5221 [sender.py:send_exit():372] handling runtime: 86
|
53 |
+
2021-11-08 08:07:05,244 INFO Thread-8 :5221 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/wandb-summary.json
|
54 |
+
2021-11-08 08:07:05,244 INFO Thread-8 :5221 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/output.log
|
55 |
+
2021-11-08 08:07:06,223 INFO SenderThread:5221 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
|
56 |
+
2021-11-08 08:07:07,292 INFO SenderThread:5221 [sender.py:send_exit():382] send defer
|
57 |
+
2021-11-08 08:07:07,292 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
|
58 |
+
2021-11-08 08:07:07,293 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
|
59 |
+
2021-11-08 08:07:07,294 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 0
|
60 |
+
2021-11-08 08:07:07,294 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
|
61 |
+
2021-11-08 08:07:07,294 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 0
|
62 |
+
2021-11-08 08:07:07,294 INFO SenderThread:5221 [sender.py:transition_state():395] send defer: 1
|
63 |
+
2021-11-08 08:07:07,294 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
|
64 |
+
2021-11-08 08:07:07,294 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 1
|
65 |
+
2021-11-08 08:07:07,312 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
|
66 |
+
2021-11-08 08:07:07,312 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 1
|
67 |
+
2021-11-08 08:07:07,312 INFO SenderThread:5221 [sender.py:transition_state():395] send defer: 2
|
68 |
+
2021-11-08 08:07:07,312 DEBUG SenderThread:5221 [sender.py:send():238] send: stats
|
69 |
+
2021-11-08 08:07:07,312 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
|
70 |
+
2021-11-08 08:07:07,312 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 2
|
71 |
+
2021-11-08 08:07:07,313 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
|
72 |
+
2021-11-08 08:07:07,313 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 2
|
73 |
+
2021-11-08 08:07:07,313 INFO SenderThread:5221 [sender.py:transition_state():395] send defer: 3
|
74 |
+
2021-11-08 08:07:07,313 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
|
75 |
+
2021-11-08 08:07:07,313 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 3
|
76 |
+
2021-11-08 08:07:07,313 DEBUG SenderThread:5221 [sender.py:send():238] send: summary
|
77 |
+
2021-11-08 08:07:07,396 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
|
78 |
+
2021-11-08 08:07:08,008 INFO SenderThread:5221 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
|
79 |
+
2021-11-08 08:07:08,008 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
|
80 |
+
2021-11-08 08:07:08,009 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 3
|
81 |
+
2021-11-08 08:07:08,009 INFO SenderThread:5221 [sender.py:transition_state():395] send defer: 4
|
82 |
+
2021-11-08 08:07:08,009 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
|
83 |
+
2021-11-08 08:07:08,009 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
|
84 |
+
2021-11-08 08:07:08,010 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 4
|
85 |
+
2021-11-08 08:07:08,010 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
|
86 |
+
2021-11-08 08:07:08,010 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 4
|
87 |
+
2021-11-08 08:07:08,111 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
|
88 |
+
2021-11-08 08:07:08,245 INFO Thread-8 :5221 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/wandb-summary.json
|
89 |
+
2021-11-08 08:07:08,245 INFO Thread-8 :5221 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/config.yaml
|
90 |
+
2021-11-08 08:07:09,067 INFO SenderThread:5221 [sender.py:transition_state():395] send defer: 5
|
91 |
+
2021-11-08 08:07:10,121 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
|
92 |
+
2021-11-08 08:07:10,122 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
|
93 |
+
2021-11-08 08:07:10,122 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 5
|
94 |
+
2021-11-08 08:07:10,123 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
|
95 |
+
2021-11-08 08:07:10,123 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 5
|
96 |
+
2021-11-08 08:07:10,123 INFO SenderThread:5221 [dir_watcher.py:finish():283] shutting down directory watcher
|
97 |
+
2021-11-08 08:07:10,224 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
|
98 |
+
2021-11-08 08:07:10,246 INFO SenderThread:5221 [dir_watcher.py:finish():313] scan: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files
|
99 |
+
2021-11-08 08:07:10,247 INFO SenderThread:5221 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/requirements.txt requirements.txt
|
100 |
+
2021-11-08 08:07:10,247 INFO SenderThread:5221 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/output.log output.log
|
101 |
+
2021-11-08 08:07:10,247 INFO SenderThread:5221 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/wandb-metadata.json wandb-metadata.json
|
102 |
+
2021-11-08 08:07:10,247 INFO SenderThread:5221 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/config.yaml config.yaml
|
103 |
+
2021-11-08 08:07:10,248 INFO SenderThread:5221 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/wandb-summary.json wandb-summary.json
|
104 |
+
2021-11-08 08:07:10,248 INFO SenderThread:5221 [sender.py:transition_state():395] send defer: 6
|
105 |
+
2021-11-08 08:07:10,248 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
|
106 |
+
2021-11-08 08:07:10,252 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
|
107 |
+
2021-11-08 08:07:10,252 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 6
|
108 |
+
2021-11-08 08:07:10,255 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
|
109 |
+
2021-11-08 08:07:10,255 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 6
|
110 |
+
2021-11-08 08:07:10,255 INFO SenderThread:5221 [file_pusher.py:finish():177] shutting down file pusher
|
111 |
+
2021-11-08 08:07:10,360 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
|
112 |
+
2021-11-08 08:07:10,360 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
|
113 |
+
2021-11-08 08:07:10,462 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
|
114 |
+
2021-11-08 08:07:10,462 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
|
115 |
+
2021-11-08 08:07:10,563 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
|
116 |
+
2021-11-08 08:07:10,564 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
|
117 |
+
2021-11-08 08:07:10,665 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
|
118 |
+
2021-11-08 08:07:10,665 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
|
119 |
+
2021-11-08 08:07:10,751 INFO Thread-14 :5221 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/config.yaml
|
120 |
+
2021-11-08 08:07:10,756 INFO Thread-12 :5221 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/requirements.txt
|
121 |
+
2021-11-08 08:07:10,767 INFO Thread-13 :5221 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/output.log
|
122 |
+
2021-11-08 08:07:10,768 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
|
123 |
+
2021-11-08 08:07:10,768 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
|
124 |
+
2021-11-08 08:07:10,788 INFO Thread-15 :5221 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/files/wandb-summary.json
|
125 |
+
2021-11-08 08:07:10,869 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
|
126 |
+
2021-11-08 08:07:10,870 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
|
127 |
+
2021-11-08 08:07:10,971 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
|
128 |
+
2021-11-08 08:07:10,971 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
|
129 |
+
2021-11-08 08:07:10,988 INFO Thread-7 :5221 [sender.py:transition_state():395] send defer: 7
|
130 |
+
2021-11-08 08:07:10,989 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
|
131 |
+
2021-11-08 08:07:10,989 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 7
|
132 |
+
2021-11-08 08:07:10,989 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
|
133 |
+
2021-11-08 08:07:10,989 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 7
|
134 |
+
2021-11-08 08:07:11,073 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
|
135 |
+
2021-11-08 08:07:11,286 INFO SenderThread:5221 [sender.py:transition_state():395] send defer: 8
|
136 |
+
2021-11-08 08:07:11,286 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
|
137 |
+
2021-11-08 08:07:11,287 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
|
138 |
+
2021-11-08 08:07:11,287 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 8
|
139 |
+
2021-11-08 08:07:11,287 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
|
140 |
+
2021-11-08 08:07:11,287 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 8
|
141 |
+
2021-11-08 08:07:11,287 INFO SenderThread:5221 [sender.py:transition_state():395] send defer: 9
|
142 |
+
2021-11-08 08:07:11,288 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: defer
|
143 |
+
2021-11-08 08:07:11,288 INFO HandlerThread:5221 [handler.py:handle_request_defer():147] handle defer: 9
|
144 |
+
2021-11-08 08:07:11,288 DEBUG SenderThread:5221 [sender.py:send():238] send: final
|
145 |
+
2021-11-08 08:07:11,288 DEBUG SenderThread:5221 [sender.py:send():238] send: footer
|
146 |
+
2021-11-08 08:07:11,288 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: defer
|
147 |
+
2021-11-08 08:07:11,288 INFO SenderThread:5221 [sender.py:send_request_defer():391] handle sender defer: 9
|
148 |
+
2021-11-08 08:07:11,388 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: poll_exit
|
149 |
+
2021-11-08 08:07:11,388 DEBUG SenderThread:5221 [sender.py:send_request():252] send_request: poll_exit
|
150 |
+
2021-11-08 08:07:11,389 INFO SenderThread:5221 [file_pusher.py:join():182] waiting for file pusher
|
151 |
+
2021-11-08 08:07:12,169 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: get_summary
|
152 |
+
2021-11-08 08:07:12,170 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: sampled_history
|
153 |
+
2021-11-08 08:07:12,170 DEBUG HandlerThread:5221 [handler.py:handle_request():130] handle_request: shutdown
|
154 |
+
2021-11-08 08:07:12,171 INFO HandlerThread:5221 [handler.py:finish():731] shutting down handler
|
155 |
+
2021-11-08 08:07:12,288 INFO WriterThread:5221 [datastore.py:close():281] close: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/run-1gq01aek.wandb
|
156 |
+
2021-11-08 08:07:12,652 INFO SenderThread:5221 [sender.py:finish():1091] shutting down sender
|
157 |
+
2021-11-08 08:07:12,652 INFO SenderThread:5221 [file_pusher.py:finish():177] shutting down file pusher
|
158 |
+
2021-11-08 08:07:12,652 INFO SenderThread:5221 [file_pusher.py:join():182] waiting for file pusher
|
159 |
+
2021-11-08 08:07:12,654 INFO MainThread:5221 [internal.py:handle_exit():77] Internal process exited
|
wandb/run-20211108_080537-1gq01aek/logs/debug.log
ADDED
@@ -0,0 +1,137 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2021-11-08 08:05:37,301 INFO MainThread:3899 [wandb_setup.py:_flush():71] setting env: {}
|
2 |
+
2021-11-08 08:05:37,301 INFO MainThread:3899 [wandb_setup.py:_flush():71] setting login settings: {}
|
3 |
+
2021-11-08 08:05:37,301 INFO MainThread:3899 [wandb_init.py:_log_setup():357] Logging user logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/logs/debug.log
|
4 |
+
2021-11-08 08:05:37,301 INFO MainThread:3899 [wandb_init.py:_log_setup():358] Logging internal logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080537-1gq01aek/logs/debug-internal.log
|
5 |
+
2021-11-08 08:05:37,301 INFO MainThread:3899 [wandb_init.py:init():390] calling init triggers
|
6 |
+
2021-11-08 08:05:37,301 INFO MainThread:3899 [wandb_init.py:init():395] wandb.init called with sweep_config: {}
|
7 |
+
config: {}
|
8 |
+
2021-11-08 08:05:37,301 INFO MainThread:3899 [wandb_init.py:init():435] starting backend
|
9 |
+
2021-11-08 08:05:37,301 INFO MainThread:3899 [backend.py:_multiprocessing_setup():95] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
|
10 |
+
2021-11-08 08:05:37,369 INFO MainThread:3899 [backend.py:ensure_launched():199] starting backend process...
|
11 |
+
2021-11-08 08:05:37,422 INFO MainThread:3899 [backend.py:ensure_launched():204] started backend process with pid: 5221
|
12 |
+
2021-11-08 08:05:37,424 INFO MainThread:3899 [wandb_init.py:init():444] backend started and connected
|
13 |
+
2021-11-08 08:05:37,427 INFO MainThread:3899 [wandb_init.py:init():503] updated telemetry
|
14 |
+
2021-11-08 08:05:37,515 INFO MainThread:3899 [wandb_init.py:init():533] communicating current version
|
15 |
+
2021-11-08 08:05:38,033 INFO MainThread:3899 [wandb_init.py:init():538] got version response
|
16 |
+
2021-11-08 08:05:38,034 INFO MainThread:3899 [wandb_init.py:init():548] communicating run to backend with 30 second timeout
|
17 |
+
2021-11-08 08:05:38,200 INFO MainThread:3899 [wandb_init.py:init():576] starting run threads in backend
|
18 |
+
2021-11-08 08:05:40,336 INFO MainThread:3899 [wandb_run.py:_console_start():1693] atexit reg
|
19 |
+
2021-11-08 08:05:40,337 INFO MainThread:3899 [wandb_run.py:_redirect():1567] redirect: SettingsConsole.REDIRECT
|
20 |
+
2021-11-08 08:05:40,337 INFO MainThread:3899 [wandb_run.py:_redirect():1572] Redirecting console.
|
21 |
+
2021-11-08 08:05:40,339 INFO MainThread:3899 [wandb_run.py:_redirect():1628] Redirects installed.
|
22 |
+
2021-11-08 08:05:40,339 INFO MainThread:3899 [wandb_init.py:init():603] run started, returning control to user process
|
23 |
+
2021-11-08 08:05:40,345 INFO MainThread:3899 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov08_08-04-38_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 1000, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 25000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te-from-100k', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'checkpoint_100000', 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
|
24 |
+
2021-11-08 08:05:40,347 INFO MainThread:3899 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
|
25 |
+
2021-11-08 08:05:40,348 INFO MainThread:3899 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}
|
26 |
+
2021-11-08 08:07:02,826 INFO MainThread:3899 [wandb_run.py:_atexit_cleanup():1663] got exitcode: 255
|
27 |
+
2021-11-08 08:07:02,826 INFO MainThread:3899 [wandb_run.py:_restore():1635] restore
|
28 |
+
2021-11-08 08:07:07,294 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
29 |
+
wandb_count: 1
|
30 |
+
}
|
31 |
+
pusher_stats {
|
32 |
+
uploaded_bytes: 1825
|
33 |
+
total_bytes: 1825
|
34 |
+
}
|
35 |
+
|
36 |
+
2021-11-08 08:07:08,010 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
37 |
+
wandb_count: 1
|
38 |
+
}
|
39 |
+
pusher_stats {
|
40 |
+
uploaded_bytes: 1825
|
41 |
+
total_bytes: 1825
|
42 |
+
}
|
43 |
+
|
44 |
+
2021-11-08 08:07:10,122 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
45 |
+
wandb_count: 1
|
46 |
+
}
|
47 |
+
pusher_stats {
|
48 |
+
uploaded_bytes: 1825
|
49 |
+
total_bytes: 1825
|
50 |
+
}
|
51 |
+
|
52 |
+
2021-11-08 08:07:10,259 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
53 |
+
wandb_count: 3
|
54 |
+
}
|
55 |
+
pusher_stats {
|
56 |
+
uploaded_bytes: 1825
|
57 |
+
total_bytes: 4717
|
58 |
+
}
|
59 |
+
|
60 |
+
2021-11-08 08:07:10,361 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
61 |
+
wandb_count: 5
|
62 |
+
}
|
63 |
+
pusher_stats {
|
64 |
+
uploaded_bytes: 1825
|
65 |
+
total_bytes: 9972
|
66 |
+
}
|
67 |
+
|
68 |
+
2021-11-08 08:07:10,462 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
69 |
+
wandb_count: 5
|
70 |
+
}
|
71 |
+
pusher_stats {
|
72 |
+
uploaded_bytes: 9972
|
73 |
+
total_bytes: 9972
|
74 |
+
}
|
75 |
+
|
76 |
+
2021-11-08 08:07:10,564 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
77 |
+
wandb_count: 5
|
78 |
+
}
|
79 |
+
pusher_stats {
|
80 |
+
uploaded_bytes: 9972
|
81 |
+
total_bytes: 9972
|
82 |
+
}
|
83 |
+
|
84 |
+
2021-11-08 08:07:10,666 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
85 |
+
wandb_count: 5
|
86 |
+
}
|
87 |
+
pusher_stats {
|
88 |
+
uploaded_bytes: 9972
|
89 |
+
total_bytes: 9972
|
90 |
+
}
|
91 |
+
|
92 |
+
2021-11-08 08:07:10,769 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
93 |
+
wandb_count: 5
|
94 |
+
}
|
95 |
+
pusher_stats {
|
96 |
+
uploaded_bytes: 9972
|
97 |
+
total_bytes: 9972
|
98 |
+
}
|
99 |
+
|
100 |
+
2021-11-08 08:07:10,870 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
101 |
+
wandb_count: 5
|
102 |
+
}
|
103 |
+
pusher_stats {
|
104 |
+
uploaded_bytes: 9972
|
105 |
+
total_bytes: 9972
|
106 |
+
}
|
107 |
+
|
108 |
+
2021-11-08 08:07:10,972 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
109 |
+
wandb_count: 5
|
110 |
+
}
|
111 |
+
pusher_stats {
|
112 |
+
uploaded_bytes: 9972
|
113 |
+
total_bytes: 9972
|
114 |
+
}
|
115 |
+
|
116 |
+
2021-11-08 08:07:11,287 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
117 |
+
wandb_count: 5
|
118 |
+
}
|
119 |
+
pusher_stats {
|
120 |
+
uploaded_bytes: 9972
|
121 |
+
total_bytes: 9972
|
122 |
+
}
|
123 |
+
|
124 |
+
2021-11-08 08:07:11,652 INFO MainThread:3899 [wandb_run.py:_wait_for_finish():1793] got exit ret: done: true
|
125 |
+
exit_result {
|
126 |
+
}
|
127 |
+
file_counts {
|
128 |
+
wandb_count: 5
|
129 |
+
}
|
130 |
+
pusher_stats {
|
131 |
+
uploaded_bytes: 9972
|
132 |
+
total_bytes: 9972
|
133 |
+
}
|
134 |
+
local_info {
|
135 |
+
}
|
136 |
+
|
137 |
+
2021-11-08 08:07:33,406 INFO MainThread:3899 [wandb_run.py:_append_files():2061] logging synced files
|
wandb/run-20211108_080537-1gq01aek/run-1gq01aek.wandb
ADDED
Binary file (5.36 kB). View file
|
|
wandb/run-20211108_080817-3dz8poa1/files/config.yaml
ADDED
@@ -0,0 +1,347 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
wandb_version: 1
|
2 |
+
|
3 |
+
__cached__setup_devices:
|
4 |
+
desc: null
|
5 |
+
value: cpu
|
6 |
+
_n_gpu:
|
7 |
+
desc: null
|
8 |
+
value: 0
|
9 |
+
_wandb:
|
10 |
+
desc: null
|
11 |
+
value:
|
12 |
+
cli_version: 0.12.6
|
13 |
+
framework: huggingface
|
14 |
+
huggingface_version: 4.13.0.dev0
|
15 |
+
is_jupyter_run: false
|
16 |
+
is_kaggle_kernel: true
|
17 |
+
python_version: 3.8.10
|
18 |
+
start_time: 1636358897
|
19 |
+
t:
|
20 |
+
1:
|
21 |
+
- 1
|
22 |
+
- 2
|
23 |
+
- 3
|
24 |
+
- 11
|
25 |
+
- 12
|
26 |
+
2:
|
27 |
+
- 1
|
28 |
+
- 2
|
29 |
+
- 3
|
30 |
+
- 11
|
31 |
+
- 12
|
32 |
+
3:
|
33 |
+
- 13
|
34 |
+
4: 3.8.10
|
35 |
+
5: 0.12.6
|
36 |
+
6: 4.13.0.dev0
|
37 |
+
8:
|
38 |
+
- 2
|
39 |
+
- 5
|
40 |
+
adafactor:
|
41 |
+
desc: null
|
42 |
+
value: false
|
43 |
+
adam_beta1:
|
44 |
+
desc: null
|
45 |
+
value: 0.9
|
46 |
+
adam_beta2:
|
47 |
+
desc: null
|
48 |
+
value: 0.999
|
49 |
+
adam_epsilon:
|
50 |
+
desc: null
|
51 |
+
value: 1.0e-08
|
52 |
+
cache_dir:
|
53 |
+
desc: null
|
54 |
+
value: null
|
55 |
+
config_name:
|
56 |
+
desc: null
|
57 |
+
value: null
|
58 |
+
dataloader_drop_last:
|
59 |
+
desc: null
|
60 |
+
value: false
|
61 |
+
dataloader_num_workers:
|
62 |
+
desc: null
|
63 |
+
value: 0
|
64 |
+
dataloader_pin_memory:
|
65 |
+
desc: null
|
66 |
+
value: true
|
67 |
+
dataset_config_name:
|
68 |
+
desc: null
|
69 |
+
value: null
|
70 |
+
dataset_name:
|
71 |
+
desc: null
|
72 |
+
value: null
|
73 |
+
ddp_find_unused_parameters:
|
74 |
+
desc: null
|
75 |
+
value: null
|
76 |
+
debug:
|
77 |
+
desc: null
|
78 |
+
value: []
|
79 |
+
deepspeed:
|
80 |
+
desc: null
|
81 |
+
value: null
|
82 |
+
disable_tqdm:
|
83 |
+
desc: null
|
84 |
+
value: false
|
85 |
+
do_eval:
|
86 |
+
desc: null
|
87 |
+
value: true
|
88 |
+
do_predict:
|
89 |
+
desc: null
|
90 |
+
value: false
|
91 |
+
do_train:
|
92 |
+
desc: null
|
93 |
+
value: true
|
94 |
+
dtype:
|
95 |
+
desc: null
|
96 |
+
value: bfloat16
|
97 |
+
eval_accumulation_steps:
|
98 |
+
desc: null
|
99 |
+
value: null
|
100 |
+
eval_steps:
|
101 |
+
desc: null
|
102 |
+
value: 5000
|
103 |
+
evaluation_strategy:
|
104 |
+
desc: null
|
105 |
+
value: IntervalStrategy.NO
|
106 |
+
fp16:
|
107 |
+
desc: null
|
108 |
+
value: false
|
109 |
+
fp16_backend:
|
110 |
+
desc: null
|
111 |
+
value: auto
|
112 |
+
fp16_full_eval:
|
113 |
+
desc: null
|
114 |
+
value: false
|
115 |
+
fp16_opt_level:
|
116 |
+
desc: null
|
117 |
+
value: O1
|
118 |
+
gradient_accumulation_steps:
|
119 |
+
desc: null
|
120 |
+
value: 1
|
121 |
+
gradient_checkpointing:
|
122 |
+
desc: null
|
123 |
+
value: false
|
124 |
+
greater_is_better:
|
125 |
+
desc: null
|
126 |
+
value: null
|
127 |
+
group_by_length:
|
128 |
+
desc: null
|
129 |
+
value: false
|
130 |
+
hub_model_id:
|
131 |
+
desc: null
|
132 |
+
value: null
|
133 |
+
hub_strategy:
|
134 |
+
desc: null
|
135 |
+
value: HubStrategy.EVERY_SAVE
|
136 |
+
hub_token:
|
137 |
+
desc: null
|
138 |
+
value: null
|
139 |
+
ignore_data_skip:
|
140 |
+
desc: null
|
141 |
+
value: false
|
142 |
+
label_names:
|
143 |
+
desc: null
|
144 |
+
value: null
|
145 |
+
label_smoothing_factor:
|
146 |
+
desc: null
|
147 |
+
value: 0.0
|
148 |
+
learning_rate:
|
149 |
+
desc: null
|
150 |
+
value: 5.0e-05
|
151 |
+
length_column_name:
|
152 |
+
desc: null
|
153 |
+
value: length
|
154 |
+
line_by_line:
|
155 |
+
desc: null
|
156 |
+
value: false
|
157 |
+
load_best_model_at_end:
|
158 |
+
desc: null
|
159 |
+
value: false
|
160 |
+
local_rank:
|
161 |
+
desc: null
|
162 |
+
value: -1
|
163 |
+
log_level:
|
164 |
+
desc: null
|
165 |
+
value: -1
|
166 |
+
log_level_replica:
|
167 |
+
desc: null
|
168 |
+
value: -1
|
169 |
+
log_on_each_node:
|
170 |
+
desc: null
|
171 |
+
value: true
|
172 |
+
logging_dir:
|
173 |
+
desc: null
|
174 |
+
value: ./runs/Nov08_08-08-10_t1v-n-ba7aa1a5-w-0
|
175 |
+
logging_first_step:
|
176 |
+
desc: null
|
177 |
+
value: false
|
178 |
+
logging_nan_inf_filter:
|
179 |
+
desc: null
|
180 |
+
value: true
|
181 |
+
logging_steps:
|
182 |
+
desc: null
|
183 |
+
value: 1000
|
184 |
+
logging_strategy:
|
185 |
+
desc: null
|
186 |
+
value: IntervalStrategy.STEPS
|
187 |
+
lr_scheduler_type:
|
188 |
+
desc: null
|
189 |
+
value: SchedulerType.LINEAR
|
190 |
+
max_grad_norm:
|
191 |
+
desc: null
|
192 |
+
value: 1.0
|
193 |
+
max_seq_length:
|
194 |
+
desc: null
|
195 |
+
value: 4096
|
196 |
+
max_steps:
|
197 |
+
desc: null
|
198 |
+
value: -1
|
199 |
+
metric_for_best_model:
|
200 |
+
desc: null
|
201 |
+
value: null
|
202 |
+
mlm_probability:
|
203 |
+
desc: null
|
204 |
+
value: 0.15
|
205 |
+
model_name_or_path:
|
206 |
+
desc: null
|
207 |
+
value: muril-bigbird
|
208 |
+
model_type:
|
209 |
+
desc: null
|
210 |
+
value: null
|
211 |
+
mp_parameters:
|
212 |
+
desc: null
|
213 |
+
value: ''
|
214 |
+
no_cuda:
|
215 |
+
desc: null
|
216 |
+
value: false
|
217 |
+
num_train_epochs:
|
218 |
+
desc: null
|
219 |
+
value: 3.0
|
220 |
+
output_dir:
|
221 |
+
desc: null
|
222 |
+
value: ./
|
223 |
+
overwrite_cache:
|
224 |
+
desc: null
|
225 |
+
value: false
|
226 |
+
overwrite_output_dir:
|
227 |
+
desc: null
|
228 |
+
value: true
|
229 |
+
pad_to_max_length:
|
230 |
+
desc: null
|
231 |
+
value: true
|
232 |
+
past_index:
|
233 |
+
desc: null
|
234 |
+
value: -1
|
235 |
+
per_device_eval_batch_size:
|
236 |
+
desc: null
|
237 |
+
value: 2
|
238 |
+
per_device_train_batch_size:
|
239 |
+
desc: null
|
240 |
+
value: 1
|
241 |
+
per_gpu_eval_batch_size:
|
242 |
+
desc: null
|
243 |
+
value: null
|
244 |
+
per_gpu_train_batch_size:
|
245 |
+
desc: null
|
246 |
+
value: null
|
247 |
+
prediction_loss_only:
|
248 |
+
desc: null
|
249 |
+
value: false
|
250 |
+
preprocessing_num_workers:
|
251 |
+
desc: null
|
252 |
+
value: null
|
253 |
+
pretokenized:
|
254 |
+
desc: null
|
255 |
+
value: true
|
256 |
+
push_to_hub:
|
257 |
+
desc: null
|
258 |
+
value: true
|
259 |
+
push_to_hub_model_id:
|
260 |
+
desc: null
|
261 |
+
value: null
|
262 |
+
push_to_hub_organization:
|
263 |
+
desc: null
|
264 |
+
value: null
|
265 |
+
push_to_hub_token:
|
266 |
+
desc: null
|
267 |
+
value: null
|
268 |
+
remove_unused_columns:
|
269 |
+
desc: null
|
270 |
+
value: true
|
271 |
+
report_to:
|
272 |
+
desc: null
|
273 |
+
value:
|
274 |
+
- tensorboard
|
275 |
+
- wandb
|
276 |
+
resume_from_checkpoint:
|
277 |
+
desc: null
|
278 |
+
value: checkpoint_100000
|
279 |
+
run_name:
|
280 |
+
desc: null
|
281 |
+
value: en-ta-hi-te-from-100k
|
282 |
+
save_on_each_node:
|
283 |
+
desc: null
|
284 |
+
value: false
|
285 |
+
save_steps:
|
286 |
+
desc: null
|
287 |
+
value: 25000
|
288 |
+
save_strategy:
|
289 |
+
desc: null
|
290 |
+
value: IntervalStrategy.STEPS
|
291 |
+
save_total_limit:
|
292 |
+
desc: null
|
293 |
+
value: 1
|
294 |
+
seed:
|
295 |
+
desc: null
|
296 |
+
value: 18
|
297 |
+
sharded_ddp:
|
298 |
+
desc: null
|
299 |
+
value: []
|
300 |
+
skip_memory_metrics:
|
301 |
+
desc: null
|
302 |
+
value: true
|
303 |
+
streaming:
|
304 |
+
desc: null
|
305 |
+
value: false
|
306 |
+
tokenizer_name:
|
307 |
+
desc: null
|
308 |
+
value: muril-bigbird
|
309 |
+
tpu_metrics_debug:
|
310 |
+
desc: null
|
311 |
+
value: false
|
312 |
+
tpu_num_cores:
|
313 |
+
desc: null
|
314 |
+
value: null
|
315 |
+
train_file:
|
316 |
+
desc: null
|
317 |
+
value: /dev/shm/data-files
|
318 |
+
train_ref_file:
|
319 |
+
desc: null
|
320 |
+
value: null
|
321 |
+
use_fast_tokenizer:
|
322 |
+
desc: null
|
323 |
+
value: true
|
324 |
+
use_legacy_prediction_loop:
|
325 |
+
desc: null
|
326 |
+
value: false
|
327 |
+
validation_file:
|
328 |
+
desc: null
|
329 |
+
value: .
|
330 |
+
validation_ref_file:
|
331 |
+
desc: null
|
332 |
+
value: null
|
333 |
+
validation_split_percentage:
|
334 |
+
desc: null
|
335 |
+
value: 5
|
336 |
+
warmup_ratio:
|
337 |
+
desc: null
|
338 |
+
value: 0.0
|
339 |
+
warmup_steps:
|
340 |
+
desc: null
|
341 |
+
value: 0
|
342 |
+
weight_decay:
|
343 |
+
desc: null
|
344 |
+
value: 0.01
|
345 |
+
xpu_backend:
|
346 |
+
desc: null
|
347 |
+
value: null
|
wandb/run-20211108_080817-3dz8poa1/files/output.log
ADDED
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
Traceback (most recent call last):
|
2 |
+
File "run_muril_bb_flax_mlm.py", line 548, in <module>
|
3 |
+
model = FlaxAutoModelForMaskedLM.from_pretrained(
|
4 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/auto/auto_factory.py", line 419, in from_pretrained
|
5 |
+
return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
|
6 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/modeling_flax_utils.py", line 351, in from_pretrained
|
7 |
+
model = cls(config, *model_args, **model_kwargs)
|
8 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1359, in __init__
|
9 |
+
super().__init__(config, module, input_shape=input_shape, seed=seed, dtype=dtype)
|
10 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/modeling_flax_utils.py", line 106, in __init__
|
11 |
+
random_params = self.init_weights(self.key, input_shape)
|
12 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1371, in init_weights
|
13 |
+
return self.module.init(rngs, input_ids, attention_mask, token_type_ids, position_ids, return_dict=False)[
|
14 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback
|
15 |
+
return fun(*args, **kwargs)
|
16 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 1122, in init
|
17 |
+
_, v_out = self.init_with_output(
|
18 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback
|
19 |
+
return fun(*args, **kwargs)
|
20 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 1091, in init_with_output
|
21 |
+
return self.apply(
|
22 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback
|
23 |
+
return fun(*args, **kwargs)
|
24 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 1058, in apply
|
25 |
+
return apply(
|
26 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/core/scope.py", line 706, in wrapper
|
27 |
+
y = fn(root, *args, **kwargs)
|
28 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 1313, in scope_fn
|
29 |
+
return fn(module.clone(parent=scope), *args, **kwargs)
|
30 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
|
31 |
+
return prewrapped_fn(self, *args, **kwargs)
|
32 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
|
33 |
+
return self._call_wrapped_method(fun, args, kwargs)
|
34 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
|
35 |
+
y = fun(self, *args, **kwargs)
|
36 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1607, in __call__
|
37 |
+
outputs = self.bert(
|
38 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
|
39 |
+
return prewrapped_fn(self, *args, **kwargs)
|
40 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
|
41 |
+
return self._call_wrapped_method(fun, args, kwargs)
|
42 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
|
43 |
+
y = fun(self, *args, **kwargs)
|
44 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1452, in __call__
|
45 |
+
outputs = self.encoder(
|
46 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
|
47 |
+
return prewrapped_fn(self, *args, **kwargs)
|
48 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
|
49 |
+
return self._call_wrapped_method(fun, args, kwargs)
|
50 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
|
51 |
+
y = fun(self, *args, **kwargs)
|
52 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1259, in __call__
|
53 |
+
return self.layer(
|
54 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
|
55 |
+
return prewrapped_fn(self, *args, **kwargs)
|
56 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
|
57 |
+
return self._call_wrapped_method(fun, args, kwargs)
|
58 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
|
59 |
+
y = fun(self, *args, **kwargs)
|
60 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1220, in __call__
|
61 |
+
layer_outputs = layer(
|
62 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
|
63 |
+
return prewrapped_fn(self, *args, **kwargs)
|
64 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
|
65 |
+
return self._call_wrapped_method(fun, args, kwargs)
|
66 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
|
67 |
+
y = fun(self, *args, **kwargs)
|
68 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1183, in __call__
|
69 |
+
hidden_states = self.intermediate(attention_output)
|
70 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
|
71 |
+
return prewrapped_fn(self, *args, **kwargs)
|
72 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
|
73 |
+
return self._call_wrapped_method(fun, args, kwargs)
|
74 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
|
75 |
+
y = fun(self, *args, **kwargs)
|
76 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1140, in __call__
|
77 |
+
hidden_states = self.dense(hidden_states)
|
78 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
|
79 |
+
return prewrapped_fn(self, *args, **kwargs)
|
80 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
|
81 |
+
return self._call_wrapped_method(fun, args, kwargs)
|
82 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
|
83 |
+
y = fun(self, *args, **kwargs)
|
84 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/linear.py", line 171, in __call__
|
85 |
+
kernel = self.param('kernel',
|
86 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 896, in param
|
87 |
+
v = self.scope.param(name, init_fn, *init_args)
|
88 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/core/scope.py", line 643, in param
|
89 |
+
value = init_fn(self.make_rng('params'), *init_args)
|
90 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/nn/initializers.py", line 44, in init
|
91 |
+
return random.normal(key, shape, dtype) * stddev
|
92 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/numpy/lax_numpy.py", line 6589, in deferring_binary_op
|
93 |
+
return binary_op(self, other)
|
94 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback
|
95 |
+
return fun(*args, **kwargs)
|
96 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/api.py", line 416, in cache_miss
|
97 |
+
out_flat = xla.xla_call(
|
98 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/core.py", line 1632, in bind
|
99 |
+
return call_bind(self, fun, *args, **params)
|
100 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/core.py", line 1623, in call_bind
|
101 |
+
outs = primitive.process(top_trace, fun, tracers, params)
|
102 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/core.py", line 1635, in process
|
103 |
+
return trace.process_call(self, fun, tracers, params)
|
104 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/core.py", line 627, in process_call
|
105 |
+
return primitive.impl(f, *tracers, **params)
|
106 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 581, in _xla_call_impl
|
107 |
+
compiled_fun = _xla_callable(fun, device, backend, name, donated_invars,
|
108 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/linear_util.py", line 263, in memoized_fun
|
109 |
+
ans = call(fun, *args)
|
110 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 653, in _xla_callable_uncached
|
111 |
+
return lower_xla_callable(fun, device, backend, name, donated_invars,
|
112 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 769, in compile
|
113 |
+
self._executable = XlaCompiledComputation.from_xla_computation(
|
114 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 798, in from_xla_computation
|
115 |
+
compiled = compile_or_get_cached(backend, xla_computation, options)
|
116 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 87, in compile_or_get_cached
|
117 |
+
return backend_compile(backend, computation, compile_options)
|
118 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 369, in backend_compile
|
119 |
+
return backend.compile(built_c, compile_options=options)
|
120 |
+
KeyboardInterrupt
|
wandb/run-20211108_080817-3dz8poa1/files/requirements.txt
ADDED
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
absl-py==0.15.0
|
2 |
+
aiohttp==3.8.0
|
3 |
+
aiosignal==1.2.0
|
4 |
+
astunparse==1.6.3
|
5 |
+
async-timeout==4.0.0
|
6 |
+
attrs==21.2.0
|
7 |
+
cachetools==4.2.4
|
8 |
+
certifi==2021.10.8
|
9 |
+
charset-normalizer==2.0.7
|
10 |
+
chex==0.0.8
|
11 |
+
click==8.0.3
|
12 |
+
configparser==5.1.0
|
13 |
+
cycler==0.11.0
|
14 |
+
datasets==1.15.1
|
15 |
+
dill==0.3.4
|
16 |
+
dm-tree==0.1.6
|
17 |
+
docker-pycreds==0.4.0
|
18 |
+
filelock==3.3.2
|
19 |
+
flatbuffers==2.0
|
20 |
+
flax==0.3.6
|
21 |
+
frozenlist==1.2.0
|
22 |
+
fsspec==2021.11.0
|
23 |
+
gast==0.4.0
|
24 |
+
gitdb==4.0.9
|
25 |
+
gitpython==3.1.24
|
26 |
+
google-auth-oauthlib==0.4.6
|
27 |
+
google-auth==2.3.3
|
28 |
+
google-pasta==0.2.0
|
29 |
+
grpcio==1.41.1
|
30 |
+
h5py==3.5.0
|
31 |
+
huggingface-hub==0.1.1
|
32 |
+
idna==3.3
|
33 |
+
jax==0.2.24
|
34 |
+
jaxlib==0.1.73
|
35 |
+
joblib==1.1.0
|
36 |
+
kaggle==1.5.12
|
37 |
+
keras-preprocessing==1.1.2
|
38 |
+
keras==2.7.0
|
39 |
+
kiwisolver==1.3.2
|
40 |
+
libclang==12.0.0
|
41 |
+
libtpu-nightly==0.1.dev20211018
|
42 |
+
markdown==3.3.4
|
43 |
+
matplotlib==3.4.3
|
44 |
+
msgpack==1.0.2
|
45 |
+
multidict==5.2.0
|
46 |
+
multiprocess==0.70.12.2
|
47 |
+
numpy==1.21.4
|
48 |
+
oauthlib==3.1.1
|
49 |
+
opt-einsum==3.3.0
|
50 |
+
optax==0.0.9
|
51 |
+
packaging==21.2
|
52 |
+
pandas==1.3.4
|
53 |
+
pathtools==0.1.2
|
54 |
+
pillow==8.4.0
|
55 |
+
pip==21.3.1
|
56 |
+
pkg-resources==0.0.0
|
57 |
+
promise==2.3
|
58 |
+
protobuf==3.19.1
|
59 |
+
psutil==5.8.0
|
60 |
+
pyarrow==6.0.0
|
61 |
+
pyasn1-modules==0.2.8
|
62 |
+
pyasn1==0.4.8
|
63 |
+
pyparsing==2.4.7
|
64 |
+
python-dateutil==2.8.2
|
65 |
+
python-slugify==5.0.2
|
66 |
+
pytz==2021.3
|
67 |
+
pyyaml==6.0
|
68 |
+
regex==2021.11.2
|
69 |
+
requests-oauthlib==1.3.0
|
70 |
+
requests==2.26.0
|
71 |
+
rsa==4.7.2
|
72 |
+
sacremoses==0.0.46
|
73 |
+
scipy==1.7.1
|
74 |
+
sentry-sdk==1.4.3
|
75 |
+
setuptools==44.0.0
|
76 |
+
shortuuid==1.0.1
|
77 |
+
six==1.16.0
|
78 |
+
smmap==5.0.0
|
79 |
+
subprocess32==3.5.4
|
80 |
+
tensorboard-data-server==0.6.1
|
81 |
+
tensorboard-plugin-wit==1.8.0
|
82 |
+
tensorboard==2.7.0
|
83 |
+
tensorflow-estimator==2.7.0
|
84 |
+
tensorflow-io-gcs-filesystem==0.21.0
|
85 |
+
tensorflow==2.7.0
|
86 |
+
termcolor==1.1.0
|
87 |
+
text-unidecode==1.3
|
88 |
+
tokenizers==0.10.3
|
89 |
+
toolz==0.11.1
|
90 |
+
torch==1.10.0
|
91 |
+
tqdm==4.62.3
|
92 |
+
transformers==4.13.0.dev0
|
93 |
+
typing-extensions==3.10.0.2
|
94 |
+
urllib3==1.26.7
|
95 |
+
wandb==0.12.6
|
96 |
+
werkzeug==2.0.2
|
97 |
+
wheel==0.37.0
|
98 |
+
wrapt==1.13.3
|
99 |
+
xxhash==2.0.2
|
100 |
+
yarl==1.7.2
|
101 |
+
yaspin==2.1.0
|
wandb/run-20211108_080817-3dz8poa1/files/wandb-metadata.json
ADDED
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
|
3 |
+
"python": "3.8.10",
|
4 |
+
"heartbeatAt": "2021-11-08T08:08:20.712947",
|
5 |
+
"startedAt": "2021-11-08T08:08:17.853714",
|
6 |
+
"docker": null,
|
7 |
+
"cpu_count": 96,
|
8 |
+
"cuda": null,
|
9 |
+
"args": [
|
10 |
+
"--model_name_or_path",
|
11 |
+
"muril-bigbird",
|
12 |
+
"--tokenizer_name",
|
13 |
+
"muril-bigbird",
|
14 |
+
"--dtype",
|
15 |
+
"bfloat16",
|
16 |
+
"--pretokenized",
|
17 |
+
"--resume_from_checkpoint",
|
18 |
+
"checkpoint_100000",
|
19 |
+
"--train_file",
|
20 |
+
"/dev/shm/data-files",
|
21 |
+
"--validation_file",
|
22 |
+
".",
|
23 |
+
"--max_seq_length",
|
24 |
+
"4096",
|
25 |
+
"--pad_to_max_length",
|
26 |
+
"--output_dir",
|
27 |
+
"./",
|
28 |
+
"--overwrite_output_dir",
|
29 |
+
"--do_train",
|
30 |
+
"--do_eval",
|
31 |
+
"--eval_steps",
|
32 |
+
"5000",
|
33 |
+
"--per_device_train_batch_size",
|
34 |
+
"1",
|
35 |
+
"--per_device_eval_batch_size",
|
36 |
+
"2",
|
37 |
+
"--learning_rate",
|
38 |
+
"5e-5",
|
39 |
+
"--weight_decay",
|
40 |
+
"0.01",
|
41 |
+
"--num_train_epochs",
|
42 |
+
"3",
|
43 |
+
"--warmup_steps",
|
44 |
+
"0",
|
45 |
+
"--logging_steps",
|
46 |
+
"1000",
|
47 |
+
"--save_steps",
|
48 |
+
"25000",
|
49 |
+
"--save_total_limit",
|
50 |
+
"1",
|
51 |
+
"--seed",
|
52 |
+
"18",
|
53 |
+
"--push_to_hub",
|
54 |
+
"--run_name",
|
55 |
+
"en-ta-hi-te-from-100k"
|
56 |
+
],
|
57 |
+
"state": "running",
|
58 |
+
"program": "run_muril_bb_flax_mlm.py",
|
59 |
+
"codePath": "run_muril_bb_flax_mlm.py",
|
60 |
+
"git": {
|
61 |
+
"remote": "https://huggingface.co/nbroad/muril-bigbird-base-1mil",
|
62 |
+
"commit": "5fca06e3f8deb06ee5f938513b5f093fc3b66a62"
|
63 |
+
},
|
64 |
+
"email": "nicholas@nmbroad.com",
|
65 |
+
"root": "/home/nicholasbroad/muril-bigbird-base-1mil",
|
66 |
+
"host": "t1v-n-ba7aa1a5-w-0",
|
67 |
+
"username": "nicholasbroad",
|
68 |
+
"executable": "/home/nicholasbroad/venv/bin/python"
|
69 |
+
}
|
wandb/run-20211108_080817-3dz8poa1/files/wandb-summary.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"_wandb": {"runtime": 13}}
|
wandb/run-20211108_080817-3dz8poa1/logs/debug-internal.log
ADDED
@@ -0,0 +1,142 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2021-11-08 08:08:18,489 INFO MainThread:6780 [internal.py:wandb_internal():87] W&B internal server running at pid: 6780, started at: 2021-11-08 08:08:18.489329
|
2 |
+
2021-11-08 08:08:18,491 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: check_version
|
3 |
+
2021-11-08 08:08:18,491 INFO WriterThread:6780 [datastore.py:open_for_write():77] open: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/run-3dz8poa1.wandb
|
4 |
+
2021-11-08 08:08:18,492 DEBUG SenderThread:6780 [sender.py:send():238] send: header
|
5 |
+
2021-11-08 08:08:18,493 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: check_version
|
6 |
+
2021-11-08 08:08:18,528 DEBUG SenderThread:6780 [sender.py:send():238] send: run
|
7 |
+
2021-11-08 08:08:18,688 INFO SenderThread:6780 [dir_watcher.py:__init__():169] watching files in: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files
|
8 |
+
2021-11-08 08:08:18,688 INFO SenderThread:6780 [sender.py:_start_run_threads():825] run started: 3dz8poa1 with start time 1636358897
|
9 |
+
2021-11-08 08:08:18,688 DEBUG SenderThread:6780 [sender.py:send():238] send: summary
|
10 |
+
2021-11-08 08:08:18,688 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: run_start
|
11 |
+
2021-11-08 08:08:18,689 INFO SenderThread:6780 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
|
12 |
+
2021-11-08 08:08:19,690 INFO Thread-8 :6780 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/wandb-summary.json
|
13 |
+
2021-11-08 08:08:20,712 DEBUG HandlerThread:6780 [meta.py:__init__():40] meta init
|
14 |
+
2021-11-08 08:08:20,712 DEBUG HandlerThread:6780 [meta.py:__init__():54] meta init done
|
15 |
+
2021-11-08 08:08:20,712 DEBUG HandlerThread:6780 [meta.py:probe():214] probe
|
16 |
+
2021-11-08 08:08:20,713 DEBUG HandlerThread:6780 [meta.py:_setup_git():204] setup git
|
17 |
+
2021-11-08 08:08:20,743 DEBUG HandlerThread:6780 [meta.py:_setup_git():211] setup git done
|
18 |
+
2021-11-08 08:08:20,743 DEBUG HandlerThread:6780 [meta.py:_save_pip():58] save pip
|
19 |
+
2021-11-08 08:08:20,743 DEBUG HandlerThread:6780 [meta.py:_save_pip():72] save pip done
|
20 |
+
2021-11-08 08:08:20,743 DEBUG HandlerThread:6780 [meta.py:probe():252] probe done
|
21 |
+
2021-11-08 08:08:20,746 DEBUG SenderThread:6780 [sender.py:send():238] send: files
|
22 |
+
2021-11-08 08:08:20,746 INFO SenderThread:6780 [sender.py:_save_file():960] saving file wandb-metadata.json with policy now
|
23 |
+
2021-11-08 08:08:20,752 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: stop_status
|
24 |
+
2021-11-08 08:08:20,752 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: stop_status
|
25 |
+
2021-11-08 08:08:20,883 DEBUG SenderThread:6780 [sender.py:send():238] send: config
|
26 |
+
2021-11-08 08:08:20,883 DEBUG SenderThread:6780 [sender.py:send():238] send: config
|
27 |
+
2021-11-08 08:08:20,883 DEBUG SenderThread:6780 [sender.py:send():238] send: config
|
28 |
+
2021-11-08 08:08:21,236 INFO Thread-11 :6780 [upload_job.py:push():137] Uploaded file /tmp/tmpiwrpz1kvwandb/13hbo1qc-wandb-metadata.json
|
29 |
+
2021-11-08 08:08:21,693 INFO Thread-8 :6780 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/output.log
|
30 |
+
2021-11-08 08:08:21,693 INFO Thread-8 :6780 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/wandb-metadata.json
|
31 |
+
2021-11-08 08:08:21,693 INFO Thread-8 :6780 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/requirements.txt
|
32 |
+
2021-11-08 08:08:29,437 WARNING MainThread:6780 [internal.py:wandb_internal():146] Internal process interrupt: 1
|
33 |
+
2021-11-08 08:08:31,697 INFO Thread-8 :6780 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/output.log
|
34 |
+
2021-11-08 08:08:31,797 DEBUG SenderThread:6780 [sender.py:send():238] send: telemetry
|
35 |
+
2021-11-08 08:08:31,797 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
|
36 |
+
2021-11-08 08:08:31,797 DEBUG SenderThread:6780 [sender.py:send():238] send: exit
|
37 |
+
2021-11-08 08:08:31,797 INFO SenderThread:6780 [sender.py:send_exit():370] handling exit code: 255
|
38 |
+
2021-11-08 08:08:31,798 INFO SenderThread:6780 [sender.py:send_exit():372] handling runtime: 13
|
39 |
+
2021-11-08 08:08:31,798 INFO SenderThread:6780 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
|
40 |
+
2021-11-08 08:08:31,798 INFO SenderThread:6780 [sender.py:send_exit():382] send defer
|
41 |
+
2021-11-08 08:08:31,798 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
|
42 |
+
2021-11-08 08:08:31,799 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
|
43 |
+
2021-11-08 08:08:31,799 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 0
|
44 |
+
2021-11-08 08:08:31,799 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
|
45 |
+
2021-11-08 08:08:31,799 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 0
|
46 |
+
2021-11-08 08:08:31,799 INFO SenderThread:6780 [sender.py:transition_state():395] send defer: 1
|
47 |
+
2021-11-08 08:08:31,800 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
|
48 |
+
2021-11-08 08:08:31,800 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 1
|
49 |
+
2021-11-08 08:08:31,849 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
|
50 |
+
2021-11-08 08:08:31,849 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 1
|
51 |
+
2021-11-08 08:08:31,849 INFO SenderThread:6780 [sender.py:transition_state():395] send defer: 2
|
52 |
+
2021-11-08 08:08:31,849 DEBUG SenderThread:6780 [sender.py:send():238] send: stats
|
53 |
+
2021-11-08 08:08:31,850 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
|
54 |
+
2021-11-08 08:08:31,850 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 2
|
55 |
+
2021-11-08 08:08:31,850 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
|
56 |
+
2021-11-08 08:08:31,850 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 2
|
57 |
+
2021-11-08 08:08:31,850 INFO SenderThread:6780 [sender.py:transition_state():395] send defer: 3
|
58 |
+
2021-11-08 08:08:31,850 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
|
59 |
+
2021-11-08 08:08:31,850 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 3
|
60 |
+
2021-11-08 08:08:31,850 DEBUG SenderThread:6780 [sender.py:send():238] send: summary
|
61 |
+
2021-11-08 08:08:31,851 INFO SenderThread:6780 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
|
62 |
+
2021-11-08 08:08:31,851 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
|
63 |
+
2021-11-08 08:08:31,851 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 3
|
64 |
+
2021-11-08 08:08:31,851 INFO SenderThread:6780 [sender.py:transition_state():395] send defer: 4
|
65 |
+
2021-11-08 08:08:31,851 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
|
66 |
+
2021-11-08 08:08:31,851 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 4
|
67 |
+
2021-11-08 08:08:31,851 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
|
68 |
+
2021-11-08 08:08:31,851 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 4
|
69 |
+
2021-11-08 08:08:31,900 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
|
70 |
+
2021-11-08 08:08:32,052 INFO SenderThread:6780 [sender.py:transition_state():395] send defer: 5
|
71 |
+
2021-11-08 08:08:32,053 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
|
72 |
+
2021-11-08 08:08:32,053 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
|
73 |
+
2021-11-08 08:08:32,053 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 5
|
74 |
+
2021-11-08 08:08:32,053 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
|
75 |
+
2021-11-08 08:08:32,054 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 5
|
76 |
+
2021-11-08 08:08:32,054 INFO SenderThread:6780 [dir_watcher.py:finish():283] shutting down directory watcher
|
77 |
+
2021-11-08 08:08:32,154 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
|
78 |
+
2021-11-08 08:08:32,697 INFO Thread-8 :6780 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/wandb-summary.json
|
79 |
+
2021-11-08 08:08:32,698 INFO SenderThread:6780 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/config.yaml
|
80 |
+
2021-11-08 08:08:32,698 INFO SenderThread:6780 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/output.log
|
81 |
+
2021-11-08 08:08:32,698 INFO SenderThread:6780 [dir_watcher.py:finish():313] scan: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files
|
82 |
+
2021-11-08 08:08:32,698 INFO SenderThread:6780 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/requirements.txt requirements.txt
|
83 |
+
2021-11-08 08:08:32,699 INFO SenderThread:6780 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/output.log output.log
|
84 |
+
2021-11-08 08:08:32,699 INFO SenderThread:6780 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/wandb-metadata.json wandb-metadata.json
|
85 |
+
2021-11-08 08:08:32,699 INFO SenderThread:6780 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/config.yaml config.yaml
|
86 |
+
2021-11-08 08:08:32,699 INFO SenderThread:6780 [dir_watcher.py:finish():327] scan save: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/wandb-summary.json wandb-summary.json
|
87 |
+
2021-11-08 08:08:32,699 INFO SenderThread:6780 [sender.py:transition_state():395] send defer: 6
|
88 |
+
2021-11-08 08:08:32,699 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
|
89 |
+
2021-11-08 08:08:32,700 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
|
90 |
+
2021-11-08 08:08:32,700 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 6
|
91 |
+
2021-11-08 08:08:32,700 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
|
92 |
+
2021-11-08 08:08:32,700 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 6
|
93 |
+
2021-11-08 08:08:32,700 INFO SenderThread:6780 [file_pusher.py:finish():177] shutting down file pusher
|
94 |
+
2021-11-08 08:08:32,804 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
|
95 |
+
2021-11-08 08:08:32,805 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
|
96 |
+
2021-11-08 08:08:32,908 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
|
97 |
+
2021-11-08 08:08:32,908 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
|
98 |
+
2021-11-08 08:08:33,009 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
|
99 |
+
2021-11-08 08:08:33,010 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
|
100 |
+
2021-11-08 08:08:33,111 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
|
101 |
+
2021-11-08 08:08:33,111 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
|
102 |
+
2021-11-08 08:08:33,204 INFO Thread-15 :6780 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/wandb-summary.json
|
103 |
+
2021-11-08 08:08:33,206 INFO Thread-12 :6780 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/requirements.txt
|
104 |
+
2021-11-08 08:08:33,213 INFO Thread-14 :6780 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/config.yaml
|
105 |
+
2021-11-08 08:08:33,213 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
|
106 |
+
2021-11-08 08:08:33,213 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
|
107 |
+
2021-11-08 08:08:33,263 INFO Thread-13 :6780 [upload_job.py:push():137] Uploaded file /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/files/output.log
|
108 |
+
2021-11-08 08:08:33,315 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
|
109 |
+
2021-11-08 08:08:33,315 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
|
110 |
+
2021-11-08 08:08:33,416 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
|
111 |
+
2021-11-08 08:08:33,417 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
|
112 |
+
2021-11-08 08:08:33,463 INFO Thread-7 :6780 [sender.py:transition_state():395] send defer: 7
|
113 |
+
2021-11-08 08:08:33,463 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
|
114 |
+
2021-11-08 08:08:33,464 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 7
|
115 |
+
2021-11-08 08:08:33,464 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
|
116 |
+
2021-11-08 08:08:33,464 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 7
|
117 |
+
2021-11-08 08:08:33,518 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
|
118 |
+
2021-11-08 08:08:33,592 INFO SenderThread:6780 [sender.py:transition_state():395] send defer: 8
|
119 |
+
2021-11-08 08:08:33,592 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
|
120 |
+
2021-11-08 08:08:33,592 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
|
121 |
+
2021-11-08 08:08:33,592 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 8
|
122 |
+
2021-11-08 08:08:33,593 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
|
123 |
+
2021-11-08 08:08:33,593 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 8
|
124 |
+
2021-11-08 08:08:33,593 INFO SenderThread:6780 [sender.py:transition_state():395] send defer: 9
|
125 |
+
2021-11-08 08:08:33,593 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: defer
|
126 |
+
2021-11-08 08:08:33,593 INFO HandlerThread:6780 [handler.py:handle_request_defer():147] handle defer: 9
|
127 |
+
2021-11-08 08:08:33,594 DEBUG SenderThread:6780 [sender.py:send():238] send: final
|
128 |
+
2021-11-08 08:08:33,594 DEBUG SenderThread:6780 [sender.py:send():238] send: footer
|
129 |
+
2021-11-08 08:08:33,594 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: defer
|
130 |
+
2021-11-08 08:08:33,594 INFO SenderThread:6780 [sender.py:send_request_defer():391] handle sender defer: 9
|
131 |
+
2021-11-08 08:08:33,693 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: poll_exit
|
132 |
+
2021-11-08 08:08:33,693 DEBUG SenderThread:6780 [sender.py:send_request():252] send_request: poll_exit
|
133 |
+
2021-11-08 08:08:33,694 INFO SenderThread:6780 [file_pusher.py:join():182] waiting for file pusher
|
134 |
+
2021-11-08 08:08:33,969 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: get_summary
|
135 |
+
2021-11-08 08:08:33,969 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: sampled_history
|
136 |
+
2021-11-08 08:08:33,970 DEBUG HandlerThread:6780 [handler.py:handle_request():130] handle_request: shutdown
|
137 |
+
2021-11-08 08:08:33,970 INFO HandlerThread:6780 [handler.py:finish():731] shutting down handler
|
138 |
+
2021-11-08 08:08:34,594 INFO WriterThread:6780 [datastore.py:close():281] close: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/run-3dz8poa1.wandb
|
139 |
+
2021-11-08 08:08:34,968 INFO SenderThread:6780 [sender.py:finish():1091] shutting down sender
|
140 |
+
2021-11-08 08:08:34,968 INFO SenderThread:6780 [file_pusher.py:finish():177] shutting down file pusher
|
141 |
+
2021-11-08 08:08:34,968 INFO SenderThread:6780 [file_pusher.py:join():182] waiting for file pusher
|
142 |
+
2021-11-08 08:08:34,971 INFO MainThread:6780 [internal.py:handle_exit():77] Internal process exited
|
wandb/run-20211108_080817-3dz8poa1/logs/debug.log
ADDED
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2021-11-08 08:08:17,854 INFO MainThread:5478 [wandb_setup.py:_flush():71] setting env: {}
|
2 |
+
2021-11-08 08:08:17,855 INFO MainThread:5478 [wandb_setup.py:_flush():71] setting login settings: {}
|
3 |
+
2021-11-08 08:08:17,855 INFO MainThread:5478 [wandb_init.py:_log_setup():357] Logging user logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/logs/debug.log
|
4 |
+
2021-11-08 08:08:17,855 INFO MainThread:5478 [wandb_init.py:_log_setup():358] Logging internal logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211108_080817-3dz8poa1/logs/debug-internal.log
|
5 |
+
2021-11-08 08:08:17,855 INFO MainThread:5478 [wandb_init.py:init():390] calling init triggers
|
6 |
+
2021-11-08 08:08:17,855 INFO MainThread:5478 [wandb_init.py:init():395] wandb.init called with sweep_config: {}
|
7 |
+
config: {}
|
8 |
+
2021-11-08 08:08:17,855 INFO MainThread:5478 [wandb_init.py:init():435] starting backend
|
9 |
+
2021-11-08 08:08:17,855 INFO MainThread:5478 [backend.py:_multiprocessing_setup():95] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
|
10 |
+
2021-11-08 08:08:17,887 INFO MainThread:5478 [backend.py:ensure_launched():199] starting backend process...
|
11 |
+
2021-11-08 08:08:17,917 INFO MainThread:5478 [backend.py:ensure_launched():204] started backend process with pid: 6780
|
12 |
+
2021-11-08 08:08:17,919 INFO MainThread:5478 [wandb_init.py:init():444] backend started and connected
|
13 |
+
2021-11-08 08:08:17,923 INFO MainThread:5478 [wandb_init.py:init():503] updated telemetry
|
14 |
+
2021-11-08 08:08:18,001 INFO MainThread:5478 [wandb_init.py:init():533] communicating current version
|
15 |
+
2021-11-08 08:08:18,527 INFO MainThread:5478 [wandb_init.py:init():538] got version response
|
16 |
+
2021-11-08 08:08:18,527 INFO MainThread:5478 [wandb_init.py:init():548] communicating run to backend with 30 second timeout
|
17 |
+
2021-11-08 08:08:18,688 INFO MainThread:5478 [wandb_init.py:init():576] starting run threads in backend
|
18 |
+
2021-11-08 08:08:20,751 INFO MainThread:5478 [wandb_run.py:_console_start():1693] atexit reg
|
19 |
+
2021-11-08 08:08:20,751 INFO MainThread:5478 [wandb_run.py:_redirect():1567] redirect: SettingsConsole.REDIRECT
|
20 |
+
2021-11-08 08:08:20,752 INFO MainThread:5478 [wandb_run.py:_redirect():1572] Redirecting console.
|
21 |
+
2021-11-08 08:08:20,754 INFO MainThread:5478 [wandb_run.py:_redirect():1628] Redirects installed.
|
22 |
+
2021-11-08 08:08:20,754 INFO MainThread:5478 [wandb_init.py:init():603] run started, returning control to user process
|
23 |
+
2021-11-08 08:08:20,760 INFO MainThread:5478 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov08_08-08-10_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 1000, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 25000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te-from-100k', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'checkpoint_100000', 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
|
24 |
+
2021-11-08 08:08:20,762 INFO MainThread:5478 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
|
25 |
+
2021-11-08 08:08:20,763 INFO MainThread:5478 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}
|
26 |
+
2021-11-08 08:08:29,658 INFO MainThread:5478 [wandb_run.py:_atexit_cleanup():1663] got exitcode: 255
|
27 |
+
2021-11-08 08:08:29,658 INFO MainThread:5478 [wandb_run.py:_restore():1635] restore
|
28 |
+
2021-11-08 08:08:31,799 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
29 |
+
wandb_count: 1
|
30 |
+
}
|
31 |
+
pusher_stats {
|
32 |
+
uploaded_bytes: 1825
|
33 |
+
total_bytes: 1825
|
34 |
+
}
|
35 |
+
|
36 |
+
2021-11-08 08:08:32,053 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
37 |
+
wandb_count: 1
|
38 |
+
}
|
39 |
+
pusher_stats {
|
40 |
+
uploaded_bytes: 1825
|
41 |
+
total_bytes: 1825
|
42 |
+
}
|
43 |
+
|
44 |
+
2021-11-08 08:08:32,703 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
45 |
+
wandb_count: 5
|
46 |
+
}
|
47 |
+
pusher_stats {
|
48 |
+
uploaded_bytes: 1825
|
49 |
+
total_bytes: 18699
|
50 |
+
}
|
51 |
+
|
52 |
+
2021-11-08 08:08:32,807 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
53 |
+
wandb_count: 5
|
54 |
+
}
|
55 |
+
pusher_stats {
|
56 |
+
uploaded_bytes: 1825
|
57 |
+
total_bytes: 18699
|
58 |
+
}
|
59 |
+
|
60 |
+
2021-11-08 08:08:32,909 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
61 |
+
wandb_count: 5
|
62 |
+
}
|
63 |
+
pusher_stats {
|
64 |
+
uploaded_bytes: 18699
|
65 |
+
total_bytes: 18699
|
66 |
+
}
|
67 |
+
|
68 |
+
2021-11-08 08:08:33,010 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
69 |
+
wandb_count: 5
|
70 |
+
}
|
71 |
+
pusher_stats {
|
72 |
+
uploaded_bytes: 18699
|
73 |
+
total_bytes: 18699
|
74 |
+
}
|
75 |
+
|
76 |
+
2021-11-08 08:08:33,112 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
77 |
+
wandb_count: 5
|
78 |
+
}
|
79 |
+
pusher_stats {
|
80 |
+
uploaded_bytes: 18699
|
81 |
+
total_bytes: 18699
|
82 |
+
}
|
83 |
+
|
84 |
+
2021-11-08 08:08:33,214 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
85 |
+
wandb_count: 5
|
86 |
+
}
|
87 |
+
pusher_stats {
|
88 |
+
uploaded_bytes: 18699
|
89 |
+
total_bytes: 18699
|
90 |
+
}
|
91 |
+
|
92 |
+
2021-11-08 08:08:33,315 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
93 |
+
wandb_count: 5
|
94 |
+
}
|
95 |
+
pusher_stats {
|
96 |
+
uploaded_bytes: 18699
|
97 |
+
total_bytes: 18699
|
98 |
+
}
|
99 |
+
|
100 |
+
2021-11-08 08:08:33,417 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
101 |
+
wandb_count: 5
|
102 |
+
}
|
103 |
+
pusher_stats {
|
104 |
+
uploaded_bytes: 18699
|
105 |
+
total_bytes: 18699
|
106 |
+
}
|
107 |
+
|
108 |
+
2021-11-08 08:08:33,592 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
109 |
+
wandb_count: 5
|
110 |
+
}
|
111 |
+
pusher_stats {
|
112 |
+
uploaded_bytes: 18699
|
113 |
+
total_bytes: 18699
|
114 |
+
}
|
115 |
+
|
116 |
+
2021-11-08 08:08:33,968 INFO MainThread:5478 [wandb_run.py:_wait_for_finish():1793] got exit ret: done: true
|
117 |
+
exit_result {
|
118 |
+
}
|
119 |
+
file_counts {
|
120 |
+
wandb_count: 5
|
121 |
+
}
|
122 |
+
pusher_stats {
|
123 |
+
uploaded_bytes: 18699
|
124 |
+
total_bytes: 18699
|
125 |
+
}
|
126 |
+
local_info {
|
127 |
+
}
|
128 |
+
|
129 |
+
2021-11-08 08:08:35,338 INFO MainThread:5478 [wandb_run.py:_append_files():2061] logging synced files
|
wandb/run-20211108_080817-3dz8poa1/run-3dz8poa1.wandb
ADDED
Binary file (13.5 kB). View file
|
|
wandb/run-20211108_080940-r077t6tw/files/config.yaml
ADDED
@@ -0,0 +1,347 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
wandb_version: 1
|
2 |
+
|
3 |
+
__cached__setup_devices:
|
4 |
+
desc: null
|
5 |
+
value: cpu
|
6 |
+
_n_gpu:
|
7 |
+
desc: null
|
8 |
+
value: 0
|
9 |
+
_wandb:
|
10 |
+
desc: null
|
11 |
+
value:
|
12 |
+
cli_version: 0.12.6
|
13 |
+
framework: huggingface
|
14 |
+
huggingface_version: 4.13.0.dev0
|
15 |
+
is_jupyter_run: false
|
16 |
+
is_kaggle_kernel: true
|
17 |
+
python_version: 3.8.10
|
18 |
+
start_time: 1636358980
|
19 |
+
t:
|
20 |
+
1:
|
21 |
+
- 1
|
22 |
+
- 2
|
23 |
+
- 3
|
24 |
+
- 11
|
25 |
+
- 12
|
26 |
+
2:
|
27 |
+
- 1
|
28 |
+
- 2
|
29 |
+
- 3
|
30 |
+
- 11
|
31 |
+
- 12
|
32 |
+
3:
|
33 |
+
- 13
|
34 |
+
4: 3.8.10
|
35 |
+
5: 0.12.6
|
36 |
+
6: 4.13.0.dev0
|
37 |
+
8:
|
38 |
+
- 2
|
39 |
+
- 5
|
40 |
+
adafactor:
|
41 |
+
desc: null
|
42 |
+
value: false
|
43 |
+
adam_beta1:
|
44 |
+
desc: null
|
45 |
+
value: 0.9
|
46 |
+
adam_beta2:
|
47 |
+
desc: null
|
48 |
+
value: 0.999
|
49 |
+
adam_epsilon:
|
50 |
+
desc: null
|
51 |
+
value: 1.0e-08
|
52 |
+
cache_dir:
|
53 |
+
desc: null
|
54 |
+
value: null
|
55 |
+
config_name:
|
56 |
+
desc: null
|
57 |
+
value: null
|
58 |
+
dataloader_drop_last:
|
59 |
+
desc: null
|
60 |
+
value: false
|
61 |
+
dataloader_num_workers:
|
62 |
+
desc: null
|
63 |
+
value: 0
|
64 |
+
dataloader_pin_memory:
|
65 |
+
desc: null
|
66 |
+
value: true
|
67 |
+
dataset_config_name:
|
68 |
+
desc: null
|
69 |
+
value: null
|
70 |
+
dataset_name:
|
71 |
+
desc: null
|
72 |
+
value: null
|
73 |
+
ddp_find_unused_parameters:
|
74 |
+
desc: null
|
75 |
+
value: null
|
76 |
+
debug:
|
77 |
+
desc: null
|
78 |
+
value: []
|
79 |
+
deepspeed:
|
80 |
+
desc: null
|
81 |
+
value: null
|
82 |
+
disable_tqdm:
|
83 |
+
desc: null
|
84 |
+
value: false
|
85 |
+
do_eval:
|
86 |
+
desc: null
|
87 |
+
value: true
|
88 |
+
do_predict:
|
89 |
+
desc: null
|
90 |
+
value: false
|
91 |
+
do_train:
|
92 |
+
desc: null
|
93 |
+
value: true
|
94 |
+
dtype:
|
95 |
+
desc: null
|
96 |
+
value: bfloat16
|
97 |
+
eval_accumulation_steps:
|
98 |
+
desc: null
|
99 |
+
value: null
|
100 |
+
eval_steps:
|
101 |
+
desc: null
|
102 |
+
value: 5000
|
103 |
+
evaluation_strategy:
|
104 |
+
desc: null
|
105 |
+
value: IntervalStrategy.NO
|
106 |
+
fp16:
|
107 |
+
desc: null
|
108 |
+
value: false
|
109 |
+
fp16_backend:
|
110 |
+
desc: null
|
111 |
+
value: auto
|
112 |
+
fp16_full_eval:
|
113 |
+
desc: null
|
114 |
+
value: false
|
115 |
+
fp16_opt_level:
|
116 |
+
desc: null
|
117 |
+
value: O1
|
118 |
+
gradient_accumulation_steps:
|
119 |
+
desc: null
|
120 |
+
value: 1
|
121 |
+
gradient_checkpointing:
|
122 |
+
desc: null
|
123 |
+
value: false
|
124 |
+
greater_is_better:
|
125 |
+
desc: null
|
126 |
+
value: null
|
127 |
+
group_by_length:
|
128 |
+
desc: null
|
129 |
+
value: false
|
130 |
+
hub_model_id:
|
131 |
+
desc: null
|
132 |
+
value: null
|
133 |
+
hub_strategy:
|
134 |
+
desc: null
|
135 |
+
value: HubStrategy.EVERY_SAVE
|
136 |
+
hub_token:
|
137 |
+
desc: null
|
138 |
+
value: null
|
139 |
+
ignore_data_skip:
|
140 |
+
desc: null
|
141 |
+
value: false
|
142 |
+
label_names:
|
143 |
+
desc: null
|
144 |
+
value: null
|
145 |
+
label_smoothing_factor:
|
146 |
+
desc: null
|
147 |
+
value: 0.0
|
148 |
+
learning_rate:
|
149 |
+
desc: null
|
150 |
+
value: 5.0e-05
|
151 |
+
length_column_name:
|
152 |
+
desc: null
|
153 |
+
value: length
|
154 |
+
line_by_line:
|
155 |
+
desc: null
|
156 |
+
value: false
|
157 |
+
load_best_model_at_end:
|
158 |
+
desc: null
|
159 |
+
value: false
|
160 |
+
local_rank:
|
161 |
+
desc: null
|
162 |
+
value: -1
|
163 |
+
log_level:
|
164 |
+
desc: null
|
165 |
+
value: -1
|
166 |
+
log_level_replica:
|
167 |
+
desc: null
|
168 |
+
value: -1
|
169 |
+
log_on_each_node:
|
170 |
+
desc: null
|
171 |
+
value: true
|
172 |
+
logging_dir:
|
173 |
+
desc: null
|
174 |
+
value: ./runs/Nov08_08-09-31_t1v-n-ba7aa1a5-w-0
|
175 |
+
logging_first_step:
|
176 |
+
desc: null
|
177 |
+
value: false
|
178 |
+
logging_nan_inf_filter:
|
179 |
+
desc: null
|
180 |
+
value: true
|
181 |
+
logging_steps:
|
182 |
+
desc: null
|
183 |
+
value: 1000
|
184 |
+
logging_strategy:
|
185 |
+
desc: null
|
186 |
+
value: IntervalStrategy.STEPS
|
187 |
+
lr_scheduler_type:
|
188 |
+
desc: null
|
189 |
+
value: SchedulerType.LINEAR
|
190 |
+
max_grad_norm:
|
191 |
+
desc: null
|
192 |
+
value: 1.0
|
193 |
+
max_seq_length:
|
194 |
+
desc: null
|
195 |
+
value: 4096
|
196 |
+
max_steps:
|
197 |
+
desc: null
|
198 |
+
value: -1
|
199 |
+
metric_for_best_model:
|
200 |
+
desc: null
|
201 |
+
value: null
|
202 |
+
mlm_probability:
|
203 |
+
desc: null
|
204 |
+
value: 0.15
|
205 |
+
model_name_or_path:
|
206 |
+
desc: null
|
207 |
+
value: muril-bigbird
|
208 |
+
model_type:
|
209 |
+
desc: null
|
210 |
+
value: null
|
211 |
+
mp_parameters:
|
212 |
+
desc: null
|
213 |
+
value: ''
|
214 |
+
no_cuda:
|
215 |
+
desc: null
|
216 |
+
value: false
|
217 |
+
num_train_epochs:
|
218 |
+
desc: null
|
219 |
+
value: 3.0
|
220 |
+
output_dir:
|
221 |
+
desc: null
|
222 |
+
value: ./
|
223 |
+
overwrite_cache:
|
224 |
+
desc: null
|
225 |
+
value: false
|
226 |
+
overwrite_output_dir:
|
227 |
+
desc: null
|
228 |
+
value: true
|
229 |
+
pad_to_max_length:
|
230 |
+
desc: null
|
231 |
+
value: true
|
232 |
+
past_index:
|
233 |
+
desc: null
|
234 |
+
value: -1
|
235 |
+
per_device_eval_batch_size:
|
236 |
+
desc: null
|
237 |
+
value: 2
|
238 |
+
per_device_train_batch_size:
|
239 |
+
desc: null
|
240 |
+
value: 1
|
241 |
+
per_gpu_eval_batch_size:
|
242 |
+
desc: null
|
243 |
+
value: null
|
244 |
+
per_gpu_train_batch_size:
|
245 |
+
desc: null
|
246 |
+
value: null
|
247 |
+
prediction_loss_only:
|
248 |
+
desc: null
|
249 |
+
value: false
|
250 |
+
preprocessing_num_workers:
|
251 |
+
desc: null
|
252 |
+
value: null
|
253 |
+
pretokenized:
|
254 |
+
desc: null
|
255 |
+
value: true
|
256 |
+
push_to_hub:
|
257 |
+
desc: null
|
258 |
+
value: true
|
259 |
+
push_to_hub_model_id:
|
260 |
+
desc: null
|
261 |
+
value: null
|
262 |
+
push_to_hub_organization:
|
263 |
+
desc: null
|
264 |
+
value: null
|
265 |
+
push_to_hub_token:
|
266 |
+
desc: null
|
267 |
+
value: null
|
268 |
+
remove_unused_columns:
|
269 |
+
desc: null
|
270 |
+
value: true
|
271 |
+
report_to:
|
272 |
+
desc: null
|
273 |
+
value:
|
274 |
+
- tensorboard
|
275 |
+
- wandb
|
276 |
+
resume_from_checkpoint:
|
277 |
+
desc: null
|
278 |
+
value: checkpoint_100000
|
279 |
+
run_name:
|
280 |
+
desc: null
|
281 |
+
value: en-ta-hi-te-from-100k
|
282 |
+
save_on_each_node:
|
283 |
+
desc: null
|
284 |
+
value: false
|
285 |
+
save_steps:
|
286 |
+
desc: null
|
287 |
+
value: 25000
|
288 |
+
save_strategy:
|
289 |
+
desc: null
|
290 |
+
value: IntervalStrategy.STEPS
|
291 |
+
save_total_limit:
|
292 |
+
desc: null
|
293 |
+
value: 1
|
294 |
+
seed:
|
295 |
+
desc: null
|
296 |
+
value: 18
|
297 |
+
sharded_ddp:
|
298 |
+
desc: null
|
299 |
+
value: []
|
300 |
+
skip_memory_metrics:
|
301 |
+
desc: null
|
302 |
+
value: true
|
303 |
+
streaming:
|
304 |
+
desc: null
|
305 |
+
value: false
|
306 |
+
tokenizer_name:
|
307 |
+
desc: null
|
308 |
+
value: muril-bigbird
|
309 |
+
tpu_metrics_debug:
|
310 |
+
desc: null
|
311 |
+
value: false
|
312 |
+
tpu_num_cores:
|
313 |
+
desc: null
|
314 |
+
value: null
|
315 |
+
train_file:
|
316 |
+
desc: null
|
317 |
+
value: /dev/shm/data-files
|
318 |
+
train_ref_file:
|
319 |
+
desc: null
|
320 |
+
value: null
|
321 |
+
use_fast_tokenizer:
|
322 |
+
desc: null
|
323 |
+
value: true
|
324 |
+
use_legacy_prediction_loop:
|
325 |
+
desc: null
|
326 |
+
value: false
|
327 |
+
validation_file:
|
328 |
+
desc: null
|
329 |
+
value: .
|
330 |
+
validation_ref_file:
|
331 |
+
desc: null
|
332 |
+
value: null
|
333 |
+
validation_split_percentage:
|
334 |
+
desc: null
|
335 |
+
value: 5
|
336 |
+
warmup_ratio:
|
337 |
+
desc: null
|
338 |
+
value: 0.0
|
339 |
+
warmup_steps:
|
340 |
+
desc: null
|
341 |
+
value: 0
|
342 |
+
weight_decay:
|
343 |
+
desc: null
|
344 |
+
value: 0.01
|
345 |
+
xpu_backend:
|
346 |
+
desc: null
|
347 |
+
value: null
|
wandb/run-20211108_080940-r077t6tw/files/output.log
ADDED
@@ -0,0 +1,108 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
Traceback (most recent call last):
|
2 |
+
File "run_muril_bb_flax_mlm.py", line 548, in <module>
|
3 |
+
model = FlaxAutoModelForMaskedLM.from_pretrained(
|
4 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/auto/auto_factory.py", line 419, in from_pretrained
|
5 |
+
return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
|
6 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/modeling_flax_utils.py", line 351, in from_pretrained
|
7 |
+
model = cls(config, *model_args, **model_kwargs)
|
8 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1359, in __init__
|
9 |
+
super().__init__(config, module, input_shape=input_shape, seed=seed, dtype=dtype)
|
10 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/modeling_flax_utils.py", line 106, in __init__
|
11 |
+
random_params = self.init_weights(self.key, input_shape)
|
12 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1371, in init_weights
|
13 |
+
return self.module.init(rngs, input_ids, attention_mask, token_type_ids, position_ids, return_dict=False)[
|
14 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback
|
15 |
+
return fun(*args, **kwargs)
|
16 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 1122, in init
|
17 |
+
_, v_out = self.init_with_output(
|
18 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback
|
19 |
+
return fun(*args, **kwargs)
|
20 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 1091, in init_with_output
|
21 |
+
return self.apply(
|
22 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback
|
23 |
+
return fun(*args, **kwargs)
|
24 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 1058, in apply
|
25 |
+
return apply(
|
26 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/core/scope.py", line 706, in wrapper
|
27 |
+
y = fn(root, *args, **kwargs)
|
28 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 1313, in scope_fn
|
29 |
+
return fn(module.clone(parent=scope), *args, **kwargs)
|
30 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
|
31 |
+
return prewrapped_fn(self, *args, **kwargs)
|
32 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
|
33 |
+
return self._call_wrapped_method(fun, args, kwargs)
|
34 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
|
35 |
+
y = fun(self, *args, **kwargs)
|
36 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1607, in __call__
|
37 |
+
outputs = self.bert(
|
38 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
|
39 |
+
return prewrapped_fn(self, *args, **kwargs)
|
40 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
|
41 |
+
return self._call_wrapped_method(fun, args, kwargs)
|
42 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
|
43 |
+
y = fun(self, *args, **kwargs)
|
44 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 1449, in __call__
|
45 |
+
hidden_states = self.embeddings(
|
46 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
|
47 |
+
return prewrapped_fn(self, *args, **kwargs)
|
48 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
|
49 |
+
return self._call_wrapped_method(fun, args, kwargs)
|
50 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
|
51 |
+
y = fun(self, *args, **kwargs)
|
52 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/transformers/models/big_bird/modeling_flax_big_bird.py", line 208, in __call__
|
53 |
+
token_type_embeddings = self.token_type_embeddings(token_type_ids.astype("i4"))
|
54 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/transforms.py", line 883, in wrapped_fn
|
55 |
+
return prewrapped_fn(self, *args, **kwargs)
|
56 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
|
57 |
+
return self._call_wrapped_method(fun, args, kwargs)
|
58 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 594, in _call_wrapped_method
|
59 |
+
self._try_setup()
|
60 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 790, in _try_setup
|
61 |
+
self.setup()
|
62 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 318, in wrapped_module_method
|
63 |
+
return self._call_wrapped_method(fun, args, kwargs)
|
64 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 603, in _call_wrapped_method
|
65 |
+
y = fun(self, *args, **kwargs)
|
66 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/linear.py", line 405, in setup
|
67 |
+
self.embedding = self.param('embedding',
|
68 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/linen/module.py", line 896, in param
|
69 |
+
v = self.scope.param(name, init_fn, *init_args)
|
70 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/flax/core/scope.py", line 643, in param
|
71 |
+
value = init_fn(self.make_rng('params'), *init_args)
|
72 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/nn/initializers.py", line 44, in init
|
73 |
+
return random.normal(key, shape, dtype) * stddev
|
74 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/random.py", line 521, in normal
|
75 |
+
return _normal(key, shape, dtype) # type: ignore
|
76 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 162, in reraise_with_filtered_traceback
|
77 |
+
return fun(*args, **kwargs)
|
78 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/api.py", line 416, in cache_miss
|
79 |
+
out_flat = xla.xla_call(
|
80 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/core.py", line 1632, in bind
|
81 |
+
return call_bind(self, fun, *args, **params)
|
82 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/core.py", line 1623, in call_bind
|
83 |
+
outs = primitive.process(top_trace, fun, tracers, params)
|
84 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/core.py", line 1635, in process
|
85 |
+
return trace.process_call(self, fun, tracers, params)
|
86 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/core.py", line 627, in process_call
|
87 |
+
return primitive.impl(f, *tracers, **params)
|
88 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 581, in _xla_call_impl
|
89 |
+
compiled_fun = _xla_callable(fun, device, backend, name, donated_invars,
|
90 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/linear_util.py", line 263, in memoized_fun
|
91 |
+
ans = call(fun, *args)
|
92 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 653, in _xla_callable_uncached
|
93 |
+
return lower_xla_callable(fun, device, backend, name, donated_invars,
|
94 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 723, in lower_xla_callable
|
95 |
+
out_nodes = jaxpr_subcomp(ctx, jaxpr, xla_consts, *xla_args)
|
96 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 471, in jaxpr_subcomp
|
97 |
+
ans = rule(ctx, map(aval, eqn.invars), map(aval, eqn.outvars),
|
98 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 1217, in f_new
|
99 |
+
return jaxpr_subcomp(ctx, jaxpr, _xla_consts(ctx.builder, consts),
|
100 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 456, in jaxpr_subcomp
|
101 |
+
op_metadata = make_op_metadata(
|
102 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/interpreters/xla.py", line 122, in make_op_metadata
|
103 |
+
frame = source_info_util.user_frame(source_info) if source_info else None
|
104 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/source_info_util.py", line 48, in user_frame
|
105 |
+
return next(user_frames(source_info), None)
|
106 |
+
File "/home/nicholasbroad/venv/lib/python3.8/site-packages/jax/_src/source_info_util.py", line 44, in user_frames
|
107 |
+
return (x for x in (source_info.frames if source_info else [])
|
108 |
+
KeyboardInterrupt
|