Saving weights and logs of step 330000

checkpoint_330000

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1f3d169158f2c9e550dc3faa5c6cb6a12a521fa57dd73211eb0d8ed2529d039
+size 1530270447

events.out.tfevents.1626429561.t1v-n-f5c06ea1-w-0.782479.3.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0dfd97eaf4cc0a53b9da14bec5ce1bed8204f791c1cc01b5f8e10574c521ab5b
-size 13538170
+oid sha256:968e47ce5036297240debd5f269c8afd988281dc089a30a1cbea0d3083893fc3
+size 15794596

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac92325730ea7333b5d45a9fa077b185498b80875fc86bed0ba9cf8e12b5fe2f
+oid sha256:959d6674371c3d3cf3a7a09f7d69cf73ebb948dc9fdfd80a687dbc6605a61b3c
 size 510090043

wandb/run-20210716_095921-13hxxunp/files/output.log

@@ -10499,3 +10499,1633 @@ Training...: 89999it [10:28:46,  2.68it/s]████████████
 [20:34:21] - INFO - absl - Saved checkpoint at checkpoint_315000██████████| 500/500 [00:59<00:00,  7.90it/s]
 [20:34:23] - INFO - huggingface_hub.repository - git version 2.25.1
 git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
+[20:34:23] - DEBUG - huggingface_hub.repository - [Repository] is a valid git repo
+[20:35:42] - INFO - huggingface_hub.repository - Uploading LFS objects: 100% (3/3), 2.1 GB | 40 MB/s, done.
+
+
+
+
+Training...: 90049it [10:30:35,  2.70it/s]
+
+
+
+
+
+Training...: 90099it [10:30:55,  2.67it/s]
+
+
+
+
+Training...: 90149it [10:31:16,  2.71it/s]
+
+
+
+
+
+Training...: 90211it [10:31:50,  3.66it/s]
+
+
+
+Training...: 90249it [10:31:56,  2.81it/s]
+
+
+
+
+
+Training...: 90299it [10:32:16,  2.88it/s]
+
+
+
+
+
+Training...: 90349it [10:32:36,  2.76it/s]
+
+
+
+
+
+Training...: 90399it [10:32:56,  2.96it/s]
+
+
+
+
+
+
+Training...: 90457it [10:33:30,  1.31it/s]
+
+
+
+Training...: 90499it [10:33:36,  3.04it/s]
+
+
+
+
+Training...: 90549it [10:33:56,  2.82it/s]
+
+
+
+
+Training...: 90599it [10:34:16,  2.80it/s]
+
+
+
+
+Training...: 90649it [10:34:36,  2.96it/s]
+
+
+
+
+
+Training...: 90699it [10:34:56,  2.76it/s]
+
+
+
+
+Training...: 90749it [10:35:16,  2.85it/s]
+
+
+
+
+
+Training...: 90810it [10:35:50,  2.64it/s]
+
+
+
+Training...: 90849it [10:35:56,  2.83it/s]
+
+
+
+
+Training...: 90899it [10:36:17,  2.94it/s]
+
+
+
+
+Training...: 90949it [10:36:37,  2.76it/s]
+
+
+
+
+Training...: 90999it [10:36:57,  2.81it/s]
+
+
+
+
+Training...: 91049it [10:37:17,  2.77it/s]
+
+
+
+
+Training...: 91099it [10:37:37,  2.80it/s]
+
+
+
+
+Training...: 91149it [10:37:57,  2.99it/s]
+
+
+
+
+Training...: 91199it [10:38:17,  2.77it/s]
+
+
+
+
+Training...: 91249it [10:38:37,  2.84it/s]
+
+
+
+
+Training...: 91299it [10:38:57,  2.71it/s]
+
+
+
+
+Training...: 91349it [10:39:17,  2.77it/s]
+
+
+
+
+Training...: 91399it [10:39:37,  2.93it/s]
+
+
+
+
+Training...: 91449it [10:39:57,  2.79it/s]
+
+
+
+
+Training...: 91499it [10:40:17,  2.82it/s]
+
+
+
+
+Training...: 91549it [10:40:37,  2.90it/s]
+
+
+
+
+Training...: 91599it [10:40:57,  2.76it/s]
+
+
+
+
+Training...: 91649it [10:41:17,  2.79it/s]
+
+
+
+
+Training...: 91699it [10:41:38,  2.85it/s]
+
+
+
+
+Training...: 91749it [10:41:58,  2.99it/s]
+
+
+
+
+Training...: 91799it [10:42:18,  2.77it/s]
+
+
+
+
+Training...: 91849it [10:42:38,  2.81it/s]
+
+
+
+
+Training...: 91899it [10:42:58,  2.75it/s]
+
+
+
+
+Training...: 91949it [10:43:18,  2.80it/s]
+
+
+
+
+Training...: 91999it [10:43:38,  2.98it/s]
+
+
+
+
+Training...: 92049it [10:43:58,  2.71it/s]
+
+
+
+
+Training...: 92099it [10:44:18,  2.99it/s]
+
+
+
+
+Training...: 92149it [10:44:38,  2.75it/s]
+
+
+
+
+Training...: 92199it [10:44:58,  2.77it/s]
+
+
+
+
+Training...: 92249it [10:45:18,  2.98it/s]
+
+
+
+
+Training...: 92299it [10:45:38,  2.77it/s]
+
+
+
+
+Training...: 92349it [10:45:58,  2.96it/s]
+
+
+
+
+Training...: 92399it [10:46:18,  2.78it/s]
+
+
+
+
+Training...: 92449it [10:46:38,  2.78it/s]
+
+
+
+
+Training...: 92499it [10:46:59,  2.94it/s]
+
+
+
+
+Training...: 92549it [10:47:19,  2.76it/s]
+
+
+
+
+Training...: 92599it [10:47:39,  2.84it/s]
+
+
+
+
+Training...: 92649it [10:47:59,  2.82it/s]
+
+
+
+
+Training...: 92699it [10:48:19,  2.80it/s]
+
+
+
+
+Training...: 92749it [10:48:39,  2.88it/s]
+
+
+
+
+Training...: 92799it [10:48:59,  2.84it/s]
+
+
+
+
+Training...: 92849it [10:49:19,  2.94it/s]
+
+
+
+
+Training...: 92899it [10:49:39,  2.85it/s]
+
+
+
+
+Training...: 92949it [10:49:59,  2.96it/s]
+
+
+
+
+Training...: 92999it [10:50:19,  2.89it/s]
+Training...: 92999it [10:50:32,  2.89it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...: 93056it [10:51:57,  1.09it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 93105it [10:52:17,  1.05s/it]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 93150it [10:52:36,  4.00s/it]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 93200it [10:52:57,  4.03s/it]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 93251it [10:53:17,  2.84s/it]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 93301it [10:53:37,  2.84s/it]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 93351it [10:53:57,  2.77s/it]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 93401it [10:54:17,  2.88s/it]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 93451it [10:54:37,  2.88s/it]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 93503it [10:54:57,  1.63s/it]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 93549it [10:55:04,  2.73it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 93601it [10:55:37,  2.88s/it]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 93651it [10:55:57,  2.88s/it]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 93704it [10:56:18,  1.40s/it]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 93753it [10:56:38,  1.60s/it]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 93800it [10:56:57,  3.89s/it]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 93852it [10:57:18,  2.27s/it]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 93908it [10:57:38,  1.56it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 93958it [10:57:58,  1.97it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 94008it [10:58:18,  2.17it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 94062it [10:58:39,  3.00it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 94108it [10:58:58,  1.57it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 94157it [10:59:18,  1.44it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 94209it [10:59:39,  2.07it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 94259it [10:59:59,  2.08it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 94311it [11:00:19,  2.71it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 94364it [11:00:39,  3.75it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 94408it [11:00:59,  1.54it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 94460it [11:01:19,  2.17it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+Training...: 94499it [11:01:25,  2.76it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 94561it [11:01:59,  2.90it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+Training...: 94599it [11:02:05,  2.79it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 94649it [11:02:25,  2.77it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+Training...: 94711it [11:02:59,  2.91it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 94761it [11:03:19,  2.94it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 94813it [11:03:40,  3.70it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 94865it [11:04:00,  4.73it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 94907it [11:04:20,  1.27it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 94960it [11:04:40,  2.63it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95011it [11:05:00,  3.43it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95062it [11:05:20,  3.67it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+Training...: 95099it [11:05:26,  2.78it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95149it [11:05:46,  2.82it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95199it [11:06:06,  2.86it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95249it [11:06:26,  2.82it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95299it [11:06:46,  2.72it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95349it [11:07:06,  2.82it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95399it [11:07:26,  2.75it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95449it [11:07:46,  2.83it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95499it [11:08:06,  2.86it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95549it [11:08:27,  2.81it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95599it [11:08:47,  2.73it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95649it [11:09:07,  2.81it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95699it [11:09:27,  2.71it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95749it [11:09:47,  2.82it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95799it [11:10:07,  2.88it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95849it [11:10:27,  2.78it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95899it [11:10:47,  2.79it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95949it [11:11:07,  2.79it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 95999it [11:11:27,  2.84it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96049it [11:11:47,  2.77it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96099it [11:12:07,  2.81it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96149it [11:12:27,  2.83it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96199it [11:12:47,  2.76it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96249it [11:13:08,  2.15it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96299it [11:13:28,  2.76it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96349it [11:13:48,  2.77it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96399it [11:14:08,  2.86it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96449it [11:14:28,  2.79it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96499it [11:14:48,  2.88it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96549it [11:15:08,  2.80it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96599it [11:15:28,  2.81it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96649it [11:15:48,  2.87it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96699it [11:16:08,  2.76it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96749it [11:16:28,  2.76it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96799it [11:16:48,  2.78it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96849it [11:17:08,  2.75it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96899it [11:17:28,  2.81it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96949it [11:17:48,  2.73it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 96999it [11:18:08,  2.82it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97049it [11:18:28,  2.81it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97099it [11:18:49,  2.74it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97149it [11:19:09,  2.82it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97199it [11:19:29,  2.71it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97249it [11:19:49,  2.84it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97299it [11:20:09,  2.85it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97349it [11:20:29,  2.74it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97399it [11:20:49,  3.00it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97449it [11:21:09,  2.75it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97499it [11:21:29,  2.84it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97549it [11:21:49,  2.75it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97599it [11:22:09,  2.78it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97649it [11:22:29,  2.74it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97699it [11:22:49,  2.74it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97749it [11:23:09,  2.78it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97799it [11:23:29,  2.85it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97849it [11:23:49,  2.75it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97899it [11:24:10,  2.79it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97949it [11:24:30,  2.73it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 97999it [11:24:50,  2.99it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98049it [11:25:10,  2.76it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98099it [11:25:30,  2.94it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98149it [11:25:50,  2.79it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98199it [11:26:10,  2.71it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98249it [11:26:30,  2.84it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98299it [11:26:50,  2.77it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98349it [11:27:10,  2.75it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98399it [11:27:30,  2.81it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98449it [11:27:50,  2.83it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98499it [11:28:10,  2.79it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98549it [11:28:30,  2.70it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98599it [11:28:50,  2.78it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98649it [11:29:10,  2.75it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98699it [11:29:31,  2.44it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98749it [11:29:51,  2.74it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98799it [11:30:11,  2.82it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98849it [11:30:31,  2.91it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98899it [11:30:51,  2.77it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98949it [11:31:11,  2.79it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+Training...: 98999it [11:31:31,  2.78it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+Training...: 98999it [11:31:44,  2.78it/s]████████████████████████████████| 500/500 [00:59<00:00,  7.91it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Training...: 99057it [11:33:09,  1.50it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 99108it [11:33:29,  1.56it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 99158it [11:33:49,  1.54it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 99205it [11:34:09,  1.12s/it]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 99258it [11:34:29,  1.54it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 99306it [11:34:49,  1.25it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 99363it [11:35:09,  3.92it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 99412it [11:35:29,  2.95it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 99461it [11:35:49,  2.84it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 99507it [11:36:09,  1.31it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 99557it [11:36:29,  1.33it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 99612it [11:36:50,  2.91it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 99664it [11:37:10,  3.95it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 99714it [11:37:30,  3.74it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 99764it [11:37:50,  3.80it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 99810it [11:38:10,  2.20it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 99861it [11:38:30,  2.87it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 99914it [11:38:50,  3.97it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 99959it [11:39:10,  1.92it/s]███████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100012it [11:39:30,  3.51it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100062it [11:39:50,  3.65it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100114it [11:40:10,  3.69it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100161it [11:40:30,  2.89it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100211it [11:40:50,  2.91it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100261it [11:41:10,  2.72it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100311it [11:41:30,  2.85it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100361it [11:41:51,  2.95it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100411it [11:42:11,  3.32it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100465it [11:42:31,  4.73it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100511it [11:42:51,  2.87it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100562it [11:43:11,  3.81it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100611it [11:43:31,  2.79it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100661it [11:43:51,  2.77it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100714it [11:44:11,  3.87it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100766it [11:44:31,  5.07it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100815it [11:44:51,  4.63it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100866it [11:45:11,  4.86it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100914it [11:45:31,  4.97it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 100963it [11:45:51,  3.73it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 101013it [11:46:11,  3.74it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 101064it [11:46:31,  3.86it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 101113it [11:46:52,  4.53it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 101167it [11:47:12,  5.88it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 101216it [11:47:32,  4.63it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 101265it [11:47:52,  5.43it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 101314it [11:48:12,  3.76it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 101366it [11:48:32,  5.82it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 101418it [11:48:52,  5.99it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+Training...: 101449it [11:48:58,  2.73it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 101499it [11:49:18,  2.74it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 101549it [11:49:38,  2.81it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 101616it [11:50:12,  5.88it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+Training...: 101649it [11:50:18,  2.77it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 101699it [11:50:38,  2.78it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 101749it [11:50:58,  2.74it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 101815it [11:51:33,  4.75it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+Training...: 101849it [11:51:38,  2.82it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 101899it [11:51:58,  2.79it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 101949it [11:52:18,  2.70it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 101999it [11:52:38,  2.85it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 102049it [11:52:58,  2.90it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 102099it [11:53:18,  2.84it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 102149it [11:53:38,  2.85it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 102199it [11:53:59,  2.72it/s]██████████���███████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 102249it [11:54:19,  2.91it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 102299it [11:54:39,  2.75it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 102349it [11:54:59,  2.74it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 102399it [11:55:19,  2.83it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 102449it [11:55:39,  2.77it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 102499it [11:55:59,  2.79it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 102549it [11:56:19,  2.82it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 102599it [11:56:39,  2.83it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 102649it [11:56:59,  2.89it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 102699it [11:57:19,  2.72it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 102749it [11:57:39,  2.76it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 102822it [11:58:14,  8.59it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+Training...: 102849it [11:58:19,  2.75it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 102899it [11:58:39,  2.80it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 102971it [11:59:14,  9.10it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+Training...: 102999it [11:59:19,  2.72it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 103049it [11:59:40,  2.71it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 103099it [12:00:00,  2.73it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 103149it [12:00:20,  2.80it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 103211it [12:00:54,  2.66it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 103265it [12:01:14,  4.97it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+Training...: 103299it [12:01:20,  2.77it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 103349it [12:01:40,  2.74it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 103399it [12:02:00,  2.73it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 103449it [12:02:20,  2.76it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 103499it [12:02:40,  2.92it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 103549it [12:03:00,  2.73it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 103599it [12:03:20,  2.79it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 103649it [12:03:40,  2.74it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 103699it [12:04:00,  2.74it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 103773it [12:04:35,  9.99it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 103813it [12:04:55,  3.64it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 103866it [12:05:15,  5.04it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 103916it [12:05:35,  4.82it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 103963it [12:05:55,  3.35it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+Training...: 103999it [12:06:01,  2.81it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 104066it [12:06:35,  5.62it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 104114it [12:06:55,  4.89it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+Training...: 104149it [12:07:01,  2.77it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 104199it [12:07:21,  2.80it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 104249it [12:07:41,  2.82it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 104299it [12:08:01,  2.75it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 104349it [12:08:21,  2.80it/s]██████████████████████████��███████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 104399it [12:08:41,  2.76it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 104449it [12:09:01,  2.78it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 104499it [12:09:21,  2.79it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 104549it [12:09:41,  2.72it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 104599it [12:10:01,  2.78it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 104669it [12:10:36,  7.35it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 104716it [12:10:56,  4.88it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 104767it [12:11:16,  6.16it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 104817it [12:11:36,  6.01it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+Training...: 104849it [12:11:42,  2.82it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 104899it [12:12:02,  2.76it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 104949it [12:12:22,  2.76it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+Training...: 104999it [12:12:42,  2.71it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+Training...: 104999it [12:12:55,  2.71it/s]██████████████████████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+[22:19:21] - INFO - absl - Saved checkpoint at checkpoint_330000█████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
+[22:19:22] - INFO - huggingface_hub.repository - git version 2.25.1
+git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)

wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json

@@ -1 +1 @@
-{"training_step": 315000, "learning_rate": 2.4782844775472768e-05, "train_loss": 1.9440785646438599, "_runtime": 38092, "_timestamp": 1626467654, "_step": 1814, "eval_step": 312000, "eval_accuracy": 0.6361072659492493, "eval_loss": 1.8521945476531982}
+{"training_step": 330000, "learning_rate": 2.4526265406166203e-05, "train_loss": 2.006321430206299, "_runtime": 44391, "_timestamp": 1626473953, "_step": 2117, "eval_step": 330000, "eval_accuracy": 0.6395835280418396, "eval_loss": 1.8301599025726318}