Saving weights and logs of step 50000
Browse files- checkpoint_50000 +1 -1
- events.out.tfevents.1636287687.t1v-n-ba7aa1a5-w-0.13180.0.v2 +2 -2
- events.out.tfevents.1636318087.t1v-n-ba7aa1a5-w-0.20231.0.v2 +3 -0
- flax_model.msgpack +1 -1
- muril-bigbird/flax_model.msgpack +1 -1
- run_muril_bb_flax.sh +5 -5
- run_muril_bb_flax_mlm.py +3 -3
- untrained_flax_model.msgpack +3 -0
- wandb/debug-internal.log +1 -1
- wandb/debug.log +1 -1
- wandb/latest-run +1 -1
- wandb/run-20211107_122127-18mmtpt3/files/config.yaml +6 -0
- wandb/run-20211107_122127-18mmtpt3/files/output.log +2447 -0
- wandb/run-20211107_122127-18mmtpt3/files/wandb-summary.json +1 -1
- wandb/run-20211107_122127-18mmtpt3/logs/debug-internal.log +0 -0
- wandb/run-20211107_122127-18mmtpt3/logs/debug.log +98 -0
- wandb/run-20211107_122127-18mmtpt3/run-18mmtpt3.wandb +0 -0
- wandb/run-20211107_204808-2ns3lo5n/files/config.yaml +341 -0
- wandb/run-20211107_204808-2ns3lo5n/files/output.log +98 -0
- wandb/run-20211107_204808-2ns3lo5n/files/requirements.txt +101 -0
- wandb/run-20211107_204808-2ns3lo5n/files/wandb-metadata.json +67 -0
- wandb/run-20211107_204808-2ns3lo5n/files/wandb-summary.json +1 -0
- wandb/run-20211107_204808-2ns3lo5n/logs/debug-internal.log +226 -0
- wandb/run-20211107_204808-2ns3lo5n/logs/debug.log +25 -0
- wandb/run-20211107_204808-2ns3lo5n/run-2ns3lo5n.wandb +0 -0
checkpoint_50000
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 2886112612
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:b4b3bf816e2cbc7650c0a002c813cc9fc134e3d1aed78679a7f08cc2e70bc304
|
3 |
size 2886112612
|
events.out.tfevents.1636287687.t1v-n-ba7aa1a5-w-0.13180.0.v2
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:acdd1d6f4159c76941ac48d6cec8f0992e5c4de06d320d2b71a8a9ef5b2353ae
|
3 |
+
size 8911911
|
events.out.tfevents.1636318087.t1v-n-ba7aa1a5-w-0.20231.0.v2
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:ff6e8edb09c1a4da70a68ce6e234d81e713fac71cb512923e146133875bc8d13
|
3 |
+
size 401
|
flax_model.msgpack
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 962037499
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:9cac41482764ea6b546c23005d6f0fc5447d39960bb9edbe4a412eaef5f6ebee
|
3 |
size 962037499
|
muril-bigbird/flax_model.msgpack
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 962037499
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:92b4eb4b41a4fdd29fa296ca969236da8c1c0c923dab34bfff0448484f2a8228
|
3 |
size 962037499
|
run_muril_bb_flax.sh
CHANGED
@@ -13,16 +13,16 @@ python run_muril_bb_flax_mlm.py \
|
|
13 |
--overwrite_output_dir \
|
14 |
--do_train \
|
15 |
--do_eval \
|
16 |
-
--eval_steps
|
17 |
--per_device_train_batch_size 1 \
|
18 |
--per_device_eval_batch_size 2 \
|
19 |
-
--learning_rate
|
20 |
--weight_decay 0.01 \
|
21 |
--num_train_epochs 3 \
|
22 |
-
--warmup_steps
|
23 |
-
--logging_steps
|
24 |
--save_steps 50000 \
|
25 |
--save_total_limit 1 \
|
26 |
--seed 18 \
|
27 |
--push_to_hub \
|
28 |
-
--run_name "en-ta-hi-te"
|
|
|
13 |
--overwrite_output_dir \
|
14 |
--do_train \
|
15 |
--do_eval \
|
16 |
+
--eval_steps 5000 \
|
17 |
--per_device_train_batch_size 1 \
|
18 |
--per_device_eval_batch_size 2 \
|
19 |
+
--learning_rate 5e-5 \
|
20 |
--weight_decay 0.01 \
|
21 |
--num_train_epochs 3 \
|
22 |
+
--warmup_steps 0 \
|
23 |
+
--logging_steps 500 \
|
24 |
--save_steps 50000 \
|
25 |
--save_total_limit 1 \
|
26 |
--seed 18 \
|
27 |
--push_to_hub \
|
28 |
+
--run_name "en-ta-hi-te-from-50k"
|
run_muril_bb_flax_mlm.py
CHANGED
@@ -343,10 +343,10 @@ if __name__ == "__main__":
|
|
343 |
files = p.glob("*/*.parquet")
|
344 |
files = [str(x) for x in files]
|
345 |
random.shuffle(files)
|
346 |
-
files = files[:
|
347 |
datasets= load_dataset("parquet", data_files=files, split="train", cache_dir="/dev/shm")
|
348 |
datasets = datasets.shuffle(seed=training_args.seed)
|
349 |
-
datasets = datasets.train_test_split(test_size = 0.
|
350 |
datasets["validation"] = datasets["test"]
|
351 |
|
352 |
elif data_args.dataset_name is not None:
|
@@ -612,7 +612,7 @@ if __name__ == "__main__":
|
|
612 |
state = restore_checkpoint(training_args.resume_from_checkpoint, state)
|
613 |
resume_step = mb_item(state.step.item())
|
614 |
else:
|
615 |
-
resume_step =
|
616 |
|
617 |
# Define gradient update step fn
|
618 |
def train_step(state, batch, dropout_rng):
|
|
|
343 |
files = p.glob("*/*.parquet")
|
344 |
files = [str(x) for x in files]
|
345 |
random.shuffle(files)
|
346 |
+
files = files[50:100]
|
347 |
datasets= load_dataset("parquet", data_files=files, split="train", cache_dir="/dev/shm")
|
348 |
datasets = datasets.shuffle(seed=training_args.seed)
|
349 |
+
datasets = datasets.train_test_split(test_size = 0.02, seed=training_args.seed)
|
350 |
datasets["validation"] = datasets["test"]
|
351 |
|
352 |
elif data_args.dataset_name is not None:
|
|
|
612 |
state = restore_checkpoint(training_args.resume_from_checkpoint, state)
|
613 |
resume_step = mb_item(state.step.item())
|
614 |
else:
|
615 |
+
resume_step = 50000
|
616 |
|
617 |
# Define gradient update step fn
|
618 |
def train_step(state, batch, dropout_rng):
|
untrained_flax_model.msgpack
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:0b4ee04dcd8c60be62794b6e99438b6a9caf8cd063f11f69236b0652ad6e1dd9
|
3 |
+
size 962037499
|
wandb/debug-internal.log
CHANGED
@@ -1 +1 @@
|
|
1 |
-
run-
|
|
|
1 |
+
run-20211107_204808-2ns3lo5n/logs/debug-internal.log
|
wandb/debug.log
CHANGED
@@ -1 +1 @@
|
|
1 |
-
run-
|
|
|
1 |
+
run-20211107_204808-2ns3lo5n/logs/debug.log
|
wandb/latest-run
CHANGED
@@ -1 +1 @@
|
|
1 |
-
run-
|
|
|
1 |
+
run-20211107_204808-2ns3lo5n
|
wandb/run-20211107_122127-18mmtpt3/files/config.yaml
CHANGED
@@ -23,6 +23,12 @@ _wandb:
|
|
23 |
- 3
|
24 |
- 11
|
25 |
- 12
|
|
|
|
|
|
|
|
|
|
|
|
|
26 |
3:
|
27 |
- 13
|
28 |
4: 3.8.10
|
|
|
23 |
- 3
|
24 |
- 11
|
25 |
- 12
|
26 |
+
2:
|
27 |
+
- 1
|
28 |
+
- 2
|
29 |
+
- 3
|
30 |
+
- 11
|
31 |
+
- 12
|
32 |
3:
|
33 |
- 13
|
34 |
4: 3.8.10
|
wandb/run-20211107_122127-18mmtpt3/files/output.log
CHANGED
@@ -11989,3 +11989,2450 @@ tcmalloc: large alloc 3161391104 bytes == 0x44822a000 @ 0x7f2a10a2f680 0x7f2a10
|
|
11989 |
tcmalloc: large alloc 2886115328 bytes == 0x392038000 @ 0x7f2a10a2f680 0x7f2a10a50824 0x5f9f21 0x7f286173ec6f 0x7f286173f1fe 0x5042c6 0x56bf09 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x5f6343 0x56cf2a 0x5f6166 0x570e46 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x68d5b7 0x67cd01 0x67cd7f 0x67ce21 0x67ef47 0x6b7242 0x6b75cd 0x7f2a108460b3 0x5fb18e
|
11990 |
[19:17:43] - INFO - absl - Saved checkpoint at ./checkpoint_50000
|
11991 |
Adding files tracked by Git LFS: ['checkpoint_50000']. This may take a bit of time if the files are large.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
11989 |
tcmalloc: large alloc 2886115328 bytes == 0x392038000 @ 0x7f2a10a2f680 0x7f2a10a50824 0x5f9f21 0x7f286173ec6f 0x7f286173f1fe 0x5042c6 0x56bf09 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x5f6343 0x56cf2a 0x5f6166 0x570e46 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x68d5b7 0x67cd01 0x67cd7f 0x67ce21 0x67ef47 0x6b7242 0x6b75cd 0x7f2a108460b3 0x5fb18e
|
11990 |
[19:17:43] - INFO - absl - Saved checkpoint at ./checkpoint_50000
|
11991 |
Adding files tracked by Git LFS: ['checkpoint_50000']. This may take a bit of time if the files are large.
|
11992 |
+
[19:17:48] - WARNING - huggingface_hub.repository - Adding files tracked by Git LFS: ['checkpoint_50000']. This may take a bit of time if the files are large.
|
11993 |
+
|
11994 |
+
|
11995 |
+
|
11996 |
+
|
11997 |
+
|
11998 |
+
|
11999 |
+
|
12000 |
+
|
12001 |
+
|
12002 |
+
|
12003 |
+
|
12004 |
+
|
12005 |
+
|
12006 |
+
|
12007 |
+
|
12008 |
+
|
12009 |
+
|
12010 |
+
|
12011 |
+
|
12012 |
+
|
12013 |
+
|
12014 |
+
|
12015 |
+
|
12016 |
+
|
12017 |
+
|
12018 |
+
|
12019 |
+
|
12020 |
+
|
12021 |
+
|
12022 |
+
|
12023 |
+
|
12024 |
+
|
12025 |
+
|
12026 |
+
|
12027 |
+
|
12028 |
+
|
12029 |
+
|
12030 |
+
|
12031 |
+
|
12032 |
+
|
12033 |
+
|
12034 |
+
|
12035 |
+
|
12036 |
+
|
12037 |
+
|
12038 |
+
|
12039 |
+
|
12040 |
+
|
12041 |
+
|
12042 |
+
|
12043 |
+
|
12044 |
+
|
12045 |
+
|
12046 |
+
|
12047 |
+
|
12048 |
+
|
12049 |
+
|
12050 |
+
|
12051 |
+
|
12052 |
+
|
12053 |
+
|
12054 |
+
|
12055 |
+
|
12056 |
+
|
12057 |
+
|
12058 |
+
|
12059 |
+
|
12060 |
+
|
12061 |
+
|
12062 |
+
|
12063 |
+
|
12064 |
+
|
12065 |
+
|
12066 |
+
|
12067 |
+
|
12068 |
+
|
12069 |
+
|
12070 |
+
|
12071 |
+
|
12072 |
+
|
12073 |
+
|
12074 |
+
|
12075 |
+
|
12076 |
+
|
12077 |
+
|
12078 |
+
|
12079 |
+
|
12080 |
+
|
12081 |
+
|
12082 |
+
|
12083 |
+
|
12084 |
+
|
12085 |
+
|
12086 |
+
|
12087 |
+
|
12088 |
+
|
12089 |
+
|
12090 |
+
|
12091 |
+
|
12092 |
+
|
12093 |
+
|
12094 |
+
|
12095 |
+
|
12096 |
+
|
12097 |
+
|
12098 |
+
|
12099 |
+
|
12100 |
+
|
12101 |
+
|
12102 |
+
|
12103 |
+
|
12104 |
+
|
12105 |
+
|
12106 |
+
|
12107 |
+
|
12108 |
+
|
12109 |
+
|
12110 |
+
|
12111 |
+
|
12112 |
+
|
12113 |
+
|
12114 |
+
|
12115 |
+
|
12116 |
+
|
12117 |
+
|
12118 |
+
|
12119 |
+
|
12120 |
+
|
12121 |
+
|
12122 |
+
|
12123 |
+
|
12124 |
+
|
12125 |
+
|
12126 |
+
|
12127 |
+
|
12128 |
+
|
12129 |
+
|
12130 |
+
|
12131 |
+
|
12132 |
+
|
12133 |
+
|
12134 |
+
|
12135 |
+
|
12136 |
+
|
12137 |
+
|
12138 |
+
|
12139 |
+
|
12140 |
+
|
12141 |
+
|
12142 |
+
|
12143 |
+
|
12144 |
+
|
12145 |
+
|
12146 |
+
|
12147 |
+
|
12148 |
+
|
12149 |
+
|
12150 |
+
|
12151 |
+
|
12152 |
+
|
12153 |
+
|
12154 |
+
|
12155 |
+
|
12156 |
+
|
12157 |
+
|
12158 |
+
|
12159 |
+
|
12160 |
+
|
12161 |
+
|
12162 |
+
|
12163 |
+
|
12164 |
+
|
12165 |
+
|
12166 |
+
|
12167 |
+
|
12168 |
+
|
12169 |
+
|
12170 |
+
|
12171 |
+
|
12172 |
+
|
12173 |
+
|
12174 |
+
|
12175 |
+
|
12176 |
+
|
12177 |
+
|
12178 |
+
|
12179 |
+
|
12180 |
+
|
12181 |
+
|
12182 |
+
|
12183 |
+
|
12184 |
+
|
12185 |
+
|
12186 |
+
|
12187 |
+
|
12188 |
+
|
12189 |
+
|
12190 |
+
|
12191 |
+
|
12192 |
+
|
12193 |
+
|
12194 |
+
|
12195 |
+
|
12196 |
+
|
12197 |
+
|
12198 |
+
|
12199 |
+
|
12200 |
+
|
12201 |
+
|
12202 |
+
|
12203 |
+
|
12204 |
+
|
12205 |
+
|
12206 |
+
|
12207 |
+
|
12208 |
+
|
12209 |
+
|
12210 |
+
|
12211 |
+
|
12212 |
+
|
12213 |
+
|
12214 |
+
|
12215 |
+
|
12216 |
+
|
12217 |
+
|
12218 |
+
|
12219 |
+
|
12220 |
+
|
12221 |
+
|
12222 |
+
|
12223 |
+
|
12224 |
+
|
12225 |
+
|
12226 |
+
|
12227 |
+
Training...: 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 51015/60729 [7:03:33<48:19, 3.35it/s]
|
12228 |
+
|
12229 |
+
|
12230 |
+
|
12231 |
+
|
12232 |
+
|
12233 |
+
|
12234 |
+
|
12235 |
+
|
12236 |
+
|
12237 |
+
|
12238 |
+
|
12239 |
+
|
12240 |
+
|
12241 |
+
|
12242 |
+
|
12243 |
+
|
12244 |
+
|
12245 |
+
|
12246 |
+
|
12247 |
+
|
12248 |
+
|
12249 |
+
|
12250 |
+
|
12251 |
+
|
12252 |
+
|
12253 |
+
|
12254 |
+
|
12255 |
+
|
12256 |
+
|
12257 |
+
|
12258 |
+
|
12259 |
+
|
12260 |
+
|
12261 |
+
|
12262 |
+
|
12263 |
+
|
12264 |
+
|
12265 |
+
|
12266 |
+
|
12267 |
+
|
12268 |
+
|
12269 |
+
|
12270 |
+
|
12271 |
+
|
12272 |
+
|
12273 |
+
|
12274 |
+
|
12275 |
+
|
12276 |
+
|
12277 |
+
|
12278 |
+
|
12279 |
+
|
12280 |
+
|
12281 |
+
|
12282 |
+
|
12283 |
+
|
12284 |
+
|
12285 |
+
|
12286 |
+
|
12287 |
+
|
12288 |
+
|
12289 |
+
|
12290 |
+
|
12291 |
+
|
12292 |
+
|
12293 |
+
|
12294 |
+
|
12295 |
+
|
12296 |
+
|
12297 |
+
|
12298 |
+
|
12299 |
+
|
12300 |
+
|
12301 |
+
|
12302 |
+
|
12303 |
+
|
12304 |
+
|
12305 |
+
|
12306 |
+
|
12307 |
+
|
12308 |
+
|
12309 |
+
|
12310 |
+
|
12311 |
+
|
12312 |
+
|
12313 |
+
|
12314 |
+
|
12315 |
+
|
12316 |
+
|
12317 |
+
|
12318 |
+
|
12319 |
+
|
12320 |
+
|
12321 |
+
|
12322 |
+
|
12323 |
+
|
12324 |
+
|
12325 |
+
|
12326 |
+
|
12327 |
+
|
12328 |
+
|
12329 |
+
|
12330 |
+
|
12331 |
+
|
12332 |
+
|
12333 |
+
|
12334 |
+
|
12335 |
+
|
12336 |
+
|
12337 |
+
|
12338 |
+
|
12339 |
+
|
12340 |
+
|
12341 |
+
|
12342 |
+
|
12343 |
+
|
12344 |
+
|
12345 |
+
|
12346 |
+
|
12347 |
+
|
12348 |
+
|
12349 |
+
|
12350 |
+
|
12351 |
+
|
12352 |
+
|
12353 |
+
|
12354 |
+
|
12355 |
+
|
12356 |
+
|
12357 |
+
|
12358 |
+
|
12359 |
+
|
12360 |
+
|
12361 |
+
|
12362 |
+
|
12363 |
+
|
12364 |
+
|
12365 |
+
|
12366 |
+
|
12367 |
+
|
12368 |
+
|
12369 |
+
|
12370 |
+
|
12371 |
+
|
12372 |
+
|
12373 |
+
|
12374 |
+
|
12375 |
+
|
12376 |
+
|
12377 |
+
|
12378 |
+
|
12379 |
+
|
12380 |
+
|
12381 |
+
|
12382 |
+
|
12383 |
+
|
12384 |
+
|
12385 |
+
|
12386 |
+
|
12387 |
+
|
12388 |
+
|
12389 |
+
|
12390 |
+
|
12391 |
+
|
12392 |
+
|
12393 |
+
|
12394 |
+
|
12395 |
+
|
12396 |
+
|
12397 |
+
|
12398 |
+
|
12399 |
+
|
12400 |
+
|
12401 |
+
|
12402 |
+
|
12403 |
+
|
12404 |
+
|
12405 |
+
|
12406 |
+
|
12407 |
+
|
12408 |
+
|
12409 |
+
|
12410 |
+
|
12411 |
+
|
12412 |
+
|
12413 |
+
|
12414 |
+
|
12415 |
+
|
12416 |
+
|
12417 |
+
|
12418 |
+
|
12419 |
+
|
12420 |
+
|
12421 |
+
|
12422 |
+
|
12423 |
+
|
12424 |
+
|
12425 |
+
|
12426 |
+
|
12427 |
+
|
12428 |
+
|
12429 |
+
|
12430 |
+
|
12431 |
+
|
12432 |
+
|
12433 |
+
|
12434 |
+
|
12435 |
+
|
12436 |
+
|
12437 |
+
|
12438 |
+
|
12439 |
+
|
12440 |
+
|
12441 |
+
|
12442 |
+
|
12443 |
+
|
12444 |
+
|
12445 |
+
|
12446 |
+
|
12447 |
+
|
12448 |
+
|
12449 |
+
|
12450 |
+
|
12451 |
+
|
12452 |
+
|
12453 |
+
|
12454 |
+
|
12455 |
+
|
12456 |
+
|
12457 |
+
|
12458 |
+
|
12459 |
+
|
12460 |
+
|
12461 |
+
|
12462 |
+
Training...: 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 52000/60729 [7:11:21<1:10:34, 2.06it/s]
|
12463 |
+
|
12464 |
+
|
12465 |
+
|
12466 |
+
|
12467 |
+
|
12468 |
+
|
12469 |
+
|
12470 |
+
|
12471 |
+
|
12472 |
+
|
12473 |
+
|
12474 |
+
|
12475 |
+
|
12476 |
+
|
12477 |
+
|
12478 |
+
|
12479 |
+
|
12480 |
+
|
12481 |
+
|
12482 |
+
|
12483 |
+
|
12484 |
+
|
12485 |
+
|
12486 |
+
|
12487 |
+
|
12488 |
+
|
12489 |
+
|
12490 |
+
|
12491 |
+
|
12492 |
+
|
12493 |
+
|
12494 |
+
|
12495 |
+
|
12496 |
+
|
12497 |
+
|
12498 |
+
|
12499 |
+
|
12500 |
+
|
12501 |
+
|
12502 |
+
|
12503 |
+
|
12504 |
+
|
12505 |
+
|
12506 |
+
|
12507 |
+
|
12508 |
+
|
12509 |
+
|
12510 |
+
|
12511 |
+
|
12512 |
+
|
12513 |
+
|
12514 |
+
|
12515 |
+
|
12516 |
+
|
12517 |
+
|
12518 |
+
|
12519 |
+
|
12520 |
+
|
12521 |
+
|
12522 |
+
|
12523 |
+
|
12524 |
+
|
12525 |
+
|
12526 |
+
|
12527 |
+
|
12528 |
+
|
12529 |
+
|
12530 |
+
|
12531 |
+
|
12532 |
+
|
12533 |
+
|
12534 |
+
|
12535 |
+
|
12536 |
+
|
12537 |
+
|
12538 |
+
|
12539 |
+
|
12540 |
+
|
12541 |
+
|
12542 |
+
|
12543 |
+
|
12544 |
+
|
12545 |
+
|
12546 |
+
|
12547 |
+
|
12548 |
+
|
12549 |
+
|
12550 |
+
|
12551 |
+
|
12552 |
+
|
12553 |
+
|
12554 |
+
|
12555 |
+
|
12556 |
+
|
12557 |
+
|
12558 |
+
|
12559 |
+
|
12560 |
+
|
12561 |
+
|
12562 |
+
|
12563 |
+
|
12564 |
+
|
12565 |
+
|
12566 |
+
|
12567 |
+
|
12568 |
+
|
12569 |
+
|
12570 |
+
|
12571 |
+
|
12572 |
+
|
12573 |
+
|
12574 |
+
|
12575 |
+
|
12576 |
+
|
12577 |
+
|
12578 |
+
|
12579 |
+
|
12580 |
+
|
12581 |
+
|
12582 |
+
|
12583 |
+
|
12584 |
+
|
12585 |
+
|
12586 |
+
|
12587 |
+
|
12588 |
+
|
12589 |
+
|
12590 |
+
|
12591 |
+
|
12592 |
+
|
12593 |
+
|
12594 |
+
|
12595 |
+
|
12596 |
+
|
12597 |
+
|
12598 |
+
|
12599 |
+
|
12600 |
+
|
12601 |
+
|
12602 |
+
|
12603 |
+
|
12604 |
+
|
12605 |
+
|
12606 |
+
|
12607 |
+
|
12608 |
+
|
12609 |
+
|
12610 |
+
|
12611 |
+
|
12612 |
+
|
12613 |
+
|
12614 |
+
|
12615 |
+
|
12616 |
+
|
12617 |
+
|
12618 |
+
|
12619 |
+
|
12620 |
+
|
12621 |
+
|
12622 |
+
|
12623 |
+
|
12624 |
+
|
12625 |
+
|
12626 |
+
|
12627 |
+
|
12628 |
+
|
12629 |
+
|
12630 |
+
|
12631 |
+
|
12632 |
+
|
12633 |
+
|
12634 |
+
|
12635 |
+
|
12636 |
+
|
12637 |
+
|
12638 |
+
|
12639 |
+
|
12640 |
+
|
12641 |
+
|
12642 |
+
|
12643 |
+
|
12644 |
+
|
12645 |
+
|
12646 |
+
|
12647 |
+
|
12648 |
+
|
12649 |
+
|
12650 |
+
|
12651 |
+
|
12652 |
+
|
12653 |
+
|
12654 |
+
|
12655 |
+
|
12656 |
+
|
12657 |
+
|
12658 |
+
|
12659 |
+
|
12660 |
+
|
12661 |
+
|
12662 |
+
|
12663 |
+
|
12664 |
+
|
12665 |
+
|
12666 |
+
|
12667 |
+
|
12668 |
+
|
12669 |
+
|
12670 |
+
|
12671 |
+
|
12672 |
+
|
12673 |
+
|
12674 |
+
|
12675 |
+
|
12676 |
+
|
12677 |
+
|
12678 |
+
|
12679 |
+
|
12680 |
+
|
12681 |
+
|
12682 |
+
|
12683 |
+
|
12684 |
+
|
12685 |
+
|
12686 |
+
|
12687 |
+
|
12688 |
+
|
12689 |
+
|
12690 |
+
|
12691 |
+
|
12692 |
+
|
12693 |
+
|
12694 |
+
|
12695 |
+
|
12696 |
+
|
12697 |
+
|
12698 |
+
|
12699 |
+
Training...: 87%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍ | 53000/60729 [7:19:27<1:02:28, 2.06it/s]
|
12700 |
+
Step... (53000 | Loss: 1.6916450262069702, Learning Rate: 7.502715016016737e-05)
|
12701 |
+
Step... (45000 | Loss: 1.4846699237823486, Acc: 0.711020290851593): 0%| | 0/3 [7:20:51<?, ?it/s]
|
12702 |
+
|
12703 |
+
|
12704 |
+
|
12705 |
+
|
12706 |
+
|
12707 |
+
|
12708 |
+
|
12709 |
+
|
12710 |
+
|
12711 |
+
|
12712 |
+
|
12713 |
+
|
12714 |
+
|
12715 |
+
|
12716 |
+
|
12717 |
+
|
12718 |
+
|
12719 |
+
|
12720 |
+
|
12721 |
+
|
12722 |
+
|
12723 |
+
|
12724 |
+
|
12725 |
+
|
12726 |
+
|
12727 |
+
|
12728 |
+
|
12729 |
+
|
12730 |
+
|
12731 |
+
|
12732 |
+
|
12733 |
+
|
12734 |
+
|
12735 |
+
|
12736 |
+
|
12737 |
+
|
12738 |
+
|
12739 |
+
|
12740 |
+
|
12741 |
+
|
12742 |
+
|
12743 |
+
|
12744 |
+
|
12745 |
+
|
12746 |
+
|
12747 |
+
|
12748 |
+
|
12749 |
+
|
12750 |
+
|
12751 |
+
|
12752 |
+
|
12753 |
+
|
12754 |
+
|
12755 |
+
|
12756 |
+
|
12757 |
+
|
12758 |
+
|
12759 |
+
|
12760 |
+
|
12761 |
+
|
12762 |
+
|
12763 |
+
|
12764 |
+
|
12765 |
+
|
12766 |
+
|
12767 |
+
|
12768 |
+
|
12769 |
+
|
12770 |
+
|
12771 |
+
|
12772 |
+
|
12773 |
+
|
12774 |
+
|
12775 |
+
|
12776 |
+
|
12777 |
+
|
12778 |
+
|
12779 |
+
|
12780 |
+
|
12781 |
+
|
12782 |
+
|
12783 |
+
|
12784 |
+
|
12785 |
+
|
12786 |
+
|
12787 |
+
|
12788 |
+
|
12789 |
+
|
12790 |
+
|
12791 |
+
|
12792 |
+
|
12793 |
+
|
12794 |
+
|
12795 |
+
|
12796 |
+
|
12797 |
+
|
12798 |
+
|
12799 |
+
|
12800 |
+
|
12801 |
+
|
12802 |
+
|
12803 |
+
|
12804 |
+
|
12805 |
+
|
12806 |
+
|
12807 |
+
|
12808 |
+
|
12809 |
+
|
12810 |
+
|
12811 |
+
|
12812 |
+
|
12813 |
+
|
12814 |
+
|
12815 |
+
|
12816 |
+
|
12817 |
+
|
12818 |
+
|
12819 |
+
|
12820 |
+
|
12821 |
+
|
12822 |
+
|
12823 |
+
|
12824 |
+
|
12825 |
+
|
12826 |
+
|
12827 |
+
|
12828 |
+
|
12829 |
+
|
12830 |
+
|
12831 |
+
|
12832 |
+
|
12833 |
+
|
12834 |
+
|
12835 |
+
|
12836 |
+
|
12837 |
+
|
12838 |
+
|
12839 |
+
|
12840 |
+
|
12841 |
+
|
12842 |
+
|
12843 |
+
|
12844 |
+
|
12845 |
+
|
12846 |
+
|
12847 |
+
|
12848 |
+
|
12849 |
+
|
12850 |
+
|
12851 |
+
|
12852 |
+
|
12853 |
+
|
12854 |
+
|
12855 |
+
|
12856 |
+
|
12857 |
+
|
12858 |
+
|
12859 |
+
|
12860 |
+
|
12861 |
+
|
12862 |
+
|
12863 |
+
|
12864 |
+
|
12865 |
+
|
12866 |
+
|
12867 |
+
|
12868 |
+
|
12869 |
+
|
12870 |
+
|
12871 |
+
|
12872 |
+
|
12873 |
+
|
12874 |
+
|
12875 |
+
|
12876 |
+
|
12877 |
+
|
12878 |
+
|
12879 |
+
|
12880 |
+
|
12881 |
+
|
12882 |
+
|
12883 |
+
|
12884 |
+
|
12885 |
+
|
12886 |
+
|
12887 |
+
|
12888 |
+
|
12889 |
+
|
12890 |
+
|
12891 |
+
|
12892 |
+
|
12893 |
+
|
12894 |
+
|
12895 |
+
|
12896 |
+
|
12897 |
+
|
12898 |
+
|
12899 |
+
|
12900 |
+
|
12901 |
+
|
12902 |
+
|
12903 |
+
|
12904 |
+
|
12905 |
+
|
12906 |
+
|
12907 |
+
|
12908 |
+
|
12909 |
+
|
12910 |
+
|
12911 |
+
|
12912 |
+
|
12913 |
+
|
12914 |
+
|
12915 |
+
|
12916 |
+
|
12917 |
+
|
12918 |
+
|
12919 |
+
|
12920 |
+
|
12921 |
+
|
12922 |
+
|
12923 |
+
|
12924 |
+
|
12925 |
+
|
12926 |
+
|
12927 |
+
|
12928 |
+
|
12929 |
+
|
12930 |
+
|
12931 |
+
|
12932 |
+
|
12933 |
+
|
12934 |
+
|
12935 |
+
Training...: 89%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 54000/60729 [7:27:34<54:23, 2.06it/s]
|
12936 |
+
|
12937 |
+
|
12938 |
+
|
12939 |
+
|
12940 |
+
|
12941 |
+
|
12942 |
+
|
12943 |
+
|
12944 |
+
|
12945 |
+
|
12946 |
+
|
12947 |
+
|
12948 |
+
|
12949 |
+
|
12950 |
+
|
12951 |
+
|
12952 |
+
|
12953 |
+
|
12954 |
+
|
12955 |
+
|
12956 |
+
|
12957 |
+
|
12958 |
+
|
12959 |
+
|
12960 |
+
|
12961 |
+
|
12962 |
+
|
12963 |
+
|
12964 |
+
|
12965 |
+
|
12966 |
+
|
12967 |
+
|
12968 |
+
|
12969 |
+
|
12970 |
+
|
12971 |
+
|
12972 |
+
|
12973 |
+
|
12974 |
+
|
12975 |
+
|
12976 |
+
|
12977 |
+
|
12978 |
+
|
12979 |
+
|
12980 |
+
|
12981 |
+
|
12982 |
+
|
12983 |
+
|
12984 |
+
|
12985 |
+
|
12986 |
+
|
12987 |
+
|
12988 |
+
|
12989 |
+
|
12990 |
+
|
12991 |
+
|
12992 |
+
|
12993 |
+
|
12994 |
+
|
12995 |
+
|
12996 |
+
|
12997 |
+
|
12998 |
+
|
12999 |
+
|
13000 |
+
|
13001 |
+
|
13002 |
+
|
13003 |
+
|
13004 |
+
|
13005 |
+
|
13006 |
+
|
13007 |
+
|
13008 |
+
|
13009 |
+
|
13010 |
+
|
13011 |
+
|
13012 |
+
|
13013 |
+
|
13014 |
+
|
13015 |
+
|
13016 |
+
|
13017 |
+
|
13018 |
+
|
13019 |
+
|
13020 |
+
|
13021 |
+
|
13022 |
+
|
13023 |
+
|
13024 |
+
|
13025 |
+
|
13026 |
+
|
13027 |
+
|
13028 |
+
|
13029 |
+
|
13030 |
+
|
13031 |
+
|
13032 |
+
|
13033 |
+
|
13034 |
+
|
13035 |
+
|
13036 |
+
|
13037 |
+
|
13038 |
+
|
13039 |
+
|
13040 |
+
|
13041 |
+
|
13042 |
+
|
13043 |
+
|
13044 |
+
|
13045 |
+
|
13046 |
+
|
13047 |
+
|
13048 |
+
|
13049 |
+
|
13050 |
+
|
13051 |
+
|
13052 |
+
|
13053 |
+
|
13054 |
+
|
13055 |
+
|
13056 |
+
|
13057 |
+
|
13058 |
+
|
13059 |
+
|
13060 |
+
|
13061 |
+
|
13062 |
+
|
13063 |
+
|
13064 |
+
|
13065 |
+
|
13066 |
+
|
13067 |
+
|
13068 |
+
|
13069 |
+
|
13070 |
+
|
13071 |
+
|
13072 |
+
|
13073 |
+
|
13074 |
+
|
13075 |
+
|
13076 |
+
|
13077 |
+
|
13078 |
+
|
13079 |
+
|
13080 |
+
|
13081 |
+
|
13082 |
+
|
13083 |
+
|
13084 |
+
|
13085 |
+
|
13086 |
+
|
13087 |
+
|
13088 |
+
|
13089 |
+
|
13090 |
+
|
13091 |
+
|
13092 |
+
|
13093 |
+
|
13094 |
+
|
13095 |
+
|
13096 |
+
|
13097 |
+
|
13098 |
+
|
13099 |
+
|
13100 |
+
|
13101 |
+
|
13102 |
+
|
13103 |
+
|
13104 |
+
|
13105 |
+
|
13106 |
+
|
13107 |
+
|
13108 |
+
|
13109 |
+
|
13110 |
+
|
13111 |
+
|
13112 |
+
|
13113 |
+
|
13114 |
+
|
13115 |
+
|
13116 |
+
|
13117 |
+
|
13118 |
+
|
13119 |
+
|
13120 |
+
|
13121 |
+
|
13122 |
+
|
13123 |
+
|
13124 |
+
|
13125 |
+
|
13126 |
+
|
13127 |
+
|
13128 |
+
|
13129 |
+
|
13130 |
+
|
13131 |
+
|
13132 |
+
|
13133 |
+
|
13134 |
+
|
13135 |
+
|
13136 |
+
|
13137 |
+
|
13138 |
+
|
13139 |
+
|
13140 |
+
|
13141 |
+
|
13142 |
+
|
13143 |
+
|
13144 |
+
|
13145 |
+
|
13146 |
+
|
13147 |
+
|
13148 |
+
|
13149 |
+
|
13150 |
+
|
13151 |
+
|
13152 |
+
|
13153 |
+
|
13154 |
+
|
13155 |
+
|
13156 |
+
|
13157 |
+
|
13158 |
+
|
13159 |
+
|
13160 |
+
|
13161 |
+
|
13162 |
+
|
13163 |
+
|
13164 |
+
|
13165 |
+
|
13166 |
+
|
13167 |
+
|
13168 |
+
|
13169 |
+
|
13170 |
+
|
13171 |
+
|
13172 |
+
Training...: 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 55015/60729 [7:35:58<28:27, 3.35it/s]
|
13173 |
+
|
13174 |
+
|
13175 |
+
|
13176 |
+
|
13177 |
+
|
13178 |
+
|
13179 |
+
|
13180 |
+
|
13181 |
+
|
13182 |
+
|
13183 |
+
|
13184 |
+
|
13185 |
+
|
13186 |
+
|
13187 |
+
|
13188 |
+
|
13189 |
+
|
13190 |
+
|
13191 |
+
|
13192 |
+
|
13193 |
+
|
13194 |
+
|
13195 |
+
|
13196 |
+
|
13197 |
+
|
13198 |
+
|
13199 |
+
|
13200 |
+
|
13201 |
+
|
13202 |
+
|
13203 |
+
|
13204 |
+
|
13205 |
+
|
13206 |
+
|
13207 |
+
|
13208 |
+
|
13209 |
+
|
13210 |
+
|
13211 |
+
|
13212 |
+
|
13213 |
+
|
13214 |
+
|
13215 |
+
|
13216 |
+
|
13217 |
+
|
13218 |
+
|
13219 |
+
|
13220 |
+
|
13221 |
+
|
13222 |
+
|
13223 |
+
|
13224 |
+
|
13225 |
+
|
13226 |
+
|
13227 |
+
|
13228 |
+
|
13229 |
+
|
13230 |
+
|
13231 |
+
|
13232 |
+
|
13233 |
+
|
13234 |
+
|
13235 |
+
|
13236 |
+
|
13237 |
+
|
13238 |
+
|
13239 |
+
|
13240 |
+
|
13241 |
+
|
13242 |
+
|
13243 |
+
|
13244 |
+
|
13245 |
+
|
13246 |
+
|
13247 |
+
|
13248 |
+
|
13249 |
+
|
13250 |
+
|
13251 |
+
|
13252 |
+
|
13253 |
+
|
13254 |
+
|
13255 |
+
|
13256 |
+
|
13257 |
+
|
13258 |
+
|
13259 |
+
|
13260 |
+
|
13261 |
+
|
13262 |
+
|
13263 |
+
|
13264 |
+
|
13265 |
+
|
13266 |
+
|
13267 |
+
|
13268 |
+
|
13269 |
+
|
13270 |
+
|
13271 |
+
|
13272 |
+
|
13273 |
+
|
13274 |
+
|
13275 |
+
|
13276 |
+
|
13277 |
+
|
13278 |
+
|
13279 |
+
|
13280 |
+
|
13281 |
+
|
13282 |
+
|
13283 |
+
|
13284 |
+
|
13285 |
+
|
13286 |
+
|
13287 |
+
|
13288 |
+
|
13289 |
+
|
13290 |
+
|
13291 |
+
|
13292 |
+
|
13293 |
+
|
13294 |
+
|
13295 |
+
|
13296 |
+
|
13297 |
+
|
13298 |
+
|
13299 |
+
|
13300 |
+
|
13301 |
+
|
13302 |
+
|
13303 |
+
|
13304 |
+
|
13305 |
+
|
13306 |
+
|
13307 |
+
|
13308 |
+
|
13309 |
+
|
13310 |
+
|
13311 |
+
|
13312 |
+
|
13313 |
+
|
13314 |
+
|
13315 |
+
|
13316 |
+
|
13317 |
+
|
13318 |
+
|
13319 |
+
|
13320 |
+
|
13321 |
+
|
13322 |
+
|
13323 |
+
|
13324 |
+
|
13325 |
+
|
13326 |
+
|
13327 |
+
|
13328 |
+
|
13329 |
+
|
13330 |
+
|
13331 |
+
|
13332 |
+
|
13333 |
+
|
13334 |
+
|
13335 |
+
|
13336 |
+
|
13337 |
+
|
13338 |
+
|
13339 |
+
|
13340 |
+
|
13341 |
+
|
13342 |
+
|
13343 |
+
|
13344 |
+
|
13345 |
+
|
13346 |
+
|
13347 |
+
|
13348 |
+
|
13349 |
+
|
13350 |
+
|
13351 |
+
|
13352 |
+
|
13353 |
+
|
13354 |
+
|
13355 |
+
|
13356 |
+
|
13357 |
+
|
13358 |
+
|
13359 |
+
|
13360 |
+
|
13361 |
+
|
13362 |
+
|
13363 |
+
|
13364 |
+
|
13365 |
+
|
13366 |
+
|
13367 |
+
|
13368 |
+
|
13369 |
+
|
13370 |
+
|
13371 |
+
|
13372 |
+
|
13373 |
+
|
13374 |
+
|
13375 |
+
|
13376 |
+
|
13377 |
+
|
13378 |
+
|
13379 |
+
|
13380 |
+
|
13381 |
+
|
13382 |
+
|
13383 |
+
|
13384 |
+
|
13385 |
+
|
13386 |
+
|
13387 |
+
|
13388 |
+
|
13389 |
+
|
13390 |
+
|
13391 |
+
|
13392 |
+
|
13393 |
+
|
13394 |
+
|
13395 |
+
|
13396 |
+
|
13397 |
+
|
13398 |
+
|
13399 |
+
|
13400 |
+
|
13401 |
+
|
13402 |
+
|
13403 |
+
|
13404 |
+
|
13405 |
+
|
13406 |
+
|
13407 |
+
Training...: 92%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 56000/60729 [7:43:46<38:13, 2.06it/s]
|
13408 |
+
|
13409 |
+
|
13410 |
+
|
13411 |
+
|
13412 |
+
|
13413 |
+
|
13414 |
+
|
13415 |
+
|
13416 |
+
|
13417 |
+
|
13418 |
+
|
13419 |
+
|
13420 |
+
|
13421 |
+
|
13422 |
+
|
13423 |
+
|
13424 |
+
|
13425 |
+
|
13426 |
+
|
13427 |
+
|
13428 |
+
|
13429 |
+
|
13430 |
+
|
13431 |
+
|
13432 |
+
|
13433 |
+
|
13434 |
+
|
13435 |
+
|
13436 |
+
|
13437 |
+
|
13438 |
+
|
13439 |
+
|
13440 |
+
|
13441 |
+
|
13442 |
+
|
13443 |
+
|
13444 |
+
|
13445 |
+
|
13446 |
+
|
13447 |
+
|
13448 |
+
|
13449 |
+
|
13450 |
+
|
13451 |
+
|
13452 |
+
|
13453 |
+
|
13454 |
+
|
13455 |
+
|
13456 |
+
|
13457 |
+
|
13458 |
+
|
13459 |
+
|
13460 |
+
|
13461 |
+
|
13462 |
+
|
13463 |
+
|
13464 |
+
|
13465 |
+
|
13466 |
+
|
13467 |
+
|
13468 |
+
|
13469 |
+
|
13470 |
+
|
13471 |
+
|
13472 |
+
|
13473 |
+
|
13474 |
+
|
13475 |
+
|
13476 |
+
|
13477 |
+
|
13478 |
+
|
13479 |
+
|
13480 |
+
|
13481 |
+
|
13482 |
+
|
13483 |
+
|
13484 |
+
|
13485 |
+
|
13486 |
+
|
13487 |
+
|
13488 |
+
|
13489 |
+
|
13490 |
+
|
13491 |
+
|
13492 |
+
|
13493 |
+
|
13494 |
+
|
13495 |
+
|
13496 |
+
|
13497 |
+
|
13498 |
+
|
13499 |
+
|
13500 |
+
|
13501 |
+
|
13502 |
+
|
13503 |
+
|
13504 |
+
|
13505 |
+
|
13506 |
+
|
13507 |
+
|
13508 |
+
|
13509 |
+
|
13510 |
+
|
13511 |
+
|
13512 |
+
|
13513 |
+
|
13514 |
+
|
13515 |
+
|
13516 |
+
|
13517 |
+
|
13518 |
+
|
13519 |
+
|
13520 |
+
|
13521 |
+
|
13522 |
+
|
13523 |
+
|
13524 |
+
|
13525 |
+
|
13526 |
+
|
13527 |
+
|
13528 |
+
|
13529 |
+
|
13530 |
+
|
13531 |
+
|
13532 |
+
|
13533 |
+
|
13534 |
+
|
13535 |
+
|
13536 |
+
|
13537 |
+
|
13538 |
+
|
13539 |
+
|
13540 |
+
|
13541 |
+
|
13542 |
+
|
13543 |
+
|
13544 |
+
|
13545 |
+
|
13546 |
+
|
13547 |
+
|
13548 |
+
|
13549 |
+
|
13550 |
+
|
13551 |
+
|
13552 |
+
|
13553 |
+
|
13554 |
+
|
13555 |
+
|
13556 |
+
|
13557 |
+
|
13558 |
+
|
13559 |
+
|
13560 |
+
|
13561 |
+
|
13562 |
+
|
13563 |
+
|
13564 |
+
|
13565 |
+
|
13566 |
+
|
13567 |
+
|
13568 |
+
|
13569 |
+
|
13570 |
+
|
13571 |
+
|
13572 |
+
|
13573 |
+
|
13574 |
+
|
13575 |
+
|
13576 |
+
|
13577 |
+
|
13578 |
+
|
13579 |
+
|
13580 |
+
|
13581 |
+
|
13582 |
+
|
13583 |
+
|
13584 |
+
|
13585 |
+
|
13586 |
+
|
13587 |
+
|
13588 |
+
|
13589 |
+
|
13590 |
+
|
13591 |
+
|
13592 |
+
|
13593 |
+
|
13594 |
+
|
13595 |
+
|
13596 |
+
|
13597 |
+
|
13598 |
+
|
13599 |
+
|
13600 |
+
|
13601 |
+
|
13602 |
+
|
13603 |
+
|
13604 |
+
|
13605 |
+
|
13606 |
+
|
13607 |
+
|
13608 |
+
|
13609 |
+
|
13610 |
+
|
13611 |
+
|
13612 |
+
|
13613 |
+
|
13614 |
+
|
13615 |
+
|
13616 |
+
|
13617 |
+
|
13618 |
+
|
13619 |
+
|
13620 |
+
|
13621 |
+
|
13622 |
+
|
13623 |
+
|
13624 |
+
|
13625 |
+
|
13626 |
+
|
13627 |
+
|
13628 |
+
|
13629 |
+
|
13630 |
+
|
13631 |
+
|
13632 |
+
|
13633 |
+
|
13634 |
+
|
13635 |
+
|
13636 |
+
|
13637 |
+
|
13638 |
+
|
13639 |
+
|
13640 |
+
|
13641 |
+
|
13642 |
+
|
13643 |
+
|
13644 |
+
Training...: 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 57009/60729 [7:52:10<49:44, 1.25it/s]
|
13645 |
+
|
13646 |
+
|
13647 |
+
|
13648 |
+
|
13649 |
+
|
13650 |
+
|
13651 |
+
|
13652 |
+
|
13653 |
+
|
13654 |
+
|
13655 |
+
|
13656 |
+
|
13657 |
+
|
13658 |
+
|
13659 |
+
|
13660 |
+
|
13661 |
+
|
13662 |
+
|
13663 |
+
|
13664 |
+
|
13665 |
+
|
13666 |
+
|
13667 |
+
|
13668 |
+
|
13669 |
+
|
13670 |
+
|
13671 |
+
|
13672 |
+
|
13673 |
+
|
13674 |
+
|
13675 |
+
|
13676 |
+
|
13677 |
+
|
13678 |
+
|
13679 |
+
|
13680 |
+
|
13681 |
+
|
13682 |
+
|
13683 |
+
|
13684 |
+
|
13685 |
+
|
13686 |
+
|
13687 |
+
|
13688 |
+
|
13689 |
+
|
13690 |
+
|
13691 |
+
|
13692 |
+
|
13693 |
+
|
13694 |
+
|
13695 |
+
|
13696 |
+
|
13697 |
+
|
13698 |
+
|
13699 |
+
|
13700 |
+
|
13701 |
+
|
13702 |
+
|
13703 |
+
|
13704 |
+
|
13705 |
+
|
13706 |
+
|
13707 |
+
|
13708 |
+
|
13709 |
+
|
13710 |
+
|
13711 |
+
|
13712 |
+
|
13713 |
+
|
13714 |
+
|
13715 |
+
|
13716 |
+
|
13717 |
+
|
13718 |
+
|
13719 |
+
|
13720 |
+
|
13721 |
+
|
13722 |
+
|
13723 |
+
|
13724 |
+
|
13725 |
+
|
13726 |
+
|
13727 |
+
|
13728 |
+
|
13729 |
+
|
13730 |
+
|
13731 |
+
|
13732 |
+
|
13733 |
+
|
13734 |
+
|
13735 |
+
|
13736 |
+
|
13737 |
+
|
13738 |
+
|
13739 |
+
|
13740 |
+
|
13741 |
+
|
13742 |
+
|
13743 |
+
|
13744 |
+
|
13745 |
+
|
13746 |
+
|
13747 |
+
|
13748 |
+
|
13749 |
+
|
13750 |
+
|
13751 |
+
|
13752 |
+
|
13753 |
+
|
13754 |
+
|
13755 |
+
|
13756 |
+
|
13757 |
+
|
13758 |
+
|
13759 |
+
|
13760 |
+
|
13761 |
+
|
13762 |
+
|
13763 |
+
|
13764 |
+
|
13765 |
+
|
13766 |
+
|
13767 |
+
|
13768 |
+
|
13769 |
+
|
13770 |
+
|
13771 |
+
|
13772 |
+
|
13773 |
+
|
13774 |
+
|
13775 |
+
|
13776 |
+
|
13777 |
+
|
13778 |
+
|
13779 |
+
|
13780 |
+
|
13781 |
+
|
13782 |
+
|
13783 |
+
|
13784 |
+
|
13785 |
+
|
13786 |
+
|
13787 |
+
|
13788 |
+
|
13789 |
+
|
13790 |
+
|
13791 |
+
|
13792 |
+
|
13793 |
+
|
13794 |
+
|
13795 |
+
|
13796 |
+
|
13797 |
+
|
13798 |
+
|
13799 |
+
|
13800 |
+
|
13801 |
+
|
13802 |
+
|
13803 |
+
|
13804 |
+
|
13805 |
+
|
13806 |
+
|
13807 |
+
|
13808 |
+
|
13809 |
+
|
13810 |
+
|
13811 |
+
|
13812 |
+
|
13813 |
+
|
13814 |
+
|
13815 |
+
|
13816 |
+
|
13817 |
+
|
13818 |
+
|
13819 |
+
|
13820 |
+
|
13821 |
+
|
13822 |
+
|
13823 |
+
|
13824 |
+
|
13825 |
+
|
13826 |
+
|
13827 |
+
|
13828 |
+
|
13829 |
+
|
13830 |
+
|
13831 |
+
|
13832 |
+
|
13833 |
+
|
13834 |
+
|
13835 |
+
|
13836 |
+
|
13837 |
+
|
13838 |
+
|
13839 |
+
|
13840 |
+
|
13841 |
+
|
13842 |
+
|
13843 |
+
|
13844 |
+
|
13845 |
+
|
13846 |
+
|
13847 |
+
|
13848 |
+
|
13849 |
+
|
13850 |
+
|
13851 |
+
|
13852 |
+
|
13853 |
+
|
13854 |
+
|
13855 |
+
|
13856 |
+
|
13857 |
+
|
13858 |
+
|
13859 |
+
|
13860 |
+
|
13861 |
+
|
13862 |
+
|
13863 |
+
|
13864 |
+
|
13865 |
+
|
13866 |
+
|
13867 |
+
|
13868 |
+
|
13869 |
+
|
13870 |
+
|
13871 |
+
|
13872 |
+
|
13873 |
+
|
13874 |
+
|
13875 |
+
|
13876 |
+
|
13877 |
+
|
13878 |
+
|
13879 |
+
|
13880 |
+
Training...: 96%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉ | 58023/60729 [8:00:17<05:25, 8.32it/s]
|
13881 |
+
|
13882 |
+
|
13883 |
+
|
13884 |
+
|
13885 |
+
|
13886 |
+
|
13887 |
+
|
13888 |
+
|
13889 |
+
|
13890 |
+
|
13891 |
+
|
13892 |
+
|
13893 |
+
|
13894 |
+
|
13895 |
+
|
13896 |
+
|
13897 |
+
|
13898 |
+
|
13899 |
+
|
13900 |
+
|
13901 |
+
|
13902 |
+
|
13903 |
+
|
13904 |
+
|
13905 |
+
|
13906 |
+
|
13907 |
+
|
13908 |
+
|
13909 |
+
|
13910 |
+
|
13911 |
+
|
13912 |
+
|
13913 |
+
|
13914 |
+
|
13915 |
+
|
13916 |
+
|
13917 |
+
|
13918 |
+
|
13919 |
+
|
13920 |
+
|
13921 |
+
|
13922 |
+
|
13923 |
+
|
13924 |
+
|
13925 |
+
|
13926 |
+
|
13927 |
+
|
13928 |
+
|
13929 |
+
|
13930 |
+
|
13931 |
+
|
13932 |
+
|
13933 |
+
|
13934 |
+
|
13935 |
+
|
13936 |
+
|
13937 |
+
|
13938 |
+
|
13939 |
+
|
13940 |
+
|
13941 |
+
|
13942 |
+
|
13943 |
+
|
13944 |
+
|
13945 |
+
|
13946 |
+
|
13947 |
+
|
13948 |
+
|
13949 |
+
|
13950 |
+
|
13951 |
+
|
13952 |
+
|
13953 |
+
|
13954 |
+
|
13955 |
+
|
13956 |
+
|
13957 |
+
|
13958 |
+
|
13959 |
+
|
13960 |
+
|
13961 |
+
|
13962 |
+
|
13963 |
+
|
13964 |
+
|
13965 |
+
|
13966 |
+
|
13967 |
+
|
13968 |
+
|
13969 |
+
|
13970 |
+
|
13971 |
+
|
13972 |
+
|
13973 |
+
|
13974 |
+
|
13975 |
+
|
13976 |
+
|
13977 |
+
|
13978 |
+
|
13979 |
+
|
13980 |
+
|
13981 |
+
|
13982 |
+
|
13983 |
+
|
13984 |
+
|
13985 |
+
|
13986 |
+
|
13987 |
+
|
13988 |
+
|
13989 |
+
|
13990 |
+
|
13991 |
+
|
13992 |
+
|
13993 |
+
|
13994 |
+
|
13995 |
+
|
13996 |
+
|
13997 |
+
|
13998 |
+
|
13999 |
+
|
14000 |
+
|
14001 |
+
|
14002 |
+
|
14003 |
+
|
14004 |
+
|
14005 |
+
|
14006 |
+
|
14007 |
+
|
14008 |
+
|
14009 |
+
|
14010 |
+
|
14011 |
+
|
14012 |
+
|
14013 |
+
|
14014 |
+
|
14015 |
+
|
14016 |
+
|
14017 |
+
|
14018 |
+
|
14019 |
+
|
14020 |
+
|
14021 |
+
|
14022 |
+
|
14023 |
+
|
14024 |
+
|
14025 |
+
|
14026 |
+
|
14027 |
+
|
14028 |
+
|
14029 |
+
|
14030 |
+
|
14031 |
+
|
14032 |
+
|
14033 |
+
|
14034 |
+
|
14035 |
+
|
14036 |
+
|
14037 |
+
|
14038 |
+
|
14039 |
+
|
14040 |
+
|
14041 |
+
|
14042 |
+
|
14043 |
+
|
14044 |
+
|
14045 |
+
|
14046 |
+
|
14047 |
+
|
14048 |
+
|
14049 |
+
|
14050 |
+
|
14051 |
+
|
14052 |
+
|
14053 |
+
|
14054 |
+
|
14055 |
+
|
14056 |
+
|
14057 |
+
|
14058 |
+
|
14059 |
+
|
14060 |
+
|
14061 |
+
|
14062 |
+
|
14063 |
+
|
14064 |
+
|
14065 |
+
|
14066 |
+
|
14067 |
+
|
14068 |
+
|
14069 |
+
|
14070 |
+
|
14071 |
+
|
14072 |
+
|
14073 |
+
|
14074 |
+
|
14075 |
+
|
14076 |
+
|
14077 |
+
|
14078 |
+
|
14079 |
+
|
14080 |
+
|
14081 |
+
|
14082 |
+
|
14083 |
+
|
14084 |
+
|
14085 |
+
|
14086 |
+
|
14087 |
+
|
14088 |
+
|
14089 |
+
|
14090 |
+
|
14091 |
+
|
14092 |
+
|
14093 |
+
|
14094 |
+
|
14095 |
+
|
14096 |
+
|
14097 |
+
|
14098 |
+
|
14099 |
+
|
14100 |
+
|
14101 |
+
|
14102 |
+
|
14103 |
+
|
14104 |
+
|
14105 |
+
|
14106 |
+
|
14107 |
+
|
14108 |
+
|
14109 |
+
|
14110 |
+
|
14111 |
+
|
14112 |
+
|
14113 |
+
|
14114 |
+
Training...: 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏ | 59000/60729 [8:08:05<13:58, 2.06it/s]
|
14115 |
+
|
14116 |
+
|
14117 |
+
|
14118 |
+
|
14119 |
+
|
14120 |
+
|
14121 |
+
|
14122 |
+
|
14123 |
+
|
14124 |
+
|
14125 |
+
|
14126 |
+
|
14127 |
+
|
14128 |
+
|
14129 |
+
|
14130 |
+
|
14131 |
+
|
14132 |
+
|
14133 |
+
|
14134 |
+
|
14135 |
+
|
14136 |
+
|
14137 |
+
|
14138 |
+
|
14139 |
+
|
14140 |
+
|
14141 |
+
|
14142 |
+
|
14143 |
+
|
14144 |
+
|
14145 |
+
|
14146 |
+
|
14147 |
+
|
14148 |
+
|
14149 |
+
|
14150 |
+
|
14151 |
+
|
14152 |
+
|
14153 |
+
|
14154 |
+
|
14155 |
+
|
14156 |
+
|
14157 |
+
|
14158 |
+
|
14159 |
+
|
14160 |
+
|
14161 |
+
|
14162 |
+
|
14163 |
+
|
14164 |
+
|
14165 |
+
|
14166 |
+
|
14167 |
+
|
14168 |
+
|
14169 |
+
|
14170 |
+
|
14171 |
+
|
14172 |
+
|
14173 |
+
|
14174 |
+
|
14175 |
+
|
14176 |
+
|
14177 |
+
|
14178 |
+
|
14179 |
+
|
14180 |
+
|
14181 |
+
|
14182 |
+
|
14183 |
+
|
14184 |
+
|
14185 |
+
|
14186 |
+
|
14187 |
+
|
14188 |
+
|
14189 |
+
|
14190 |
+
|
14191 |
+
|
14192 |
+
|
14193 |
+
|
14194 |
+
|
14195 |
+
|
14196 |
+
|
14197 |
+
|
14198 |
+
|
14199 |
+
|
14200 |
+
|
14201 |
+
|
14202 |
+
|
14203 |
+
|
14204 |
+
|
14205 |
+
|
14206 |
+
|
14207 |
+
|
14208 |
+
|
14209 |
+
|
14210 |
+
|
14211 |
+
|
14212 |
+
|
14213 |
+
|
14214 |
+
|
14215 |
+
|
14216 |
+
|
14217 |
+
|
14218 |
+
|
14219 |
+
|
14220 |
+
|
14221 |
+
|
14222 |
+
|
14223 |
+
|
14224 |
+
|
14225 |
+
|
14226 |
+
|
14227 |
+
|
14228 |
+
|
14229 |
+
|
14230 |
+
|
14231 |
+
|
14232 |
+
|
14233 |
+
|
14234 |
+
|
14235 |
+
|
14236 |
+
|
14237 |
+
|
14238 |
+
|
14239 |
+
|
14240 |
+
|
14241 |
+
|
14242 |
+
|
14243 |
+
|
14244 |
+
|
14245 |
+
|
14246 |
+
|
14247 |
+
|
14248 |
+
|
14249 |
+
|
14250 |
+
|
14251 |
+
|
14252 |
+
|
14253 |
+
|
14254 |
+
|
14255 |
+
|
14256 |
+
|
14257 |
+
|
14258 |
+
|
14259 |
+
|
14260 |
+
|
14261 |
+
|
14262 |
+
|
14263 |
+
|
14264 |
+
|
14265 |
+
|
14266 |
+
|
14267 |
+
|
14268 |
+
|
14269 |
+
|
14270 |
+
|
14271 |
+
|
14272 |
+
|
14273 |
+
|
14274 |
+
|
14275 |
+
|
14276 |
+
|
14277 |
+
|
14278 |
+
|
14279 |
+
|
14280 |
+
|
14281 |
+
|
14282 |
+
|
14283 |
+
|
14284 |
+
|
14285 |
+
|
14286 |
+
|
14287 |
+
|
14288 |
+
|
14289 |
+
|
14290 |
+
|
14291 |
+
|
14292 |
+
|
14293 |
+
|
14294 |
+
|
14295 |
+
|
14296 |
+
|
14297 |
+
|
14298 |
+
|
14299 |
+
|
14300 |
+
|
14301 |
+
|
14302 |
+
|
14303 |
+
|
14304 |
+
|
14305 |
+
|
14306 |
+
|
14307 |
+
|
14308 |
+
|
14309 |
+
|
14310 |
+
|
14311 |
+
|
14312 |
+
|
14313 |
+
|
14314 |
+
|
14315 |
+
|
14316 |
+
|
14317 |
+
|
14318 |
+
|
14319 |
+
|
14320 |
+
|
14321 |
+
|
14322 |
+
|
14323 |
+
|
14324 |
+
|
14325 |
+
|
14326 |
+
|
14327 |
+
|
14328 |
+
|
14329 |
+
|
14330 |
+
|
14331 |
+
|
14332 |
+
|
14333 |
+
|
14334 |
+
|
14335 |
+
|
14336 |
+
|
14337 |
+
|
14338 |
+
|
14339 |
+
|
14340 |
+
|
14341 |
+
|
14342 |
+
|
14343 |
+
|
14344 |
+
|
14345 |
+
|
14346 |
+
|
14347 |
+
|
14348 |
+
|
14349 |
+
Training...: 99%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌ | 60000/60729 [8:16:28<05:53, 2.06it/s]
|
14350 |
+
Evaluating ...: 1%|█▊ | 4/462 [00:00<00:50, 9.12it/s]
|
14351 |
+
|
14352 |
+
|
14353 |
+
|
14354 |
+
|
14355 |
+
|
14356 |
+
|
14357 |
+
|
14358 |
+
|
14359 |
+
|
14360 |
+
|
14361 |
+
|
14362 |
+
|
14363 |
+
|
14364 |
+
|
14365 |
+
|
14366 |
+
|
14367 |
+
|
14368 |
+
|
14369 |
+
|
14370 |
+
|
14371 |
+
|
14372 |
+
|
14373 |
+
|
14374 |
+
|
14375 |
+
|
14376 |
+
|
14377 |
+
|
14378 |
+
|
14379 |
+
|
14380 |
+
|
14381 |
+
|
14382 |
+
|
14383 |
+
|
14384 |
+
|
14385 |
+
|
14386 |
+
|
14387 |
+
|
14388 |
+
|
14389 |
+
|
14390 |
+
|
14391 |
+
|
14392 |
+
|
14393 |
+
|
14394 |
+
|
14395 |
+
|
14396 |
+
|
14397 |
+
|
14398 |
+
|
14399 |
+
|
14400 |
+
|
14401 |
+
|
14402 |
+
|
14403 |
+
|
14404 |
+
|
14405 |
+
|
14406 |
+
|
14407 |
+
|
14408 |
+
|
14409 |
+
|
14410 |
+
|
14411 |
+
|
14412 |
+
|
14413 |
+
|
14414 |
+
|
14415 |
+
|
14416 |
+
|
14417 |
+
|
14418 |
+
|
14419 |
+
|
14420 |
+
|
14421 |
+
|
14422 |
+
|
14423 |
+
|
14424 |
+
|
14425 |
+
|
14426 |
+
|
14427 |
+
|
14428 |
+
|
14429 |
+
|
14430 |
+
|
14431 |
+
|
14432 |
+
|
14433 |
+
Step... (60000 | Loss: 1.4433614015579224, Acc: 0.7172799110412598): 0%| | 0/3 [8:20:31<?, ?it/s]5:01, 2.06it/s]
|
14434 |
+
Traceback (most recent call last):
|
14435 |
+
File "run_muril_bb_flax_mlm.py", line 695, in <module>
|
14436 |
+
state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs)
|
14437 |
+
File "<string>", line 1, in __new__
|
14438 |
+
KeyboardInterrupt
|
wandb/run-20211107_122127-18mmtpt3/files/wandb-summary.json
CHANGED
@@ -1 +1 @@
|
|
1 |
-
{"training_step":
|
|
|
1 |
+
{"training_step": 60000, "learning_rate": 7.096180343069136e-05, "train_loss": 1.5583317279815674, "_runtime": 30014, "_timestamp": 1636317701, "_step": 63, "eval_step": 60000, "eval_accuracy": 0.7172799110412598, "eval_loss": 1.4433614015579224, "_wandb": {"runtime": 30052}}
|
wandb/run-20211107_122127-18mmtpt3/logs/debug-internal.log
CHANGED
The diff for this file is too large to render.
See raw diff
|
|
wandb/run-20211107_122127-18mmtpt3/logs/debug.log
CHANGED
@@ -23,3 +23,101 @@ config: {}
|
|
23 |
2021-11-07 12:21:31,003 INFO MainThread:13180 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov07_12-21-20_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 1000, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 50000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 15000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
|
24 |
2021-11-07 12:21:31,005 INFO MainThread:13180 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
|
25 |
2021-11-07 12:21:31,006 INFO MainThread:13180 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
23 |
2021-11-07 12:21:31,003 INFO MainThread:13180 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov07_12-21-20_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 1000, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 50000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 15000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
|
24 |
2021-11-07 12:21:31,005 INFO MainThread:13180 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
|
25 |
2021-11-07 12:21:31,006 INFO MainThread:13180 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}
|
26 |
+
2021-11-07 20:42:18,919 INFO MainThread:13180 [wandb_run.py:_atexit_cleanup():1663] got exitcode: 255
|
27 |
+
2021-11-07 20:42:18,919 INFO MainThread:13180 [wandb_run.py:_restore():1635] restore
|
28 |
+
2021-11-07 20:42:21,203 INFO MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
29 |
+
wandb_count: 1
|
30 |
+
}
|
31 |
+
pusher_stats {
|
32 |
+
uploaded_bytes: 1755
|
33 |
+
total_bytes: 1755
|
34 |
+
}
|
35 |
+
|
36 |
+
2021-11-07 20:42:21,472 INFO MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
37 |
+
wandb_count: 1
|
38 |
+
}
|
39 |
+
pusher_stats {
|
40 |
+
uploaded_bytes: 1755
|
41 |
+
total_bytes: 1755
|
42 |
+
}
|
43 |
+
|
44 |
+
2021-11-07 20:42:21,632 INFO MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
45 |
+
wandb_count: 4
|
46 |
+
}
|
47 |
+
pusher_stats {
|
48 |
+
uploaded_bytes: 1755
|
49 |
+
total_bytes: 55503
|
50 |
+
}
|
51 |
+
|
52 |
+
2021-11-07 20:42:21,734 INFO MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
53 |
+
wandb_count: 5
|
54 |
+
}
|
55 |
+
pusher_stats {
|
56 |
+
uploaded_bytes: 1755
|
57 |
+
total_bytes: 55779
|
58 |
+
}
|
59 |
+
|
60 |
+
2021-11-07 20:42:21,836 INFO MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
61 |
+
wandb_count: 5
|
62 |
+
}
|
63 |
+
pusher_stats {
|
64 |
+
uploaded_bytes: 55779
|
65 |
+
total_bytes: 55779
|
66 |
+
}
|
67 |
+
|
68 |
+
2021-11-07 20:42:21,938 INFO MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
69 |
+
wandb_count: 5
|
70 |
+
}
|
71 |
+
pusher_stats {
|
72 |
+
uploaded_bytes: 55779
|
73 |
+
total_bytes: 55779
|
74 |
+
}
|
75 |
+
|
76 |
+
2021-11-07 20:42:22,040 INFO MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
77 |
+
wandb_count: 5
|
78 |
+
}
|
79 |
+
pusher_stats {
|
80 |
+
uploaded_bytes: 55779
|
81 |
+
total_bytes: 55779
|
82 |
+
}
|
83 |
+
|
84 |
+
2021-11-07 20:42:22,141 INFO MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
85 |
+
wandb_count: 5
|
86 |
+
}
|
87 |
+
pusher_stats {
|
88 |
+
uploaded_bytes: 55779
|
89 |
+
total_bytes: 55779
|
90 |
+
}
|
91 |
+
|
92 |
+
2021-11-07 20:42:22,243 INFO MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
93 |
+
wandb_count: 5
|
94 |
+
}
|
95 |
+
pusher_stats {
|
96 |
+
uploaded_bytes: 55779
|
97 |
+
total_bytes: 55779
|
98 |
+
}
|
99 |
+
|
100 |
+
2021-11-07 20:42:22,612 INFO MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: file_counts {
|
101 |
+
wandb_count: 5
|
102 |
+
}
|
103 |
+
pusher_stats {
|
104 |
+
uploaded_bytes: 55779
|
105 |
+
total_bytes: 55779
|
106 |
+
}
|
107 |
+
|
108 |
+
2021-11-07 20:42:22,963 INFO MainThread:13180 [wandb_run.py:_wait_for_finish():1793] got exit ret: done: true
|
109 |
+
exit_result {
|
110 |
+
}
|
111 |
+
file_counts {
|
112 |
+
wandb_count: 5
|
113 |
+
}
|
114 |
+
pusher_stats {
|
115 |
+
uploaded_bytes: 55779
|
116 |
+
total_bytes: 55779
|
117 |
+
}
|
118 |
+
local_info {
|
119 |
+
}
|
120 |
+
|
121 |
+
2021-11-07 20:42:24,403 INFO MainThread:13180 [wandb_run.py:_append_history():2011] rendering history
|
122 |
+
2021-11-07 20:42:24,404 INFO MainThread:13180 [wandb_run.py:_append_summary():1966] rendering summary
|
123 |
+
2021-11-07 20:42:24,404 INFO MainThread:13180 [wandb_run.py:_append_files():2061] logging synced files
|
wandb/run-20211107_122127-18mmtpt3/run-18mmtpt3.wandb
CHANGED
Binary files a/wandb/run-20211107_122127-18mmtpt3/run-18mmtpt3.wandb and b/wandb/run-20211107_122127-18mmtpt3/run-18mmtpt3.wandb differ
|
|
wandb/run-20211107_204808-2ns3lo5n/files/config.yaml
ADDED
@@ -0,0 +1,341 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
wandb_version: 1
|
2 |
+
|
3 |
+
__cached__setup_devices:
|
4 |
+
desc: null
|
5 |
+
value: cpu
|
6 |
+
_n_gpu:
|
7 |
+
desc: null
|
8 |
+
value: 0
|
9 |
+
_wandb:
|
10 |
+
desc: null
|
11 |
+
value:
|
12 |
+
cli_version: 0.12.6
|
13 |
+
framework: huggingface
|
14 |
+
huggingface_version: 4.13.0.dev0
|
15 |
+
is_jupyter_run: false
|
16 |
+
is_kaggle_kernel: true
|
17 |
+
python_version: 3.8.10
|
18 |
+
start_time: 1636318088
|
19 |
+
t:
|
20 |
+
1:
|
21 |
+
- 1
|
22 |
+
- 2
|
23 |
+
- 3
|
24 |
+
- 11
|
25 |
+
- 12
|
26 |
+
3:
|
27 |
+
- 13
|
28 |
+
4: 3.8.10
|
29 |
+
5: 0.12.6
|
30 |
+
6: 4.13.0.dev0
|
31 |
+
8:
|
32 |
+
- 2
|
33 |
+
- 5
|
34 |
+
adafactor:
|
35 |
+
desc: null
|
36 |
+
value: false
|
37 |
+
adam_beta1:
|
38 |
+
desc: null
|
39 |
+
value: 0.9
|
40 |
+
adam_beta2:
|
41 |
+
desc: null
|
42 |
+
value: 0.999
|
43 |
+
adam_epsilon:
|
44 |
+
desc: null
|
45 |
+
value: 1.0e-08
|
46 |
+
cache_dir:
|
47 |
+
desc: null
|
48 |
+
value: null
|
49 |
+
config_name:
|
50 |
+
desc: null
|
51 |
+
value: null
|
52 |
+
dataloader_drop_last:
|
53 |
+
desc: null
|
54 |
+
value: false
|
55 |
+
dataloader_num_workers:
|
56 |
+
desc: null
|
57 |
+
value: 0
|
58 |
+
dataloader_pin_memory:
|
59 |
+
desc: null
|
60 |
+
value: true
|
61 |
+
dataset_config_name:
|
62 |
+
desc: null
|
63 |
+
value: null
|
64 |
+
dataset_name:
|
65 |
+
desc: null
|
66 |
+
value: null
|
67 |
+
ddp_find_unused_parameters:
|
68 |
+
desc: null
|
69 |
+
value: null
|
70 |
+
debug:
|
71 |
+
desc: null
|
72 |
+
value: []
|
73 |
+
deepspeed:
|
74 |
+
desc: null
|
75 |
+
value: null
|
76 |
+
disable_tqdm:
|
77 |
+
desc: null
|
78 |
+
value: false
|
79 |
+
do_eval:
|
80 |
+
desc: null
|
81 |
+
value: true
|
82 |
+
do_predict:
|
83 |
+
desc: null
|
84 |
+
value: false
|
85 |
+
do_train:
|
86 |
+
desc: null
|
87 |
+
value: true
|
88 |
+
dtype:
|
89 |
+
desc: null
|
90 |
+
value: bfloat16
|
91 |
+
eval_accumulation_steps:
|
92 |
+
desc: null
|
93 |
+
value: null
|
94 |
+
eval_steps:
|
95 |
+
desc: null
|
96 |
+
value: 5000
|
97 |
+
evaluation_strategy:
|
98 |
+
desc: null
|
99 |
+
value: IntervalStrategy.NO
|
100 |
+
fp16:
|
101 |
+
desc: null
|
102 |
+
value: false
|
103 |
+
fp16_backend:
|
104 |
+
desc: null
|
105 |
+
value: auto
|
106 |
+
fp16_full_eval:
|
107 |
+
desc: null
|
108 |
+
value: false
|
109 |
+
fp16_opt_level:
|
110 |
+
desc: null
|
111 |
+
value: O1
|
112 |
+
gradient_accumulation_steps:
|
113 |
+
desc: null
|
114 |
+
value: 1
|
115 |
+
gradient_checkpointing:
|
116 |
+
desc: null
|
117 |
+
value: false
|
118 |
+
greater_is_better:
|
119 |
+
desc: null
|
120 |
+
value: null
|
121 |
+
group_by_length:
|
122 |
+
desc: null
|
123 |
+
value: false
|
124 |
+
hub_model_id:
|
125 |
+
desc: null
|
126 |
+
value: null
|
127 |
+
hub_strategy:
|
128 |
+
desc: null
|
129 |
+
value: HubStrategy.EVERY_SAVE
|
130 |
+
hub_token:
|
131 |
+
desc: null
|
132 |
+
value: null
|
133 |
+
ignore_data_skip:
|
134 |
+
desc: null
|
135 |
+
value: false
|
136 |
+
label_names:
|
137 |
+
desc: null
|
138 |
+
value: null
|
139 |
+
label_smoothing_factor:
|
140 |
+
desc: null
|
141 |
+
value: 0.0
|
142 |
+
learning_rate:
|
143 |
+
desc: null
|
144 |
+
value: 5.0e-05
|
145 |
+
length_column_name:
|
146 |
+
desc: null
|
147 |
+
value: length
|
148 |
+
line_by_line:
|
149 |
+
desc: null
|
150 |
+
value: false
|
151 |
+
load_best_model_at_end:
|
152 |
+
desc: null
|
153 |
+
value: false
|
154 |
+
local_rank:
|
155 |
+
desc: null
|
156 |
+
value: -1
|
157 |
+
log_level:
|
158 |
+
desc: null
|
159 |
+
value: -1
|
160 |
+
log_level_replica:
|
161 |
+
desc: null
|
162 |
+
value: -1
|
163 |
+
log_on_each_node:
|
164 |
+
desc: null
|
165 |
+
value: true
|
166 |
+
logging_dir:
|
167 |
+
desc: null
|
168 |
+
value: ./runs/Nov07_20-47-12_t1v-n-ba7aa1a5-w-0
|
169 |
+
logging_first_step:
|
170 |
+
desc: null
|
171 |
+
value: false
|
172 |
+
logging_nan_inf_filter:
|
173 |
+
desc: null
|
174 |
+
value: true
|
175 |
+
logging_steps:
|
176 |
+
desc: null
|
177 |
+
value: 500
|
178 |
+
logging_strategy:
|
179 |
+
desc: null
|
180 |
+
value: IntervalStrategy.STEPS
|
181 |
+
lr_scheduler_type:
|
182 |
+
desc: null
|
183 |
+
value: SchedulerType.LINEAR
|
184 |
+
max_grad_norm:
|
185 |
+
desc: null
|
186 |
+
value: 1.0
|
187 |
+
max_seq_length:
|
188 |
+
desc: null
|
189 |
+
value: 4096
|
190 |
+
max_steps:
|
191 |
+
desc: null
|
192 |
+
value: -1
|
193 |
+
metric_for_best_model:
|
194 |
+
desc: null
|
195 |
+
value: null
|
196 |
+
mlm_probability:
|
197 |
+
desc: null
|
198 |
+
value: 0.15
|
199 |
+
model_name_or_path:
|
200 |
+
desc: null
|
201 |
+
value: muril-bigbird
|
202 |
+
model_type:
|
203 |
+
desc: null
|
204 |
+
value: null
|
205 |
+
mp_parameters:
|
206 |
+
desc: null
|
207 |
+
value: ''
|
208 |
+
no_cuda:
|
209 |
+
desc: null
|
210 |
+
value: false
|
211 |
+
num_train_epochs:
|
212 |
+
desc: null
|
213 |
+
value: 3.0
|
214 |
+
output_dir:
|
215 |
+
desc: null
|
216 |
+
value: ./
|
217 |
+
overwrite_cache:
|
218 |
+
desc: null
|
219 |
+
value: false
|
220 |
+
overwrite_output_dir:
|
221 |
+
desc: null
|
222 |
+
value: true
|
223 |
+
pad_to_max_length:
|
224 |
+
desc: null
|
225 |
+
value: true
|
226 |
+
past_index:
|
227 |
+
desc: null
|
228 |
+
value: -1
|
229 |
+
per_device_eval_batch_size:
|
230 |
+
desc: null
|
231 |
+
value: 2
|
232 |
+
per_device_train_batch_size:
|
233 |
+
desc: null
|
234 |
+
value: 1
|
235 |
+
per_gpu_eval_batch_size:
|
236 |
+
desc: null
|
237 |
+
value: null
|
238 |
+
per_gpu_train_batch_size:
|
239 |
+
desc: null
|
240 |
+
value: null
|
241 |
+
prediction_loss_only:
|
242 |
+
desc: null
|
243 |
+
value: false
|
244 |
+
preprocessing_num_workers:
|
245 |
+
desc: null
|
246 |
+
value: null
|
247 |
+
pretokenized:
|
248 |
+
desc: null
|
249 |
+
value: true
|
250 |
+
push_to_hub:
|
251 |
+
desc: null
|
252 |
+
value: true
|
253 |
+
push_to_hub_model_id:
|
254 |
+
desc: null
|
255 |
+
value: null
|
256 |
+
push_to_hub_organization:
|
257 |
+
desc: null
|
258 |
+
value: null
|
259 |
+
push_to_hub_token:
|
260 |
+
desc: null
|
261 |
+
value: null
|
262 |
+
remove_unused_columns:
|
263 |
+
desc: null
|
264 |
+
value: true
|
265 |
+
report_to:
|
266 |
+
desc: null
|
267 |
+
value:
|
268 |
+
- tensorboard
|
269 |
+
- wandb
|
270 |
+
resume_from_checkpoint:
|
271 |
+
desc: null
|
272 |
+
value: null
|
273 |
+
run_name:
|
274 |
+
desc: null
|
275 |
+
value: en-ta-hi-te-from-50k
|
276 |
+
save_on_each_node:
|
277 |
+
desc: null
|
278 |
+
value: false
|
279 |
+
save_steps:
|
280 |
+
desc: null
|
281 |
+
value: 50000
|
282 |
+
save_strategy:
|
283 |
+
desc: null
|
284 |
+
value: IntervalStrategy.STEPS
|
285 |
+
save_total_limit:
|
286 |
+
desc: null
|
287 |
+
value: 1
|
288 |
+
seed:
|
289 |
+
desc: null
|
290 |
+
value: 18
|
291 |
+
sharded_ddp:
|
292 |
+
desc: null
|
293 |
+
value: []
|
294 |
+
skip_memory_metrics:
|
295 |
+
desc: null
|
296 |
+
value: true
|
297 |
+
streaming:
|
298 |
+
desc: null
|
299 |
+
value: false
|
300 |
+
tokenizer_name:
|
301 |
+
desc: null
|
302 |
+
value: muril-bigbird
|
303 |
+
tpu_metrics_debug:
|
304 |
+
desc: null
|
305 |
+
value: false
|
306 |
+
tpu_num_cores:
|
307 |
+
desc: null
|
308 |
+
value: null
|
309 |
+
train_file:
|
310 |
+
desc: null
|
311 |
+
value: /dev/shm/data-files
|
312 |
+
train_ref_file:
|
313 |
+
desc: null
|
314 |
+
value: null
|
315 |
+
use_fast_tokenizer:
|
316 |
+
desc: null
|
317 |
+
value: true
|
318 |
+
use_legacy_prediction_loop:
|
319 |
+
desc: null
|
320 |
+
value: false
|
321 |
+
validation_file:
|
322 |
+
desc: null
|
323 |
+
value: .
|
324 |
+
validation_ref_file:
|
325 |
+
desc: null
|
326 |
+
value: null
|
327 |
+
validation_split_percentage:
|
328 |
+
desc: null
|
329 |
+
value: 5
|
330 |
+
warmup_ratio:
|
331 |
+
desc: null
|
332 |
+
value: 0.0
|
333 |
+
warmup_steps:
|
334 |
+
desc: null
|
335 |
+
value: 0
|
336 |
+
weight_decay:
|
337 |
+
desc: null
|
338 |
+
value: 0.01
|
339 |
+
xpu_backend:
|
340 |
+
desc: null
|
341 |
+
value: null
|
wandb/run-20211107_204808-2ns3lo5n/files/output.log
ADDED
@@ -0,0 +1,98 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
[20:48:27] - INFO - absl - A polynomial schedule was set with a non-positive `transition_steps` value; this results in a constant schedule with value `init_value`.
|
2 |
+
Epoch ... (1/3): 0%| | 0/3 [00:00<?, ?it/s]
|
3 |
+
Training...: 0%| | 0/61250 [00:00<?, ?it/s]
|
4 |
+
Epoch ... (1/3): 0%| | 0/3 [03:12<?, ?it/s]
|
5 |
+
Training...: 0%| | 0/61250 [02:05<?, ?it/s]
|
6 |
+
|
7 |
+
|
8 |
+
|
9 |
+
|
10 |
+
|
11 |
+
|
12 |
+
|
13 |
+
|
14 |
+
|
15 |
+
|
16 |
+
|
17 |
+
|
18 |
+
|
19 |
+
|
20 |
+
|
21 |
+
|
22 |
+
|
23 |
+
|
24 |
+
|
25 |
+
|
26 |
+
|
27 |
+
|
28 |
+
|
29 |
+
|
30 |
+
|
31 |
+
|
32 |
+
|
33 |
+
|
34 |
+
|
35 |
+
|
36 |
+
|
37 |
+
|
38 |
+
|
39 |
+
|
40 |
+
|
41 |
+
|
42 |
+
|
43 |
+
|
44 |
+
|
45 |
+
|
46 |
+
|
47 |
+
|
48 |
+
|
49 |
+
|
50 |
+
|
51 |
+
|
52 |
+
|
53 |
+
|
54 |
+
|
55 |
+
|
56 |
+
|
57 |
+
|
58 |
+
|
59 |
+
|
60 |
+
|
61 |
+
|
62 |
+
|
63 |
+
|
64 |
+
|
65 |
+
|
66 |
+
|
67 |
+
|
68 |
+
|
69 |
+
|
70 |
+
|
71 |
+
|
72 |
+
|
73 |
+
|
74 |
+
|
75 |
+
|
76 |
+
|
77 |
+
|
78 |
+
|
79 |
+
|
80 |
+
|
81 |
+
|
82 |
+
|
83 |
+
|
84 |
+
|
85 |
+
|
86 |
+
|
87 |
+
|
88 |
+
|
89 |
+
|
90 |
+
|
91 |
+
|
92 |
+
|
93 |
+
|
94 |
+
|
95 |
+
tcmalloc: large alloc 1212121088 bytes == 0x3920c6000 @ 0x7fee676b5680 0x7fee676d5bdd 0x7fecb83f720d 0x7fecb8404069 0x7fecb840556e 0x7fecb8400bd3 0x7fecb84011fe 0x5042c6 0x56bf09 0x56a0ba 0x5f6343 0x56cf2a 0x5f6166 0x56bddd 0x7fecb83f7410 0x7fecb83f867a 0x7fecb83fa6f7 0x7fecb84042a3 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8400bd3 0x7fecb84011fe 0x5042c6 0x56bf09 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba
|
96 |
+
tcmalloc: large alloc 1237311488 bytes == 0x3fe6ba000 @ 0x7fee676b5680 0x7fee676d5bdd 0x7fecb83f720d 0x7fecb8405340 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8400bd3 0x7fecb84011fe 0x5042c6 0x56bf09 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x5f6343 0x56cf2a 0x5f6166 0x570e46 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x68d5b7 0x67cd01 0x67cd7f 0x67ce21 0x67ef47 0x6b7242
|
97 |
+
tcmalloc: large alloc 3161391104 bytes == 0x4482b8000 @ 0x7fee676b5680 0x7fee676d5bdd 0x7fecb83f720d 0x7fecb8405340 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8404e87 0x7fecb8400bd3 0x7fecb84011fe 0x5042c6 0x56bf09 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x5f6343 0x56cf2a 0x5f6166 0x570e46 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x68d5b7 0x67cd01 0x67cd7f 0x67ce21
|
98 |
+
tcmalloc: large alloc 2886115328 bytes == 0x3920c6000 @ 0x7fee676b5680 0x7fee676d6824 0x5f9f21 0x7fecb8400c6f 0x7fecb84011fe 0x5042c6 0x56bf09 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x5f6343 0x56cf2a 0x5f6166 0x570e46 0x56a0ba 0x5f6343 0x56cf2a 0x56a0ba 0x68d5b7 0x67cd01 0x67cd7f 0x67ce21 0x67ef47 0x6b7242 0x6b75cd 0x7fee674cc0b3 0x5fb18e
|
wandb/run-20211107_204808-2ns3lo5n/files/requirements.txt
ADDED
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
absl-py==0.15.0
|
2 |
+
aiohttp==3.8.0
|
3 |
+
aiosignal==1.2.0
|
4 |
+
astunparse==1.6.3
|
5 |
+
async-timeout==4.0.0
|
6 |
+
attrs==21.2.0
|
7 |
+
cachetools==4.2.4
|
8 |
+
certifi==2021.10.8
|
9 |
+
charset-normalizer==2.0.7
|
10 |
+
chex==0.0.8
|
11 |
+
click==8.0.3
|
12 |
+
configparser==5.1.0
|
13 |
+
cycler==0.11.0
|
14 |
+
datasets==1.15.1
|
15 |
+
dill==0.3.4
|
16 |
+
dm-tree==0.1.6
|
17 |
+
docker-pycreds==0.4.0
|
18 |
+
filelock==3.3.2
|
19 |
+
flatbuffers==2.0
|
20 |
+
flax==0.3.6
|
21 |
+
frozenlist==1.2.0
|
22 |
+
fsspec==2021.11.0
|
23 |
+
gast==0.4.0
|
24 |
+
gitdb==4.0.9
|
25 |
+
gitpython==3.1.24
|
26 |
+
google-auth-oauthlib==0.4.6
|
27 |
+
google-auth==2.3.3
|
28 |
+
google-pasta==0.2.0
|
29 |
+
grpcio==1.41.1
|
30 |
+
h5py==3.5.0
|
31 |
+
huggingface-hub==0.1.1
|
32 |
+
idna==3.3
|
33 |
+
jax==0.2.24
|
34 |
+
jaxlib==0.1.73
|
35 |
+
joblib==1.1.0
|
36 |
+
kaggle==1.5.12
|
37 |
+
keras-preprocessing==1.1.2
|
38 |
+
keras==2.7.0
|
39 |
+
kiwisolver==1.3.2
|
40 |
+
libclang==12.0.0
|
41 |
+
libtpu-nightly==0.1.dev20211018
|
42 |
+
markdown==3.3.4
|
43 |
+
matplotlib==3.4.3
|
44 |
+
msgpack==1.0.2
|
45 |
+
multidict==5.2.0
|
46 |
+
multiprocess==0.70.12.2
|
47 |
+
numpy==1.21.4
|
48 |
+
oauthlib==3.1.1
|
49 |
+
opt-einsum==3.3.0
|
50 |
+
optax==0.0.9
|
51 |
+
packaging==21.2
|
52 |
+
pandas==1.3.4
|
53 |
+
pathtools==0.1.2
|
54 |
+
pillow==8.4.0
|
55 |
+
pip==21.3.1
|
56 |
+
pkg-resources==0.0.0
|
57 |
+
promise==2.3
|
58 |
+
protobuf==3.19.1
|
59 |
+
psutil==5.8.0
|
60 |
+
pyarrow==6.0.0
|
61 |
+
pyasn1-modules==0.2.8
|
62 |
+
pyasn1==0.4.8
|
63 |
+
pyparsing==2.4.7
|
64 |
+
python-dateutil==2.8.2
|
65 |
+
python-slugify==5.0.2
|
66 |
+
pytz==2021.3
|
67 |
+
pyyaml==6.0
|
68 |
+
regex==2021.11.2
|
69 |
+
requests-oauthlib==1.3.0
|
70 |
+
requests==2.26.0
|
71 |
+
rsa==4.7.2
|
72 |
+
sacremoses==0.0.46
|
73 |
+
scipy==1.7.1
|
74 |
+
sentry-sdk==1.4.3
|
75 |
+
setuptools==44.0.0
|
76 |
+
shortuuid==1.0.1
|
77 |
+
six==1.16.0
|
78 |
+
smmap==5.0.0
|
79 |
+
subprocess32==3.5.4
|
80 |
+
tensorboard-data-server==0.6.1
|
81 |
+
tensorboard-plugin-wit==1.8.0
|
82 |
+
tensorboard==2.7.0
|
83 |
+
tensorflow-estimator==2.7.0
|
84 |
+
tensorflow-io-gcs-filesystem==0.21.0
|
85 |
+
tensorflow==2.7.0
|
86 |
+
termcolor==1.1.0
|
87 |
+
text-unidecode==1.3
|
88 |
+
tokenizers==0.10.3
|
89 |
+
toolz==0.11.1
|
90 |
+
torch==1.10.0
|
91 |
+
tqdm==4.62.3
|
92 |
+
transformers==4.13.0.dev0
|
93 |
+
typing-extensions==3.10.0.2
|
94 |
+
urllib3==1.26.7
|
95 |
+
wandb==0.12.6
|
96 |
+
werkzeug==2.0.2
|
97 |
+
wheel==0.37.0
|
98 |
+
wrapt==1.13.3
|
99 |
+
xxhash==2.0.2
|
100 |
+
yarl==1.7.2
|
101 |
+
yaspin==2.1.0
|
wandb/run-20211107_204808-2ns3lo5n/files/wandb-metadata.json
ADDED
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
|
3 |
+
"python": "3.8.10",
|
4 |
+
"heartbeatAt": "2021-11-07T20:48:11.579826",
|
5 |
+
"startedAt": "2021-11-07T20:48:08.404632",
|
6 |
+
"docker": null,
|
7 |
+
"cpu_count": 96,
|
8 |
+
"cuda": null,
|
9 |
+
"args": [
|
10 |
+
"--model_name_or_path",
|
11 |
+
"muril-bigbird",
|
12 |
+
"--tokenizer_name",
|
13 |
+
"muril-bigbird",
|
14 |
+
"--dtype",
|
15 |
+
"bfloat16",
|
16 |
+
"--pretokenized",
|
17 |
+
"--train_file",
|
18 |
+
"/dev/shm/data-files",
|
19 |
+
"--validation_file",
|
20 |
+
".",
|
21 |
+
"--max_seq_length",
|
22 |
+
"4096",
|
23 |
+
"--pad_to_max_length",
|
24 |
+
"--output_dir",
|
25 |
+
"./",
|
26 |
+
"--overwrite_output_dir",
|
27 |
+
"--do_train",
|
28 |
+
"--do_eval",
|
29 |
+
"--eval_steps",
|
30 |
+
"5000",
|
31 |
+
"--per_device_train_batch_size",
|
32 |
+
"1",
|
33 |
+
"--per_device_eval_batch_size",
|
34 |
+
"2",
|
35 |
+
"--learning_rate",
|
36 |
+
"5e-5",
|
37 |
+
"--weight_decay",
|
38 |
+
"0.01",
|
39 |
+
"--num_train_epochs",
|
40 |
+
"3",
|
41 |
+
"--warmup_steps",
|
42 |
+
"0",
|
43 |
+
"--logging_steps",
|
44 |
+
"500",
|
45 |
+
"--save_steps",
|
46 |
+
"50000",
|
47 |
+
"--save_total_limit",
|
48 |
+
"1",
|
49 |
+
"--seed",
|
50 |
+
"18",
|
51 |
+
"--push_to_hub",
|
52 |
+
"--run_name",
|
53 |
+
"en-ta-hi-te-from-50k"
|
54 |
+
],
|
55 |
+
"state": "running",
|
56 |
+
"program": "run_muril_bb_flax_mlm.py",
|
57 |
+
"codePath": "run_muril_bb_flax_mlm.py",
|
58 |
+
"git": {
|
59 |
+
"remote": "https://huggingface.co/nbroad/muril-bigbird-base-1mil",
|
60 |
+
"commit": "52d9b405c9e32ad5079f76867da58f0d64320170"
|
61 |
+
},
|
62 |
+
"email": "nicholas@nmbroad.com",
|
63 |
+
"root": "/home/nicholasbroad/muril-bigbird-base-1mil",
|
64 |
+
"host": "t1v-n-ba7aa1a5-w-0",
|
65 |
+
"username": "nicholasbroad",
|
66 |
+
"executable": "/home/nicholasbroad/venv/bin/python"
|
67 |
+
}
|
wandb/run-20211107_204808-2ns3lo5n/files/wandb-summary.json
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
{"training_step": 50000, "learning_rate": 4.999999873689376e-05, "train_loss": 1.9627385139465332, "_runtime": 422, "_timestamp": 1636318510, "_step": 1, "eval_step": 50000, "eval_accuracy": 0.6992624998092651, "eval_loss": 1.5497748851776123}
|
wandb/run-20211107_204808-2ns3lo5n/logs/debug-internal.log
ADDED
@@ -0,0 +1,226 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2021-11-07 20:48:09,073 INFO MainThread:21539 [internal.py:wandb_internal():87] W&B internal server running at pid: 21539, started at: 2021-11-07 20:48:09.073384
|
2 |
+
2021-11-07 20:48:09,075 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: check_version
|
3 |
+
2021-11-07 20:48:09,075 INFO WriterThread:21539 [datastore.py:open_for_write():77] open: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/run-2ns3lo5n.wandb
|
4 |
+
2021-11-07 20:48:09,077 DEBUG SenderThread:21539 [sender.py:send():238] send: header
|
5 |
+
2021-11-07 20:48:09,077 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: check_version
|
6 |
+
2021-11-07 20:48:09,123 DEBUG SenderThread:21539 [sender.py:send():238] send: run
|
7 |
+
2021-11-07 20:48:09,499 INFO SenderThread:21539 [dir_watcher.py:__init__():169] watching files in: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files
|
8 |
+
2021-11-07 20:48:09,499 INFO SenderThread:21539 [sender.py:_start_run_threads():825] run started: 2ns3lo5n with start time 1636318088
|
9 |
+
2021-11-07 20:48:09,500 DEBUG SenderThread:21539 [sender.py:send():238] send: summary
|
10 |
+
2021-11-07 20:48:09,500 INFO SenderThread:21539 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
|
11 |
+
2021-11-07 20:48:09,500 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: run_start
|
12 |
+
2021-11-07 20:48:10,502 INFO Thread-8 :21539 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/wandb-summary.json
|
13 |
+
2021-11-07 20:48:11,579 DEBUG HandlerThread:21539 [meta.py:__init__():40] meta init
|
14 |
+
2021-11-07 20:48:11,579 DEBUG HandlerThread:21539 [meta.py:__init__():54] meta init done
|
15 |
+
2021-11-07 20:48:11,579 DEBUG HandlerThread:21539 [meta.py:probe():214] probe
|
16 |
+
2021-11-07 20:48:11,580 DEBUG HandlerThread:21539 [meta.py:_setup_git():204] setup git
|
17 |
+
2021-11-07 20:48:11,611 DEBUG HandlerThread:21539 [meta.py:_setup_git():211] setup git done
|
18 |
+
2021-11-07 20:48:11,611 DEBUG HandlerThread:21539 [meta.py:_save_pip():58] save pip
|
19 |
+
2021-11-07 20:48:11,612 DEBUG HandlerThread:21539 [meta.py:_save_pip():72] save pip done
|
20 |
+
2021-11-07 20:48:11,612 DEBUG HandlerThread:21539 [meta.py:probe():252] probe done
|
21 |
+
2021-11-07 20:48:11,615 DEBUG SenderThread:21539 [sender.py:send():238] send: files
|
22 |
+
2021-11-07 20:48:11,615 INFO SenderThread:21539 [sender.py:_save_file():960] saving file wandb-metadata.json with policy now
|
23 |
+
2021-11-07 20:48:11,620 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
24 |
+
2021-11-07 20:48:11,621 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
25 |
+
2021-11-07 20:48:11,755 DEBUG SenderThread:21539 [sender.py:send():238] send: config
|
26 |
+
2021-11-07 20:48:11,756 DEBUG SenderThread:21539 [sender.py:send():238] send: config
|
27 |
+
2021-11-07 20:48:11,756 DEBUG SenderThread:21539 [sender.py:send():238] send: config
|
28 |
+
2021-11-07 20:48:12,094 INFO Thread-11 :21539 [upload_job.py:push():137] Uploaded file /tmp/tmpzv74dks4wandb/1kin4cdo-wandb-metadata.json
|
29 |
+
2021-11-07 20:48:12,504 INFO Thread-8 :21539 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
30 |
+
2021-11-07 20:48:12,505 INFO Thread-8 :21539 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/requirements.txt
|
31 |
+
2021-11-07 20:48:12,505 INFO Thread-8 :21539 [dir_watcher.py:_on_file_created():217] file/dir created: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/wandb-metadata.json
|
32 |
+
2021-11-07 20:48:27,125 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
33 |
+
2021-11-07 20:48:27,126 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
34 |
+
2021-11-07 20:48:30,511 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
35 |
+
2021-11-07 20:48:39,514 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/config.yaml
|
36 |
+
2021-11-07 20:48:39,657 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
37 |
+
2021-11-07 20:48:42,263 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
38 |
+
2021-11-07 20:48:42,263 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
39 |
+
2021-11-07 20:48:57,395 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
40 |
+
2021-11-07 20:48:57,395 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
41 |
+
2021-11-07 20:49:09,716 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
42 |
+
2021-11-07 20:49:12,528 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
43 |
+
2021-11-07 20:49:12,528 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
44 |
+
2021-11-07 20:49:27,664 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
45 |
+
2021-11-07 20:49:27,665 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
46 |
+
2021-11-07 20:49:36,535 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
47 |
+
2021-11-07 20:49:39,780 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
48 |
+
2021-11-07 20:49:42,824 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
49 |
+
2021-11-07 20:49:42,824 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
50 |
+
2021-11-07 20:49:57,982 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
51 |
+
2021-11-07 20:49:57,982 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
52 |
+
2021-11-07 20:50:09,846 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
53 |
+
2021-11-07 20:50:13,120 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
54 |
+
2021-11-07 20:50:13,120 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
55 |
+
2021-11-07 20:50:28,253 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
56 |
+
2021-11-07 20:50:28,253 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
57 |
+
2021-11-07 20:50:39,916 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
58 |
+
2021-11-07 20:50:43,387 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
59 |
+
2021-11-07 20:50:43,387 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
60 |
+
2021-11-07 20:50:58,521 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
61 |
+
2021-11-07 20:50:58,522 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
62 |
+
2021-11-07 20:51:09,989 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
63 |
+
2021-11-07 20:51:13,656 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
64 |
+
2021-11-07 20:51:13,656 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
65 |
+
2021-11-07 20:51:28,789 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
66 |
+
2021-11-07 20:51:28,790 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
67 |
+
2021-11-07 20:51:40,065 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
68 |
+
2021-11-07 20:51:40,800 DEBUG SenderThread:21539 [sender.py:send():238] send: history
|
69 |
+
2021-11-07 20:51:40,801 DEBUG SenderThread:21539 [sender.py:send():238] send: summary
|
70 |
+
2021-11-07 20:51:40,801 INFO SenderThread:21539 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
|
71 |
+
2021-11-07 20:51:41,580 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/wandb-summary.json
|
72 |
+
2021-11-07 20:51:42,580 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
73 |
+
2021-11-07 20:51:43,934 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
74 |
+
2021-11-07 20:51:43,934 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
75 |
+
2021-11-07 20:51:59,097 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
76 |
+
2021-11-07 20:51:59,097 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
77 |
+
2021-11-07 20:52:04,589 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
78 |
+
2021-11-07 20:52:06,590 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
79 |
+
2021-11-07 20:52:08,591 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
80 |
+
2021-11-07 20:52:10,142 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
81 |
+
2021-11-07 20:52:10,592 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
82 |
+
2021-11-07 20:52:12,593 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
83 |
+
2021-11-07 20:52:14,272 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
84 |
+
2021-11-07 20:52:14,272 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
85 |
+
2021-11-07 20:52:14,593 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
86 |
+
2021-11-07 20:52:16,595 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
87 |
+
2021-11-07 20:52:18,595 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
88 |
+
2021-11-07 20:52:20,596 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
89 |
+
2021-11-07 20:52:22,597 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
90 |
+
2021-11-07 20:52:24,598 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
91 |
+
2021-11-07 20:52:26,599 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
92 |
+
2021-11-07 20:52:28,599 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
93 |
+
2021-11-07 20:52:29,407 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
94 |
+
2021-11-07 20:52:29,408 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
95 |
+
2021-11-07 20:52:30,600 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
96 |
+
2021-11-07 20:52:32,601 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
97 |
+
2021-11-07 20:52:34,602 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
98 |
+
2021-11-07 20:52:36,603 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
99 |
+
2021-11-07 20:52:38,604 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
100 |
+
2021-11-07 20:52:40,215 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
101 |
+
2021-11-07 20:52:40,604 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
102 |
+
2021-11-07 20:52:42,606 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
103 |
+
2021-11-07 20:52:44,538 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
104 |
+
2021-11-07 20:52:44,538 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
105 |
+
2021-11-07 20:52:45,607 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
106 |
+
2021-11-07 20:52:47,608 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
107 |
+
2021-11-07 20:52:49,609 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
108 |
+
2021-11-07 20:52:51,610 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
109 |
+
2021-11-07 20:52:53,611 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
110 |
+
2021-11-07 20:52:55,612 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
111 |
+
2021-11-07 20:52:57,613 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
112 |
+
2021-11-07 20:52:59,614 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
113 |
+
2021-11-07 20:52:59,744 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
114 |
+
2021-11-07 20:52:59,744 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
115 |
+
2021-11-07 20:53:01,615 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
116 |
+
2021-11-07 20:53:03,615 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
117 |
+
2021-11-07 20:53:05,616 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
118 |
+
2021-11-07 20:53:07,617 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
119 |
+
2021-11-07 20:53:09,618 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
120 |
+
2021-11-07 20:53:10,287 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
121 |
+
2021-11-07 20:53:11,619 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
122 |
+
2021-11-07 20:53:13,620 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
123 |
+
2021-11-07 20:53:14,884 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
124 |
+
2021-11-07 20:53:14,884 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
125 |
+
2021-11-07 20:53:15,621 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
126 |
+
2021-11-07 20:53:17,621 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
127 |
+
2021-11-07 20:53:19,622 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
128 |
+
2021-11-07 20:53:21,623 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
129 |
+
2021-11-07 20:53:23,624 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
130 |
+
2021-11-07 20:53:25,625 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
131 |
+
2021-11-07 20:53:27,626 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
132 |
+
2021-11-07 20:53:29,626 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
133 |
+
2021-11-07 20:53:30,028 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
134 |
+
2021-11-07 20:53:30,028 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
135 |
+
2021-11-07 20:53:31,627 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
136 |
+
2021-11-07 20:53:33,628 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
137 |
+
2021-11-07 20:53:35,629 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
138 |
+
2021-11-07 20:53:37,630 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
139 |
+
2021-11-07 20:53:39,631 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
140 |
+
2021-11-07 20:53:40,360 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
141 |
+
2021-11-07 20:53:41,632 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
142 |
+
2021-11-07 20:53:43,633 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
143 |
+
2021-11-07 20:53:45,199 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
144 |
+
2021-11-07 20:53:45,199 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
145 |
+
2021-11-07 20:53:45,634 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
146 |
+
2021-11-07 20:53:47,635 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
147 |
+
2021-11-07 20:53:49,636 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
148 |
+
2021-11-07 20:53:51,636 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
149 |
+
2021-11-07 20:53:53,637 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
150 |
+
2021-11-07 20:53:55,638 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
151 |
+
2021-11-07 20:53:57,639 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
152 |
+
2021-11-07 20:53:59,640 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
153 |
+
2021-11-07 20:54:00,332 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
154 |
+
2021-11-07 20:54:00,332 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
155 |
+
2021-11-07 20:54:01,641 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
156 |
+
2021-11-07 20:54:03,642 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
157 |
+
2021-11-07 20:54:05,643 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
158 |
+
2021-11-07 20:54:07,644 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
159 |
+
2021-11-07 20:54:09,644 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
160 |
+
2021-11-07 20:54:10,433 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
161 |
+
2021-11-07 20:54:11,645 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
162 |
+
2021-11-07 20:54:13,646 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
163 |
+
2021-11-07 20:54:15,474 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
164 |
+
2021-11-07 20:54:15,474 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
165 |
+
2021-11-07 20:54:15,647 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
166 |
+
2021-11-07 20:54:17,647 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
167 |
+
2021-11-07 20:54:19,648 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
168 |
+
2021-11-07 20:54:21,649 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
169 |
+
2021-11-07 20:54:23,650 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
170 |
+
2021-11-07 20:54:25,651 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
171 |
+
2021-11-07 20:54:27,651 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
172 |
+
2021-11-07 20:54:29,652 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
173 |
+
2021-11-07 20:54:30,676 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
174 |
+
2021-11-07 20:54:30,677 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
175 |
+
2021-11-07 20:54:31,653 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
176 |
+
2021-11-07 20:54:34,654 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
177 |
+
2021-11-07 20:54:36,655 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
178 |
+
2021-11-07 20:54:38,656 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
179 |
+
2021-11-07 20:54:40,505 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
180 |
+
2021-11-07 20:54:40,657 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
181 |
+
2021-11-07 20:54:42,657 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
182 |
+
2021-11-07 20:54:44,658 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
183 |
+
2021-11-07 20:54:45,808 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
184 |
+
2021-11-07 20:54:45,809 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
185 |
+
2021-11-07 20:54:46,659 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
186 |
+
2021-11-07 20:54:48,660 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
187 |
+
2021-11-07 20:54:50,661 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
188 |
+
2021-11-07 20:54:52,662 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
189 |
+
2021-11-07 20:54:54,663 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
190 |
+
2021-11-07 20:54:56,664 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
191 |
+
2021-11-07 20:54:58,664 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
192 |
+
2021-11-07 20:55:00,665 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
193 |
+
2021-11-07 20:55:00,946 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
194 |
+
2021-11-07 20:55:00,947 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
195 |
+
2021-11-07 20:55:02,666 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
196 |
+
2021-11-07 20:55:10,442 DEBUG SenderThread:21539 [sender.py:send():238] send: history
|
197 |
+
2021-11-07 20:55:10,443 DEBUG SenderThread:21539 [sender.py:send():238] send: summary
|
198 |
+
2021-11-07 20:55:10,445 INFO SenderThread:21539 [sender.py:_save_file():960] saving file wandb-summary.json with policy end
|
199 |
+
2021-11-07 20:55:10,577 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
200 |
+
2021-11-07 20:55:10,669 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/wandb-summary.json
|
201 |
+
2021-11-07 20:55:17,590 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
202 |
+
2021-11-07 20:55:17,591 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
203 |
+
2021-11-07 20:55:17,672 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
204 |
+
2021-11-07 20:55:21,674 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
205 |
+
2021-11-07 20:55:23,675 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
206 |
+
2021-11-07 20:55:32,740 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
207 |
+
2021-11-07 20:55:32,740 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
208 |
+
2021-11-07 20:55:40,655 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
209 |
+
2021-11-07 20:55:47,874 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
210 |
+
2021-11-07 20:55:47,875 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
211 |
+
2021-11-07 20:55:49,684 INFO Thread-8 :21539 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/files/output.log
|
212 |
+
2021-11-07 20:56:03,011 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
213 |
+
2021-11-07 20:56:03,012 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
214 |
+
2021-11-07 20:56:10,732 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
215 |
+
2021-11-07 20:56:18,145 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
216 |
+
2021-11-07 20:56:18,145 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
217 |
+
2021-11-07 20:56:33,278 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
218 |
+
2021-11-07 20:56:33,278 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
219 |
+
2021-11-07 20:56:40,805 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
220 |
+
2021-11-07 20:56:48,418 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
221 |
+
2021-11-07 20:56:48,419 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
222 |
+
2021-11-07 20:57:03,562 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
223 |
+
2021-11-07 20:57:03,563 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
224 |
+
2021-11-07 20:57:10,879 DEBUG SenderThread:21539 [sender.py:send():238] send: stats
|
225 |
+
2021-11-07 20:57:18,693 DEBUG HandlerThread:21539 [handler.py:handle_request():130] handle_request: stop_status
|
226 |
+
2021-11-07 20:57:18,694 DEBUG SenderThread:21539 [sender.py:send_request():252] send_request: stop_status
|
wandb/run-20211107_204808-2ns3lo5n/logs/debug.log
ADDED
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2021-11-07 20:48:08,405 INFO MainThread:20231 [wandb_setup.py:_flush():71] setting env: {}
|
2 |
+
2021-11-07 20:48:08,405 INFO MainThread:20231 [wandb_setup.py:_flush():71] setting login settings: {}
|
3 |
+
2021-11-07 20:48:08,405 INFO MainThread:20231 [wandb_init.py:_log_setup():357] Logging user logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/logs/debug.log
|
4 |
+
2021-11-07 20:48:08,406 INFO MainThread:20231 [wandb_init.py:_log_setup():358] Logging internal logs to /home/nicholasbroad/muril-bigbird-base-1mil/wandb/run-20211107_204808-2ns3lo5n/logs/debug-internal.log
|
5 |
+
2021-11-07 20:48:08,406 INFO MainThread:20231 [wandb_init.py:init():390] calling init triggers
|
6 |
+
2021-11-07 20:48:08,406 INFO MainThread:20231 [wandb_init.py:init():395] wandb.init called with sweep_config: {}
|
7 |
+
config: {}
|
8 |
+
2021-11-07 20:48:08,406 INFO MainThread:20231 [wandb_init.py:init():435] starting backend
|
9 |
+
2021-11-07 20:48:08,406 INFO MainThread:20231 [backend.py:_multiprocessing_setup():95] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
|
10 |
+
2021-11-07 20:48:08,444 INFO MainThread:20231 [backend.py:ensure_launched():199] starting backend process...
|
11 |
+
2021-11-07 20:48:08,481 INFO MainThread:20231 [backend.py:ensure_launched():204] started backend process with pid: 21539
|
12 |
+
2021-11-07 20:48:08,483 INFO MainThread:20231 [wandb_init.py:init():444] backend started and connected
|
13 |
+
2021-11-07 20:48:08,486 INFO MainThread:20231 [wandb_init.py:init():503] updated telemetry
|
14 |
+
2021-11-07 20:48:08,568 INFO MainThread:20231 [wandb_init.py:init():533] communicating current version
|
15 |
+
2021-11-07 20:48:09,122 INFO MainThread:20231 [wandb_init.py:init():538] got version response
|
16 |
+
2021-11-07 20:48:09,122 INFO MainThread:20231 [wandb_init.py:init():548] communicating run to backend with 30 second timeout
|
17 |
+
2021-11-07 20:48:09,500 INFO MainThread:20231 [wandb_init.py:init():576] starting run threads in backend
|
18 |
+
2021-11-07 20:48:11,620 INFO MainThread:20231 [wandb_run.py:_console_start():1693] atexit reg
|
19 |
+
2021-11-07 20:48:11,620 INFO MainThread:20231 [wandb_run.py:_redirect():1567] redirect: SettingsConsole.REDIRECT
|
20 |
+
2021-11-07 20:48:11,621 INFO MainThread:20231 [wandb_run.py:_redirect():1572] Redirecting console.
|
21 |
+
2021-11-07 20:48:11,622 INFO MainThread:20231 [wandb_run.py:_redirect():1628] Redirects installed.
|
22 |
+
2021-11-07 20:48:11,622 INFO MainThread:20231 [wandb_init.py:init():603] run started, returning control to user process
|
23 |
+
2021-11-07 20:48:11,628 INFO MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 2, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 5e-05, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 3.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Nov07_20-47-12_t1v-n-ba7aa1a5-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'logging_nan_inf_filter': True, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 50000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 18, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'xpu_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 5000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'en-ta-hi-te-from-50k', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'HubStrategy.EVERY_SAVE', 'hub_token': None, 'gradient_checkpointing': False, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
|
24 |
+
2021-11-07 20:48:11,630 INFO MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'model_name_or_path': 'muril-bigbird', 'model_type': None, 'config_name': None, 'tokenizer_name': 'muril-bigbird', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'}
|
25 |
+
2021-11-07 20:48:11,631 INFO MainThread:20231 [wandb_run.py:_config_callback():911] config_cb None None {'dataset_name': None, 'pretokenized': True, 'streaming': False, 'dataset_config_name': None, 'train_file': '/dev/shm/data-files', 'validation_file': '.', 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False}
|
wandb/run-20211107_204808-2ns3lo5n/run-2ns3lo5n.wandb
ADDED
Binary file (52.6 kB). View file
|
|