Hugging Face's logo

flax-community
/
pino-bigbird-roberta-base

Fill-Mask
Transformers
PyTorch
JAX
TensorBoard
Dutch
big_bird
Inference Endpoints
Model card Files Metrics Community
2
dat commited on
Commit
0609cb0
•
1 Parent(s): b681451

add all

Browse files
Files changed (42) hide show
  1. events.out.tfevents.1626318666.t1v-n-f5c06ea1-w-0.655476.3.v2 +2 -2
  2. events.out.tfevents.1626355795.t1v-n-f5c06ea1-w-0.688983.3.v2 +3 -0
  3. events.out.tfevents.1626357208.t1v-n-f5c06ea1-w-0.692253.3.v2 +3 -0
  4. run.sh +1 -1
  5. run_mlm_flax_no_accum.py +16 -15
  6. wandb/debug-internal.log +1 -1
  7. wandb/debug.log +1 -1
  8. wandb/latest-run +1 -1
  9. wandb/run-20210715_031107-69jkygz3/files/output.log +30 -0
  10. wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json +1 -1
  11. wandb/run-20210715_031107-69jkygz3/logs/debug-internal.log +52 -0
  12. wandb/run-20210715_031107-69jkygz3/logs/debug.log +2 -0
  13. wandb/run-20210715_031107-69jkygz3/run-69jkygz3.wandb +0 -0
  14. wandb/run-20210715_132956-37l0zsn1/files/config.yaml +301 -0
  15. wandb/run-20210715_132956-37l0zsn1/files/output.log +195 -0
  16. wandb/run-20210715_132956-37l0zsn1/files/requirements.txt +94 -0
  17. wandb/run-20210715_132956-37l0zsn1/files/wandb-metadata.json +45 -0
  18. wandb/run-20210715_132956-37l0zsn1/files/wandb-summary.json +1 -0
  19. wandb/run-20210715_132956-37l0zsn1/logs/debug-internal.log +391 -0
  20. wandb/run-20210715_132956-37l0zsn1/logs/debug.log +27 -0
  21. wandb/run-20210715_132956-37l0zsn1/run-37l0zsn1.wandb +0 -0
  22. wandb/run-20210715_135329-1c6qs9ww/files/config.yaml +301 -0
  23. wandb/run-20210715_135329-1c6qs9ww/files/output.log +119 -0
  24. wandb/run-20210715_135329-1c6qs9ww/files/requirements.txt +94 -0
  25. wandb/run-20210715_135329-1c6qs9ww/files/wandb-metadata.json +45 -0
  26. wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json +1 -0
  27. wandb/run-20210715_135329-1c6qs9ww/logs/debug-internal.log +399 -0
  28. wandb/run-20210715_135329-1c6qs9ww/logs/debug.log +27 -0
  29. wandb/run-20210715_135329-1c6qs9ww/run-1c6qs9ww.wandb +0 -0
  30. wandb/run-20210715_141143-1vry5yso/files/config.yaml +301 -0
  31. wandb/run-20210715_141143-1vry5yso/files/output.log +64 -0
  32. wandb/run-20210715_141143-1vry5yso/files/requirements.txt +94 -0
  33. wandb/run-20210715_141143-1vry5yso/files/wandb-metadata.json +45 -0
  34. wandb/run-20210715_141143-1vry5yso/files/wandb-summary.json +1 -0
  35. wandb/run-20210715_141143-1vry5yso/logs/debug-internal.log +125 -0
  36. wandb/run-20210715_141143-1vry5yso/logs/debug.log +28 -0
  37. wandb/run-20210715_141143-1vry5yso/run-1vry5yso.wandb +0 -0
  38. wandb/run-20210715_142215-1vry5yso/files/config.yaml +20 -0
  39. wandb/run-20210715_142215-1vry5yso/logs/debug-internal.log +46 -0
  40. wandb/run-20210715_142215-1vry5yso/logs/debug.log +37 -0
  41. wandb/run-20210715_142215-1vry5yso/run-1vry5yso.wandb +0 -0
  42. wandb/wandb-resume.json +1 -0
events.out.tfevents.1626318666.t1v-n-f5c06ea1-w-0.655476.3.v2 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:87d384252cfc2c9551426c799e7901b66a0c7e2351dc2b43d1d2634656ab26a9
3
- size 13503400
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:898bcab2cec674ad70a77673aa240b8be9924c739c58fa92026ccb9953008588
3
+ size 13518440
events.out.tfevents.1626355795.t1v-n-f5c06ea1-w-0.688983.3.v2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b033e6ba5a9ecd725a4c22f2209304ae9f67b9f477543a2abd8c70b68fe4cc04
3
+ size 40
events.out.tfevents.1626357208.t1v-n-f5c06ea1-w-0.692253.3.v2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:03fa64d5ad6dbc89223988a35e988bf0d38d9e67dc8d95fd1095b990ffc23a3e
3
+ size 148305
run.sh CHANGED
@@ -15,7 +15,7 @@ python ./run_mlm_flax_no_accum.py \
15
  --adam_beta1="0.9" \
16
  --adam_beta2="0.98" \
17
  --logging_steps="50" \
18
- --eval_steps="5000" \
19
  --num_train_epochs="2"\
20
  --preprocessing_num_workers="96" \
21
  --save_steps="10000" \
 
15
  --adam_beta1="0.9" \
16
  --adam_beta2="0.98" \
17
  --logging_steps="50" \
18
+ --eval_steps="2000" \
19
  --num_train_epochs="2"\
20
  --preprocessing_num_workers="96" \
21
  --save_steps="10000" \
run_mlm_flax_no_accum.py CHANGED
@@ -59,6 +59,7 @@ from flax.training import checkpoints
59
  from flax.jax_utils import unreplicate
60
  from flax.training.checkpoints import save_checkpoint, restore_checkpoint
61
  from importlib.util import find_spec
 
62
 
63
 
64
  MODEL_CONFIG_CLASSES = list(FLAX_MODEL_FOR_MASKED_LM_MAPPING.keys())
@@ -270,16 +271,16 @@ def write_eval_metric(summary_writer, eval_metrics, step):
270
  for metric_name, value in eval_metrics.items():
271
  summary_writer.scalar(f"eval_{metric_name}", value, step)
272
 
273
- def rotate_checkpoints(ckpt_dir:str, save_total_limit:int):
274
- "Removes older checkpoints so that `save_total_limit` checkpoints are kept"
275
- # TODO: what to remove is decided using step number only, we might want to improve that
276
- ckpts = [str(x) for x in Path(ckpt_dir).glob("ckpt-*")]
277
- # sort checkpoints by step
278
- ckpts_sorted = sorted(ckpts, key=lambda x: int(x.split('-')[-1]))
279
- ckpts_to_delete = ckpts_sorted[:-save_total_limit]
280
- for ckpt in ckpts_to_delete:
281
- logger.info(f"Deleting older checkpoint [{ckpt}] due to save_total_limit ({save_total_limit})")
282
- shutil.rmtree(ckpt)
283
 
284
 
285
  if __name__ == "__main__":
@@ -530,7 +531,7 @@ if __name__ == "__main__":
530
  wandb.init(
531
  entity="wandb",
532
  project="hf-flax-pino-roberta",
533
- sync_tensorboard=True
534
  )
535
  wandb.config.update(training_args)
536
  wandb.config.update(model_args)
@@ -700,8 +701,8 @@ if __name__ == "__main__":
700
  train_metrics.append(train_metric)
701
 
702
  cur_step = epoch * (num_train_samples // train_batch_size) + step
703
- if cur_step < resume_step:
704
- continue
705
 
706
  if cur_step % training_args.logging_steps == 0 and cur_step > 0:
707
  # Save metrics
@@ -764,8 +765,8 @@ if __name__ == "__main__":
764
  push_to_hub=training_args.push_to_hub,
765
  commit_message=f"Saving weights and logs of step {cur_step}",
766
  )
767
- if training_args.save_total_limit is not None:
768
- rotate_checkpoints(training_args.output_dir, training_args.save_total_limit)
769
  if jax.process_index() == 0:
770
  params = jax.device_get(jax.tree_map(lambda x: x[0], state.params))
771
  model.save_pretrained(
 
59
  from flax.jax_utils import unreplicate
60
  from flax.training.checkpoints import save_checkpoint, restore_checkpoint
61
  from importlib.util import find_spec
62
+ import shutil
63
 
64
 
65
  MODEL_CONFIG_CLASSES = list(FLAX_MODEL_FOR_MASKED_LM_MAPPING.keys())
 
271
  for metric_name, value in eval_metrics.items():
272
  summary_writer.scalar(f"eval_{metric_name}", value, step)
273
 
274
+ #def rotate_checkpoints(ckpt_dir:str, save_total_limit:int):
275
+ # "Removes older checkpoints so that `save_total_limit` checkpoints are kept"
276
+ # # TODO: what to remove is decided using step number only, we might want to improve that
277
+ # ckpts = [str(x) for x in Path(ckpt_dir).glob("ckpt-*")]
278
+ # # sort checkpoints by step
279
+ # ckpts_sorted = sorted(ckpts, key=lambda x: int(x.split('-')[-1]))
280
+ # ckpts_to_delete = ckpts_sorted[:-save_total_limit]
281
+ # for ckpt in ckpts_to_delete:
282
+ # logger.info(f"Deleting older checkpoint [{ckpt}] due to save_total_limit ({save_total_limit})")
283
+ # shutil.rmtree(ckpt)
284
 
285
 
286
  if __name__ == "__main__":
 
531
  wandb.init(
532
  entity="wandb",
533
  project="hf-flax-pino-roberta",
534
+ sync_tensorboard=True,
535
  )
536
  wandb.config.update(training_args)
537
  wandb.config.update(model_args)
 
701
  train_metrics.append(train_metric)
702
 
703
  cur_step = epoch * (num_train_samples // train_batch_size) + step
704
+ #if cur_step < resume_step:
705
+ # continue
706
 
707
  if cur_step % training_args.logging_steps == 0 and cur_step > 0:
708
  # Save metrics
 
765
  push_to_hub=training_args.push_to_hub,
766
  commit_message=f"Saving weights and logs of step {cur_step}",
767
  )
768
+ #if training_args.save_total_limit is not None:
769
+ # rotate_checkpoints(training_args.output_dir, training_args.save_total_limit)
770
  if jax.process_index() == 0:
771
  params = jax.device_get(jax.tree_map(lambda x: x[0], state.params))
772
  model.save_pretrained(
wandb/debug-internal.log CHANGED
@@ -1 +1 @@
1
- run-20210715_031107-69jkygz3/logs/debug-internal.log
 
1
+ run-20210715_142215-1vry5yso/logs/debug-internal.log
wandb/debug.log CHANGED
@@ -1 +1 @@
1
- run-20210715_031107-69jkygz3/logs/debug.log
 
1
+ run-20210715_142215-1vry5yso/logs/debug.log
wandb/latest-run CHANGED
@@ -1 +1 @@
1
- run-20210715_031107-69jkygz3
 
1
+ run-20210715_142215-1vry5yso
wandb/run-20210715_031107-69jkygz3/files/output.log CHANGED
@@ -10534,3 +10534,33 @@ tcmalloc: large alloc 2715181056 bytes == 0x2d6562000 @ 0x7f4e39cd4680 0x7f4e39
10534
  [13:26:07] - INFO - absl - Saved checkpoint at checkpoint_90000
10535
  [13:26:09] - INFO - huggingface_hub.repository - git version 2.25.1
10536
  git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10534
  [13:26:07] - INFO - absl - Saved checkpoint at checkpoint_90000
10535
  [13:26:09] - INFO - huggingface_hub.repository - git version 2.25.1
10536
  git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
10537
+ [13:26:09] - DEBUG - huggingface_hub.repository - [Repository] is a valid git repo
10538
+ [13:27:29] - INFO - huggingface_hub.repository - Uploading LFS objects: 100% (3/3), 2.1 GB | 43 MB/s, done.
10539
+
10540
+
10541
+
10542
+ Training...: 26%|██████████████▊ | 90050/352766 [10:10:46<26:20:31, 2.77it/s]
10543
+
10544
+
10545
+
10546
+
10547
+ Training...: 26%|██████████████▊ | 90100/352766 [10:11:06<26:08:06, 2.79it/s]
10548
+
10549
+ Step... (80000 | Loss: 2.5656967163085938, Acc: 0.533167839050293): 0%| | 0/5 [10:16:51<?, ?it/s]
10550
+ Traceback (most recent call last):
10551
+ File "./run_mlm_flax_no_accum.py", line 694, in <module>
10552
+ samples = [tokenized_datasets["train"][int(idx)] for idx in batch_idx]
10553
+ File "./run_mlm_flax_no_accum.py", line 694, in <listcomp>
10554
+ samples = [tokenized_datasets["train"][int(idx)] for idx in batch_idx]
10555
+ File "/home/dat/datasets/src/datasets/arrow_dataset.py", line 1517, in __getitem__
10556
+ return self._getitem(
10557
+ File "/home/dat/datasets/src/datasets/arrow_dataset.py", line 1510, in _getitem
10558
+ formatted_output = format_table(
10559
+ File "/home/dat/datasets/src/datasets/formatting/formatting.py", line 414, in format_table
10560
+ return formatter(pa_table, query_type=query_type)
10561
+ File "/home/dat/datasets/src/datasets/formatting/formatting.py", line 194, in __call__
10562
+ return self.format_row(pa_table)
10563
+ File "/home/dat/datasets/src/datasets/formatting/formatting.py", line 223, in format_row
10564
+ return self.python_arrow_extractor().extract_row(pa_table)
10565
+ File "/home/dat/datasets/src/datasets/formatting/formatting.py", line 134, in extract_row
10566
+ return _unnest(pa_table.to_pydict())
wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json CHANGED
@@ -1 +1 @@
1
- {"training_step": 90000, "learning_rate": 2.863156441890169e-05, "train_loss": 2.512800693511963, "_runtime": 36892, "_timestamp": 1626355559, "_step": 1803, "eval_step": 80000, "eval_accuracy": 0.533167839050293, "eval_loss": 2.5656967163085938}
 
1
+ {"training_step": 90100, "learning_rate": 2.8629854568862356e-05, "train_loss": 2.7210891246795654, "_runtime": 37024, "_timestamp": 1626355691, "_step": 1805, "eval_step": 80000, "eval_accuracy": 0.533167839050293, "eval_loss": 2.5656967163085938}
wandb/run-20210715_031107-69jkygz3/logs/debug-internal.log CHANGED
@@ -23072,3 +23072,55 @@
23072
  2021-07-15 13:26:12,804 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
23073
  2021-07-15 13:26:26,096 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
23074
  2021-07-15 13:26:26,097 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23072
  2021-07-15 13:26:12,804 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
23073
  2021-07-15 13:26:26,096 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
23074
  2021-07-15 13:26:26,097 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
23075
+ 2021-07-15 13:26:41,229 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
23076
+ 2021-07-15 13:26:41,229 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
23077
+ 2021-07-15 13:26:42,234 DEBUG SenderThread:656728 [sender.py:send():179] send: stats
23078
+ 2021-07-15 13:26:56,477 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
23079
+ 2021-07-15 13:26:56,478 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
23080
+ 2021-07-15 13:27:11,612 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
23081
+ 2021-07-15 13:27:11,612 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
23082
+ 2021-07-15 13:27:12,309 DEBUG SenderThread:656728 [sender.py:send():179] send: stats
23083
+ 2021-07-15 13:27:26,746 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
23084
+ 2021-07-15 13:27:26,747 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
23085
+ 2021-07-15 13:27:32,838 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
23086
+ 2021-07-15 13:27:34,839 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
23087
+ 2021-07-15 13:27:36,840 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
23088
+ 2021-07-15 13:27:38,842 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
23089
+ 2021-07-15 13:27:41,877 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
23090
+ 2021-07-15 13:27:41,878 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
23091
+ 2021-07-15 13:27:42,386 DEBUG SenderThread:656728 [sender.py:send():179] send: stats
23092
+ 2021-07-15 13:27:50,988 DEBUG SenderThread:656728 [sender.py:send():179] send: history
23093
+ 2021-07-15 13:27:50,988 DEBUG SenderThread:656728 [sender.py:send():179] send: summary
23094
+ 2021-07-15 13:27:50,989 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
23095
+ 2021-07-15 13:27:51,847 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json
23096
+ 2021-07-15 13:27:52,848 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
23097
+ 2021-07-15 13:27:54,849 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
23098
+ 2021-07-15 13:27:56,850 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
23099
+ 2021-07-15 13:27:57,009 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
23100
+ 2021-07-15 13:27:57,009 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
23101
+ 2021-07-15 13:27:58,851 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
23102
+ 2021-07-15 13:28:11,019 DEBUG SenderThread:656728 [sender.py:send():179] send: history
23103
+ 2021-07-15 13:28:11,019 DEBUG SenderThread:656728 [sender.py:send():179] send: summary
23104
+ 2021-07-15 13:28:11,019 INFO SenderThread:656728 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
23105
+ 2021-07-15 13:28:11,857 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json
23106
+ 2021-07-15 13:28:12,168 DEBUG HandlerThread:656728 [handler.py:handle_request():124] handle_request: stop_status
23107
+ 2021-07-15 13:28:12,169 DEBUG SenderThread:656728 [sender.py:send_request():193] send_request: stop_status
23108
+ 2021-07-15 13:28:12,464 DEBUG SenderThread:656728 [sender.py:send():179] send: stats
23109
+ 2021-07-15 13:28:12,857 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
23110
+ 2021-07-15 13:28:13,729 WARNING MainThread:656728 [internal.py:wandb_internal():147] Internal process interrupt: 1
23111
+ 2021-07-15 13:28:14,628 WARNING MainThread:656728 [internal.py:wandb_internal():147] Internal process interrupt: 2
23112
+ 2021-07-15 13:28:14,628 ERROR MainThread:656728 [internal.py:wandb_internal():150] Internal process interrupted.
23113
+ 2021-07-15 13:28:14,858 INFO Thread-8 :656728 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log
23114
+ 2021-07-15 13:28:15,239 INFO SenderThread:656728 [sender.py:finish():945] shutting down sender
23115
+ 2021-07-15 13:28:15,240 INFO WriterThread:656728 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/run-69jkygz3.wandb
23116
+ 2021-07-15 13:28:15,240 INFO SenderThread:656728 [dir_watcher.py:finish():282] shutting down directory watcher
23117
+ 2021-07-15 13:28:15,240 INFO HandlerThread:656728 [handler.py:finish():638] shutting down handler
23118
+ 2021-07-15 13:28:15,859 INFO SenderThread:656728 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files
23119
+ 2021-07-15 13:28:15,860 INFO SenderThread:656728 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/requirements.txt requirements.txt
23120
+ 2021-07-15 13:28:15,860 INFO SenderThread:656728 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/output.log output.log
23121
+ 2021-07-15 13:28:15,860 INFO SenderThread:656728 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-metadata.json wandb-metadata.json
23122
+ 2021-07-15 13:28:15,860 INFO SenderThread:656728 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/config.yaml config.yaml
23123
+ 2021-07-15 13:28:15,860 INFO SenderThread:656728 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210715_031107-69jkygz3/files/wandb-summary.json wandb-summary.json
23124
+ 2021-07-15 13:28:15,861 INFO SenderThread:656728 [file_pusher.py:finish():177] shutting down file pusher
23125
+ 2021-07-15 13:28:15,861 INFO SenderThread:656728 [file_pusher.py:join():182] waiting for file pusher
23126
+ 2021-07-15 13:28:16,239 INFO MainThread:656728 [internal.py:handle_exit():78] Internal process exited
wandb/run-20210715_031107-69jkygz3/logs/debug.log CHANGED
@@ -23,3 +23,5 @@ config: {}
23
  2021-07-15 03:11:09,649 INFO MainThread:655476 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 3e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul15_03-10-59_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 50, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 30000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 20000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
24
  2021-07-15 03:11:09,651 INFO MainThread:655476 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
25
  2021-07-15 03:11:09,653 INFO MainThread:655476 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 500}
 
 
 
23
  2021-07-15 03:11:09,649 INFO MainThread:655476 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 3e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul15_03-10-59_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 50, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 30000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 20000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
24
  2021-07-15 03:11:09,651 INFO MainThread:655476 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
25
  2021-07-15 03:11:09,653 INFO MainThread:655476 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 500}
26
+ 2021-07-15 13:28:13,756 INFO MainThread:655476 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 255
27
+ 2021-07-15 13:28:13,756 INFO MainThread:655476 [wandb_run.py:_restore():1565] restore
wandb/run-20210715_031107-69jkygz3/run-69jkygz3.wandb CHANGED
Binary files a/wandb/run-20210715_031107-69jkygz3/run-69jkygz3.wandb and b/wandb/run-20210715_031107-69jkygz3/run-69jkygz3.wandb differ
 
wandb/run-20210715_132956-37l0zsn1/files/config.yaml ADDED
@@ -0,0 +1,301 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ wandb_version: 1
2
+
3
+ _wandb:
4
+ desc: null
5
+ value:
6
+ cli_version: 0.10.33
7
+ framework: huggingface
8
+ huggingface_version: 4.9.0.dev0
9
+ is_jupyter_run: false
10
+ is_kaggle_kernel: false
11
+ python_version: 3.8.10
12
+ t:
13
+ 1:
14
+ - 3
15
+ - 11
16
+ 4: 3.8.10
17
+ 5: 0.10.33
18
+ 6: 4.9.0.dev0
19
+ 8:
20
+ - 5
21
+ adafactor:
22
+ desc: null
23
+ value: false
24
+ adam_beta1:
25
+ desc: null
26
+ value: 0.9
27
+ adam_beta2:
28
+ desc: null
29
+ value: 0.98
30
+ adam_epsilon:
31
+ desc: null
32
+ value: 1.0e-08
33
+ cache_dir:
34
+ desc: null
35
+ value: null
36
+ config_name:
37
+ desc: null
38
+ value: ./
39
+ dataloader_drop_last:
40
+ desc: null
41
+ value: false
42
+ dataloader_num_workers:
43
+ desc: null
44
+ value: 0
45
+ dataloader_pin_memory:
46
+ desc: null
47
+ value: true
48
+ dataset_config_name:
49
+ desc: null
50
+ value: null
51
+ dataset_name:
52
+ desc: null
53
+ value: null
54
+ ddp_find_unused_parameters:
55
+ desc: null
56
+ value: null
57
+ debug:
58
+ desc: null
59
+ value: []
60
+ deepspeed:
61
+ desc: null
62
+ value: null
63
+ disable_tqdm:
64
+ desc: null
65
+ value: false
66
+ do_eval:
67
+ desc: null
68
+ value: false
69
+ do_predict:
70
+ desc: null
71
+ value: false
72
+ do_train:
73
+ desc: null
74
+ value: false
75
+ dtype:
76
+ desc: null
77
+ value: float32
78
+ eval_accumulation_steps:
79
+ desc: null
80
+ value: null
81
+ eval_steps:
82
+ desc: null
83
+ value: 2000
84
+ evaluation_strategy:
85
+ desc: null
86
+ value: IntervalStrategy.NO
87
+ fp16:
88
+ desc: null
89
+ value: false
90
+ fp16_backend:
91
+ desc: null
92
+ value: auto
93
+ fp16_full_eval:
94
+ desc: null
95
+ value: false
96
+ fp16_opt_level:
97
+ desc: null
98
+ value: O1
99
+ gradient_accumulation_steps:
100
+ desc: null
101
+ value: 1
102
+ greater_is_better:
103
+ desc: null
104
+ value: null
105
+ group_by_length:
106
+ desc: null
107
+ value: false
108
+ ignore_data_skip:
109
+ desc: null
110
+ value: false
111
+ label_names:
112
+ desc: null
113
+ value: null
114
+ label_smoothing_factor:
115
+ desc: null
116
+ value: 0.0
117
+ learning_rate:
118
+ desc: null
119
+ value: 3.0e-05
120
+ length_column_name:
121
+ desc: null
122
+ value: length
123
+ line_by_line:
124
+ desc: null
125
+ value: false
126
+ load_best_model_at_end:
127
+ desc: null
128
+ value: false
129
+ local_rank:
130
+ desc: null
131
+ value: -1
132
+ log_level:
133
+ desc: null
134
+ value: -1
135
+ log_level_replica:
136
+ desc: null
137
+ value: -1
138
+ log_on_each_node:
139
+ desc: null
140
+ value: true
141
+ logging_dir:
142
+ desc: null
143
+ value: ./runs/Jul15_13-29-47_t1v-n-f5c06ea1-w-0
144
+ logging_first_step:
145
+ desc: null
146
+ value: false
147
+ logging_steps:
148
+ desc: null
149
+ value: 50
150
+ logging_strategy:
151
+ desc: null
152
+ value: IntervalStrategy.STEPS
153
+ lr_scheduler_type:
154
+ desc: null
155
+ value: SchedulerType.LINEAR
156
+ max_eval_samples:
157
+ desc: null
158
+ value: 5000
159
+ max_grad_norm:
160
+ desc: null
161
+ value: 1.0
162
+ max_seq_length:
163
+ desc: null
164
+ value: 4096
165
+ max_steps:
166
+ desc: null
167
+ value: -1
168
+ metric_for_best_model:
169
+ desc: null
170
+ value: null
171
+ mlm_probability:
172
+ desc: null
173
+ value: 0.15
174
+ model_name_or_path:
175
+ desc: null
176
+ value: null
177
+ model_type:
178
+ desc: null
179
+ value: big_bird
180
+ mp_parameters:
181
+ desc: null
182
+ value: ''
183
+ no_cuda:
184
+ desc: null
185
+ value: false
186
+ num_train_epochs:
187
+ desc: null
188
+ value: 2.0
189
+ output_dir:
190
+ desc: null
191
+ value: ./
192
+ overwrite_cache:
193
+ desc: null
194
+ value: false
195
+ overwrite_output_dir:
196
+ desc: null
197
+ value: true
198
+ pad_to_max_length:
199
+ desc: null
200
+ value: false
201
+ past_index:
202
+ desc: null
203
+ value: -1
204
+ per_device_eval_batch_size:
205
+ desc: null
206
+ value: 1
207
+ per_device_train_batch_size:
208
+ desc: null
209
+ value: 1
210
+ per_gpu_eval_batch_size:
211
+ desc: null
212
+ value: null
213
+ per_gpu_train_batch_size:
214
+ desc: null
215
+ value: null
216
+ prediction_loss_only:
217
+ desc: null
218
+ value: false
219
+ preprocessing_num_workers:
220
+ desc: null
221
+ value: 96
222
+ push_to_hub:
223
+ desc: null
224
+ value: true
225
+ push_to_hub_model_id:
226
+ desc: null
227
+ value: ''
228
+ push_to_hub_organization:
229
+ desc: null
230
+ value: null
231
+ push_to_hub_token:
232
+ desc: null
233
+ value: null
234
+ remove_unused_columns:
235
+ desc: null
236
+ value: true
237
+ report_to:
238
+ desc: null
239
+ value:
240
+ - tensorboard
241
+ - wandb
242
+ resume_from_checkpoint:
243
+ desc: null
244
+ value: ./
245
+ run_name:
246
+ desc: null
247
+ value: ./
248
+ save_on_each_node:
249
+ desc: null
250
+ value: false
251
+ save_steps:
252
+ desc: null
253
+ value: 10000
254
+ save_strategy:
255
+ desc: null
256
+ value: IntervalStrategy.STEPS
257
+ save_total_limit:
258
+ desc: null
259
+ value: 5
260
+ seed:
261
+ desc: null
262
+ value: 42
263
+ sharded_ddp:
264
+ desc: null
265
+ value: []
266
+ skip_memory_metrics:
267
+ desc: null
268
+ value: true
269
+ tokenizer_name:
270
+ desc: null
271
+ value: ./
272
+ tpu_metrics_debug:
273
+ desc: null
274
+ value: false
275
+ tpu_num_cores:
276
+ desc: null
277
+ value: null
278
+ train_ref_file:
279
+ desc: null
280
+ value: null
281
+ use_fast_tokenizer:
282
+ desc: null
283
+ value: true
284
+ use_legacy_prediction_loop:
285
+ desc: null
286
+ value: false
287
+ validation_ref_file:
288
+ desc: null
289
+ value: null
290
+ validation_split_percentage:
291
+ desc: null
292
+ value: 5
293
+ warmup_ratio:
294
+ desc: null
295
+ value: 0.0
296
+ warmup_steps:
297
+ desc: null
298
+ value: 10000
299
+ weight_decay:
300
+ desc: null
301
+ value: 0.0095
wandb/run-20210715_132956-37l0zsn1/files/output.log ADDED
@@ -0,0 +1,195 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [13:30:11] - INFO - absl - Restoring checkpoint from ./checkpoint_90000
2
+ tcmalloc: large alloc 1530273792 bytes == 0x9b1a4000 @ 0x7fd1962e2680 0x7fd196303824 0x5b9a14 0x50b2ae 0x50cb1b 0x5a6f17 0x5f3010 0x56fd36 0x568d9a 0x5f5b33 0x56aadf 0x568d9a 0x68cdc7 0x67e161 0x67e1df 0x67e281 0x67e627 0x6b6e62 0x6b71ed 0x7fd1960f70b3 0x5f96de
3
+ /home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
4
+ warnings.warn(
5
+ /home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
6
+ warnings.warn(
7
+ Epoch ... (1/2): 0%| | 0/2 [00:00<?, ?it/s]
8
+
9
+
10
+
11
+
12
+
13
+
14
+
15
+
16
+
17
+
18
+
19
+
20
+
21
+
22
+
23
+
24
+
25
+
26
+
27
+
28
+
29
+
30
+
31
+
32
+
33
+
34
+
35
+
36
+
37
+
38
+
39
+
40
+
41
+
42
+
43
+
44
+
45
+
46
+
47
+
48
+
49
+
50
+
51
+
52
+
53
+
54
+
55
+
56
+
57
+
58
+
59
+
60
+
61
+
62
+
63
+
64
+
65
+
66
+
67
+
68
+
69
+
70
+
71
+
72
+
73
+
74
+
75
+
76
+
77
+
78
+
79
+
80
+
81
+
82
+
83
+
84
+
85
+
86
+
87
+
88
+
89
+
90
+
91
+
92
+
93
+
94
+
95
+
96
+
97
+
98
+
99
+
100
+
101
+
102
+
103
+
104
+
105
+
106
+
107
+
108
+
109
+
110
+
111
+
112
+
113
+
114
+
115
+
116
+
117
+
118
+
119
+
120
+
121
+
122
+
123
+
124
+
125
+
126
+
127
+
128
+
129
+
130
+
131
+
132
+
133
+
134
+
135
+
136
+
137
+
138
+
139
+
140
+
141
+
142
+
143
+
144
+
145
+
146
+
147
+
148
+
149
+
150
+
151
+
152
+
153
+
154
+
155
+
156
+
157
+
158
+
159
+
160
+
161
+
162
+
163
+
164
+
165
+
166
+
167
+
168
+
169
+
170
+
171
+
172
+
173
+
174
+
175
+
176
+
177
+
178
+
179
+
180
+
181
+
182
+
183
+
184
+
185
+
186
+
187
+
188
+
189
+
190
+
191
+
192
+
193
+
194
+
195
+
wandb/run-20210715_132956-37l0zsn1/files/requirements.txt ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ absl-py==0.13.0
2
+ aiohttp==3.7.4.post0
3
+ astunparse==1.6.3
4
+ async-timeout==3.0.1
5
+ attrs==21.2.0
6
+ cachetools==4.2.2
7
+ certifi==2021.5.30
8
+ chardet==4.0.0
9
+ charset-normalizer==2.0.1
10
+ chex==0.0.8
11
+ click==8.0.1
12
+ configparser==5.0.2
13
+ cycler==0.10.0
14
+ datasets==1.9.1.dev0
15
+ dill==0.3.4
16
+ dm-tree==0.1.6
17
+ docker-pycreds==0.4.0
18
+ filelock==3.0.12
19
+ flatbuffers==1.12
20
+ flax==0.3.4
21
+ fsspec==2021.7.0
22
+ gast==0.4.0
23
+ gitdb==4.0.7
24
+ gitpython==3.1.18
25
+ google-auth-oauthlib==0.4.4
26
+ google-auth==1.32.1
27
+ google-pasta==0.2.0
28
+ grpcio==1.34.1
29
+ h5py==3.1.0
30
+ huggingface-hub==0.0.12
31
+ idna==3.2
32
+ install==1.3.4
33
+ jax==0.2.17
34
+ jaxlib==0.1.68
35
+ joblib==1.0.1
36
+ keras-nightly==2.5.0.dev2021032900
37
+ keras-preprocessing==1.1.2
38
+ kiwisolver==1.3.1
39
+ libtpu-nightly==0.1.dev20210615
40
+ markdown==3.3.4
41
+ matplotlib==3.4.2
42
+ msgpack==1.0.2
43
+ multidict==5.1.0
44
+ multiprocess==0.70.12.2
45
+ numpy==1.19.5
46
+ oauthlib==3.1.1
47
+ opt-einsum==3.3.0
48
+ optax==0.0.9
49
+ packaging==21.0
50
+ pandas==1.3.0
51
+ pathtools==0.1.2
52
+ pillow==8.3.1
53
+ pip==20.0.2
54
+ pkg-resources==0.0.0
55
+ promise==2.3
56
+ protobuf==3.17.3
57
+ psutil==5.8.0
58
+ pyarrow==4.0.1
59
+ pyasn1-modules==0.2.8
60
+ pyasn1==0.4.8
61
+ pyparsing==2.4.7
62
+ python-dateutil==2.8.1
63
+ pytz==2021.1
64
+ pyyaml==5.4.1
65
+ regex==2021.7.6
66
+ requests-oauthlib==1.3.0
67
+ requests==2.26.0
68
+ rsa==4.7.2
69
+ sacremoses==0.0.45
70
+ scipy==1.7.0
71
+ sentry-sdk==1.3.0
72
+ setuptools==44.0.0
73
+ shortuuid==1.0.1
74
+ six==1.15.0
75
+ smmap==4.0.0
76
+ subprocess32==3.5.4
77
+ tensorboard-data-server==0.6.1
78
+ tensorboard-plugin-wit==1.8.0
79
+ tensorboard==2.5.0
80
+ tensorflow-estimator==2.5.0
81
+ tensorflow==2.5.0
82
+ termcolor==1.1.0
83
+ tokenizers==0.10.3
84
+ toolz==0.11.1
85
+ tqdm==4.61.2
86
+ transformers==4.9.0.dev0
87
+ typing-extensions==3.7.4.3
88
+ urllib3==1.26.6
89
+ wandb==0.10.33
90
+ werkzeug==2.0.1
91
+ wheel==0.36.2
92
+ wrapt==1.12.1
93
+ xxhash==2.0.2
94
+ yarl==1.6.3
wandb/run-20210715_132956-37l0zsn1/files/wandb-metadata.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
3
+ "python": "3.8.10",
4
+ "heartbeatAt": "2021-07-15T13:29:58.227318",
5
+ "startedAt": "2021-07-15T13:29:56.221807",
6
+ "docker": null,
7
+ "cpu_count": 96,
8
+ "cuda": null,
9
+ "args": [
10
+ "--push_to_hub",
11
+ "--output_dir=./",
12
+ "--model_type=big_bird",
13
+ "--config_name=./",
14
+ "--tokenizer_name=./",
15
+ "--max_seq_length=4096",
16
+ "--weight_decay=0.0095",
17
+ "--warmup_steps=10000",
18
+ "--overwrite_output_dir",
19
+ "--adam_beta1=0.9",
20
+ "--adam_beta2=0.98",
21
+ "--logging_steps=50",
22
+ "--eval_steps=2000",
23
+ "--num_train_epochs=2",
24
+ "--preprocessing_num_workers=96",
25
+ "--save_steps=10000",
26
+ "--learning_rate=3e-5",
27
+ "--per_device_train_batch_size=1",
28
+ "--per_device_eval_batch_size=1",
29
+ "--save_total_limit=5",
30
+ "--max_eval_samples=5000",
31
+ "--resume_from_checkpoint=./"
32
+ ],
33
+ "state": "running",
34
+ "program": "./run_mlm_flax_no_accum.py",
35
+ "codePath": "run_mlm_flax_no_accum.py",
36
+ "git": {
37
+ "remote": "https://huggingface.co/flax-community/pino-roberta-base",
38
+ "commit": "b6814517994e86d862ddec99dda42b9dbea5b8b0"
39
+ },
40
+ "email": null,
41
+ "root": "/home/dat/pino-roberta-base",
42
+ "host": "t1v-n-f5c06ea1-w-0",
43
+ "username": "dat",
44
+ "executable": "/home/dat/pino/bin/python"
45
+ }
wandb/run-20210715_132956-37l0zsn1/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {}
wandb/run-20210715_132956-37l0zsn1/logs/debug-internal.log ADDED
@@ -0,0 +1,391 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-07-15 13:29:56,917 INFO MainThread:690242 [internal.py:wandb_internal():88] W&B internal server running at pid: 690242, started at: 2021-07-15 13:29:56.916849
2
+ 2021-07-15 13:29:56,919 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: check_version
3
+ 2021-07-15 13:29:56,919 INFO WriterThread:690242 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/run-37l0zsn1.wandb
4
+ 2021-07-15 13:29:56,920 DEBUG SenderThread:690242 [sender.py:send():179] send: header
5
+ 2021-07-15 13:29:56,920 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: check_version
6
+ 2021-07-15 13:29:56,960 DEBUG SenderThread:690242 [sender.py:send():179] send: run
7
+ 2021-07-15 13:29:57,133 INFO SenderThread:690242 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files
8
+ 2021-07-15 13:29:57,133 INFO SenderThread:690242 [sender.py:_start_run_threads():716] run started: 37l0zsn1 with start time 1626355796
9
+ 2021-07-15 13:29:57,133 DEBUG SenderThread:690242 [sender.py:send():179] send: summary
10
+ 2021-07-15 13:29:57,133 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: run_start
11
+ 2021-07-15 13:29:57,134 INFO SenderThread:690242 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
12
+ 2021-07-15 13:29:58,138 INFO Thread-8 :690242 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/wandb-summary.json
13
+ 2021-07-15 13:29:58,227 DEBUG HandlerThread:690242 [meta.py:__init__():39] meta init
14
+ 2021-07-15 13:29:58,227 DEBUG HandlerThread:690242 [meta.py:__init__():53] meta init done
15
+ 2021-07-15 13:29:58,227 DEBUG HandlerThread:690242 [meta.py:probe():210] probe
16
+ 2021-07-15 13:29:58,228 DEBUG HandlerThread:690242 [meta.py:_setup_git():200] setup git
17
+ 2021-07-15 13:29:58,258 DEBUG HandlerThread:690242 [meta.py:_setup_git():207] setup git done
18
+ 2021-07-15 13:29:58,258 DEBUG HandlerThread:690242 [meta.py:_save_pip():57] save pip
19
+ 2021-07-15 13:29:58,259 DEBUG HandlerThread:690242 [meta.py:_save_pip():71] save pip done
20
+ 2021-07-15 13:29:58,259 DEBUG HandlerThread:690242 [meta.py:probe():252] probe done
21
+ 2021-07-15 13:29:58,262 DEBUG SenderThread:690242 [sender.py:send():179] send: files
22
+ 2021-07-15 13:29:58,262 INFO SenderThread:690242 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now
23
+ 2021-07-15 13:29:58,270 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
24
+ 2021-07-15 13:29:58,270 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
25
+ 2021-07-15 13:29:58,400 DEBUG SenderThread:690242 [sender.py:send():179] send: config
26
+ 2021-07-15 13:29:58,401 DEBUG SenderThread:690242 [sender.py:send():179] send: config
27
+ 2021-07-15 13:29:58,401 DEBUG SenderThread:690242 [sender.py:send():179] send: config
28
+ 2021-07-15 13:29:58,737 INFO Thread-11 :690242 [upload_job.py:push():137] Uploaded file /tmp/tmps2g86y4ywandb/37km806o-wandb-metadata.json
29
+ 2021-07-15 13:29:59,137 INFO Thread-8 :690242 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
30
+ 2021-07-15 13:29:59,138 INFO Thread-8 :690242 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/wandb-metadata.json
31
+ 2021-07-15 13:29:59,138 INFO Thread-8 :690242 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/requirements.txt
32
+ 2021-07-15 13:30:13,740 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
33
+ 2021-07-15 13:30:13,740 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
34
+ 2021-07-15 13:30:15,144 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
35
+ 2021-07-15 13:30:17,144 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
36
+ 2021-07-15 13:30:26,310 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
37
+ 2021-07-15 13:30:28,149 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/config.yaml
38
+ 2021-07-15 13:30:28,945 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
39
+ 2021-07-15 13:30:28,945 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
40
+ 2021-07-15 13:30:44,077 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
41
+ 2021-07-15 13:30:44,077 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
42
+ 2021-07-15 13:30:56,384 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
43
+ 2021-07-15 13:30:59,224 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
44
+ 2021-07-15 13:30:59,225 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
45
+ 2021-07-15 13:31:14,355 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
46
+ 2021-07-15 13:31:14,355 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
47
+ 2021-07-15 13:31:26,457 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
48
+ 2021-07-15 13:31:29,490 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
49
+ 2021-07-15 13:31:29,490 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
50
+ 2021-07-15 13:31:44,622 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
51
+ 2021-07-15 13:31:44,623 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
52
+ 2021-07-15 13:31:56,528 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
53
+ 2021-07-15 13:31:59,753 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
54
+ 2021-07-15 13:31:59,754 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
55
+ 2021-07-15 13:32:14,887 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
56
+ 2021-07-15 13:32:14,887 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
57
+ 2021-07-15 13:32:26,603 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
58
+ 2021-07-15 13:32:30,023 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
59
+ 2021-07-15 13:32:30,024 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
60
+ 2021-07-15 13:32:45,158 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
61
+ 2021-07-15 13:32:45,158 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
62
+ 2021-07-15 13:32:56,677 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
63
+ 2021-07-15 13:33:00,291 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
64
+ 2021-07-15 13:33:00,291 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
65
+ 2021-07-15 13:33:15,424 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
66
+ 2021-07-15 13:33:15,425 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
67
+ 2021-07-15 13:33:26,741 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
68
+ 2021-07-15 13:33:30,559 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
69
+ 2021-07-15 13:33:30,559 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
70
+ 2021-07-15 13:33:45,691 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
71
+ 2021-07-15 13:33:45,692 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
72
+ 2021-07-15 13:33:56,798 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
73
+ 2021-07-15 13:34:00,825 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
74
+ 2021-07-15 13:34:00,825 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
75
+ 2021-07-15 13:34:15,959 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
76
+ 2021-07-15 13:34:15,959 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
77
+ 2021-07-15 13:34:26,856 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
78
+ 2021-07-15 13:34:31,092 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
79
+ 2021-07-15 13:34:31,093 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
80
+ 2021-07-15 13:34:46,236 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
81
+ 2021-07-15 13:34:46,236 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
82
+ 2021-07-15 13:34:56,922 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
83
+ 2021-07-15 13:35:01,377 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
84
+ 2021-07-15 13:35:01,377 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
85
+ 2021-07-15 13:35:16,510 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
86
+ 2021-07-15 13:35:16,511 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
87
+ 2021-07-15 13:35:26,998 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
88
+ 2021-07-15 13:35:31,642 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
89
+ 2021-07-15 13:35:31,642 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
90
+ 2021-07-15 13:35:45,257 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
91
+ 2021-07-15 13:35:46,786 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
92
+ 2021-07-15 13:35:46,787 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
93
+ 2021-07-15 13:35:57,072 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
94
+ 2021-07-15 13:36:01,936 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
95
+ 2021-07-15 13:36:01,936 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
96
+ 2021-07-15 13:36:17,084 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
97
+ 2021-07-15 13:36:17,084 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
98
+ 2021-07-15 13:36:27,147 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
99
+ 2021-07-15 13:36:32,219 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
100
+ 2021-07-15 13:36:32,219 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
101
+ 2021-07-15 13:36:47,351 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
102
+ 2021-07-15 13:36:47,351 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
103
+ 2021-07-15 13:36:57,222 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
104
+ 2021-07-15 13:37:02,481 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
105
+ 2021-07-15 13:37:02,482 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
106
+ 2021-07-15 13:37:17,616 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
107
+ 2021-07-15 13:37:17,617 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
108
+ 2021-07-15 13:37:27,306 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
109
+ 2021-07-15 13:37:32,749 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
110
+ 2021-07-15 13:37:32,749 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
111
+ 2021-07-15 13:37:45,303 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
112
+ 2021-07-15 13:37:47,971 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
113
+ 2021-07-15 13:37:47,971 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
114
+ 2021-07-15 13:37:57,376 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
115
+ 2021-07-15 13:38:03,134 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
116
+ 2021-07-15 13:38:03,135 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
117
+ 2021-07-15 13:38:18,276 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
118
+ 2021-07-15 13:38:18,277 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
119
+ 2021-07-15 13:38:27,456 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
120
+ 2021-07-15 13:38:33,418 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
121
+ 2021-07-15 13:38:33,419 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
122
+ 2021-07-15 13:38:48,552 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
123
+ 2021-07-15 13:38:48,553 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
124
+ 2021-07-15 13:38:57,533 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
125
+ 2021-07-15 13:39:03,683 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
126
+ 2021-07-15 13:39:03,683 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
127
+ 2021-07-15 13:39:18,813 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
128
+ 2021-07-15 13:39:18,814 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
129
+ 2021-07-15 13:39:27,608 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
130
+ 2021-07-15 13:39:34,015 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
131
+ 2021-07-15 13:39:34,015 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
132
+ 2021-07-15 13:39:49,151 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
133
+ 2021-07-15 13:39:49,152 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
134
+ 2021-07-15 13:39:57,698 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
135
+ 2021-07-15 13:40:01,356 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
136
+ 2021-07-15 13:40:03,357 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
137
+ 2021-07-15 13:40:04,295 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
138
+ 2021-07-15 13:40:04,296 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
139
+ 2021-07-15 13:40:05,357 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
140
+ 2021-07-15 13:40:07,358 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
141
+ 2021-07-15 13:40:09,359 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
142
+ 2021-07-15 13:40:12,361 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
143
+ 2021-07-15 13:40:14,362 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
144
+ 2021-07-15 13:40:16,362 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
145
+ 2021-07-15 13:40:18,363 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
146
+ 2021-07-15 13:40:19,564 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
147
+ 2021-07-15 13:40:19,564 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
148
+ 2021-07-15 13:40:20,364 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
149
+ 2021-07-15 13:40:22,365 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
150
+ 2021-07-15 13:40:24,366 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
151
+ 2021-07-15 13:40:26,367 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
152
+ 2021-07-15 13:40:27,772 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
153
+ 2021-07-15 13:40:28,367 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
154
+ 2021-07-15 13:40:30,368 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
155
+ 2021-07-15 13:40:32,369 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
156
+ 2021-07-15 13:40:34,371 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
157
+ 2021-07-15 13:40:34,696 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
158
+ 2021-07-15 13:40:34,697 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
159
+ 2021-07-15 13:40:36,372 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
160
+ 2021-07-15 13:40:38,373 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
161
+ 2021-07-15 13:40:40,374 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
162
+ 2021-07-15 13:40:42,374 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
163
+ 2021-07-15 13:40:44,375 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
164
+ 2021-07-15 13:40:46,376 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
165
+ 2021-07-15 13:40:48,377 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
166
+ 2021-07-15 13:40:49,831 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
167
+ 2021-07-15 13:40:49,831 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
168
+ 2021-07-15 13:40:50,378 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
169
+ 2021-07-15 13:40:52,379 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
170
+ 2021-07-15 13:40:54,380 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
171
+ 2021-07-15 13:40:56,381 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
172
+ 2021-07-15 13:40:57,856 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
173
+ 2021-07-15 13:40:58,382 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
174
+ 2021-07-15 13:41:00,383 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
175
+ 2021-07-15 13:41:02,384 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
176
+ 2021-07-15 13:41:04,385 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
177
+ 2021-07-15 13:41:04,961 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
178
+ 2021-07-15 13:41:04,961 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
179
+ 2021-07-15 13:41:06,386 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
180
+ 2021-07-15 13:41:08,387 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
181
+ 2021-07-15 13:41:10,388 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
182
+ 2021-07-15 13:41:12,389 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
183
+ 2021-07-15 13:41:14,390 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
184
+ 2021-07-15 13:41:16,391 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
185
+ 2021-07-15 13:41:18,392 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
186
+ 2021-07-15 13:41:20,092 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
187
+ 2021-07-15 13:41:20,092 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
188
+ 2021-07-15 13:41:20,393 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
189
+ 2021-07-15 13:41:22,394 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
190
+ 2021-07-15 13:41:24,395 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
191
+ 2021-07-15 13:41:26,396 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
192
+ 2021-07-15 13:41:27,927 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
193
+ 2021-07-15 13:41:28,396 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
194
+ 2021-07-15 13:41:30,397 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
195
+ 2021-07-15 13:41:32,398 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
196
+ 2021-07-15 13:41:34,399 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
197
+ 2021-07-15 13:41:35,222 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
198
+ 2021-07-15 13:41:35,222 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
199
+ 2021-07-15 13:41:36,400 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
200
+ 2021-07-15 13:41:38,401 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
201
+ 2021-07-15 13:41:40,402 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
202
+ 2021-07-15 13:41:42,403 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
203
+ 2021-07-15 13:41:44,404 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
204
+ 2021-07-15 13:41:46,405 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
205
+ 2021-07-15 13:41:48,406 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
206
+ 2021-07-15 13:41:50,352 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
207
+ 2021-07-15 13:41:50,352 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
208
+ 2021-07-15 13:41:50,407 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
209
+ 2021-07-15 13:41:52,408 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
210
+ 2021-07-15 13:41:54,409 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
211
+ 2021-07-15 13:41:56,409 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
212
+ 2021-07-15 13:41:57,995 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
213
+ 2021-07-15 13:41:58,410 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
214
+ 2021-07-15 13:42:00,411 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
215
+ 2021-07-15 13:42:02,412 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
216
+ 2021-07-15 13:42:04,413 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
217
+ 2021-07-15 13:42:05,483 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
218
+ 2021-07-15 13:42:05,484 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
219
+ 2021-07-15 13:42:06,413 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
220
+ 2021-07-15 13:42:08,414 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
221
+ 2021-07-15 13:42:10,415 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
222
+ 2021-07-15 13:42:12,416 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
223
+ 2021-07-15 13:42:14,417 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
224
+ 2021-07-15 13:42:16,418 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
225
+ 2021-07-15 13:42:18,418 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
226
+ 2021-07-15 13:42:20,419 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
227
+ 2021-07-15 13:42:20,615 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
228
+ 2021-07-15 13:42:20,616 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
229
+ 2021-07-15 13:42:22,420 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
230
+ 2021-07-15 13:42:24,421 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
231
+ 2021-07-15 13:42:26,422 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
232
+ 2021-07-15 13:42:28,066 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
233
+ 2021-07-15 13:42:28,422 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
234
+ 2021-07-15 13:42:30,423 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
235
+ 2021-07-15 13:42:32,424 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
236
+ 2021-07-15 13:42:34,425 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
237
+ 2021-07-15 13:42:35,983 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
238
+ 2021-07-15 13:42:35,983 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
239
+ 2021-07-15 13:42:36,426 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
240
+ 2021-07-15 13:42:38,427 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
241
+ 2021-07-15 13:42:40,427 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
242
+ 2021-07-15 13:42:42,428 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
243
+ 2021-07-15 13:42:44,429 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
244
+ 2021-07-15 13:42:46,430 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
245
+ 2021-07-15 13:42:48,431 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
246
+ 2021-07-15 13:42:50,432 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
247
+ 2021-07-15 13:42:51,117 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
248
+ 2021-07-15 13:42:51,117 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
249
+ 2021-07-15 13:42:52,433 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
250
+ 2021-07-15 13:42:54,434 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
251
+ 2021-07-15 13:42:56,435 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
252
+ 2021-07-15 13:42:58,133 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
253
+ 2021-07-15 13:42:58,435 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
254
+ 2021-07-15 13:43:00,436 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
255
+ 2021-07-15 13:43:02,437 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
256
+ 2021-07-15 13:43:04,438 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
257
+ 2021-07-15 13:43:06,248 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
258
+ 2021-07-15 13:43:06,249 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
259
+ 2021-07-15 13:43:06,439 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
260
+ 2021-07-15 13:43:08,440 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
261
+ 2021-07-15 13:43:10,441 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
262
+ 2021-07-15 13:43:12,442 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
263
+ 2021-07-15 13:43:14,443 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
264
+ 2021-07-15 13:43:16,444 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
265
+ 2021-07-15 13:43:18,445 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
266
+ 2021-07-15 13:43:20,446 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
267
+ 2021-07-15 13:43:21,382 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
268
+ 2021-07-15 13:43:21,383 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
269
+ 2021-07-15 13:43:22,446 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
270
+ 2021-07-15 13:43:24,447 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
271
+ 2021-07-15 13:43:26,448 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
272
+ 2021-07-15 13:43:28,203 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
273
+ 2021-07-15 13:43:28,449 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
274
+ 2021-07-15 13:43:30,450 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
275
+ 2021-07-15 13:43:32,451 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
276
+ 2021-07-15 13:43:34,452 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
277
+ 2021-07-15 13:43:36,453 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
278
+ 2021-07-15 13:43:36,515 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
279
+ 2021-07-15 13:43:36,516 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
280
+ 2021-07-15 13:43:38,453 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
281
+ 2021-07-15 13:43:40,454 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
282
+ 2021-07-15 13:43:42,455 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
283
+ 2021-07-15 13:43:44,456 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
284
+ 2021-07-15 13:43:46,457 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
285
+ 2021-07-15 13:43:48,458 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
286
+ 2021-07-15 13:43:50,459 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
287
+ 2021-07-15 13:43:51,649 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
288
+ 2021-07-15 13:43:51,649 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
289
+ 2021-07-15 13:43:52,460 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
290
+ 2021-07-15 13:43:54,461 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
291
+ 2021-07-15 13:43:56,462 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
292
+ 2021-07-15 13:43:58,275 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
293
+ 2021-07-15 13:43:58,463 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
294
+ 2021-07-15 13:44:00,464 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
295
+ 2021-07-15 13:44:02,465 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
296
+ 2021-07-15 13:44:04,466 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
297
+ 2021-07-15 13:44:06,467 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
298
+ 2021-07-15 13:44:06,780 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
299
+ 2021-07-15 13:44:06,781 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
300
+ 2021-07-15 13:44:08,468 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
301
+ 2021-07-15 13:44:10,469 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
302
+ 2021-07-15 13:44:12,469 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
303
+ 2021-07-15 13:44:14,470 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
304
+ 2021-07-15 13:44:16,471 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
305
+ 2021-07-15 13:44:18,472 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
306
+ 2021-07-15 13:44:20,473 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
307
+ 2021-07-15 13:44:21,910 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
308
+ 2021-07-15 13:44:21,911 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
309
+ 2021-07-15 13:44:22,474 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
310
+ 2021-07-15 13:44:24,474 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
311
+ 2021-07-15 13:44:26,475 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
312
+ 2021-07-15 13:44:28,347 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
313
+ 2021-07-15 13:44:28,476 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
314
+ 2021-07-15 13:44:30,477 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
315
+ 2021-07-15 13:44:32,477 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
316
+ 2021-07-15 13:44:34,478 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
317
+ 2021-07-15 13:44:36,479 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
318
+ 2021-07-15 13:44:37,042 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
319
+ 2021-07-15 13:44:37,042 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
320
+ 2021-07-15 13:44:38,480 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
321
+ 2021-07-15 13:44:40,481 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
322
+ 2021-07-15 13:44:42,482 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
323
+ 2021-07-15 13:44:44,482 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
324
+ 2021-07-15 13:44:46,483 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
325
+ 2021-07-15 13:44:48,484 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
326
+ 2021-07-15 13:44:50,485 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
327
+ 2021-07-15 13:44:52,173 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
328
+ 2021-07-15 13:44:52,174 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
329
+ 2021-07-15 13:44:52,486 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
330
+ 2021-07-15 13:44:54,486 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
331
+ 2021-07-15 13:44:56,487 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
332
+ 2021-07-15 13:44:58,423 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
333
+ 2021-07-15 13:44:58,488 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
334
+ 2021-07-15 13:45:00,489 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
335
+ 2021-07-15 13:45:02,489 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
336
+ 2021-07-15 13:45:04,490 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
337
+ 2021-07-15 13:45:06,491 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
338
+ 2021-07-15 13:45:07,305 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
339
+ 2021-07-15 13:45:07,305 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
340
+ 2021-07-15 13:45:08,492 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
341
+ 2021-07-15 13:45:10,493 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
342
+ 2021-07-15 13:45:12,493 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
343
+ 2021-07-15 13:45:14,494 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
344
+ 2021-07-15 13:45:16,495 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
345
+ 2021-07-15 13:45:18,496 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
346
+ 2021-07-15 13:45:20,496 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
347
+ 2021-07-15 13:45:22,436 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
348
+ 2021-07-15 13:45:22,436 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
349
+ 2021-07-15 13:45:22,497 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
350
+ 2021-07-15 13:45:24,498 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
351
+ 2021-07-15 13:45:27,500 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
352
+ 2021-07-15 13:45:28,499 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
353
+ 2021-07-15 13:45:29,502 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
354
+ 2021-07-15 13:45:31,503 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
355
+ 2021-07-15 13:45:33,504 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
356
+ 2021-07-15 13:45:35,504 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
357
+ 2021-07-15 13:45:37,505 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
358
+ 2021-07-15 13:45:37,572 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
359
+ 2021-07-15 13:45:37,572 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
360
+ 2021-07-15 13:45:39,506 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
361
+ 2021-07-15 13:45:41,507 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
362
+ 2021-07-15 13:45:43,508 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
363
+ 2021-07-15 13:45:45,509 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
364
+ 2021-07-15 13:45:47,510 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
365
+ 2021-07-15 13:45:49,511 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
366
+ 2021-07-15 13:45:51,512 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
367
+ 2021-07-15 13:45:52,704 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
368
+ 2021-07-15 13:45:52,705 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
369
+ 2021-07-15 13:45:53,512 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
370
+ 2021-07-15 13:45:55,513 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
371
+ 2021-07-15 13:45:57,514 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
372
+ 2021-07-15 13:45:58,571 DEBUG SenderThread:690242 [sender.py:send():179] send: stats
373
+ 2021-07-15 13:45:59,515 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
374
+ 2021-07-15 13:46:01,516 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
375
+ 2021-07-15 13:46:03,516 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
376
+ 2021-07-15 13:46:05,517 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
377
+ 2021-07-15 13:46:07,518 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
378
+ 2021-07-15 13:46:07,841 DEBUG HandlerThread:690242 [handler.py:handle_request():124] handle_request: stop_status
379
+ 2021-07-15 13:46:07,841 DEBUG SenderThread:690242 [sender.py:send_request():193] send_request: stop_status
380
+ 2021-07-15 13:46:09,519 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
381
+ 2021-07-15 13:46:11,520 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
382
+ 2021-07-15 13:46:13,520 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
383
+ 2021-07-15 13:46:15,521 INFO Thread-8 :690242 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/files/output.log
384
+ 2021-07-15 13:46:15,809 WARNING MainThread:690242 [internal.py:wandb_internal():147] Internal process interrupt: 1
385
+ 2021-07-15 13:46:15,991 WARNING MainThread:690242 [internal.py:wandb_internal():147] Internal process interrupt: 2
386
+ 2021-07-15 13:46:15,991 ERROR MainThread:690242 [internal.py:wandb_internal():150] Internal process interrupted.
387
+ 2021-07-15 13:46:16,587 INFO SenderThread:690242 [sender.py:finish():945] shutting down sender
388
+ 2021-07-15 13:46:16,587 INFO SenderThread:690242 [dir_watcher.py:finish():282] shutting down directory watcher
389
+ 2021-07-15 13:46:16,588 INFO WriterThread:690242 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/run-37l0zsn1.wandb
390
+ 2021-07-15 13:46:16,589 INFO HandlerThread:690242 [handler.py:finish():638] shutting down handler
391
+ 2021-07-15 13:46:16,860 INFO MainThread:690242 [internal.py:handle_exit():78] Internal process exited
wandb/run-20210715_132956-37l0zsn1/logs/debug.log ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-07-15 13:29:56,223 INFO MainThread:688983 [wandb_setup.py:_flush():69] setting env: {}
2
+ 2021-07-15 13:29:56,223 INFO MainThread:688983 [wandb_setup.py:_flush():69] setting login settings: {}
3
+ 2021-07-15 13:29:56,223 INFO MainThread:688983 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/logs/debug.log
4
+ 2021-07-15 13:29:56,223 INFO MainThread:688983 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210715_132956-37l0zsn1/logs/debug-internal.log
5
+ 2021-07-15 13:29:56,223 INFO MainThread:688983 [wandb_init.py:init():370] calling init triggers
6
+ 2021-07-15 13:29:56,223 INFO MainThread:688983 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
7
+ config: {}
8
+ 2021-07-15 13:29:56,224 INFO MainThread:688983 [wandb_init.py:init():419] starting backend
9
+ 2021-07-15 13:29:56,224 INFO MainThread:688983 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
10
+ 2021-07-15 13:29:56,272 INFO MainThread:688983 [backend.py:ensure_launched():135] starting backend process...
11
+ 2021-07-15 13:29:56,319 INFO MainThread:688983 [backend.py:ensure_launched():139] started backend process with pid: 690242
12
+ 2021-07-15 13:29:56,321 INFO MainThread:688983 [wandb_init.py:init():424] backend started and connected
13
+ 2021-07-15 13:29:56,324 INFO MainThread:688983 [wandb_init.py:init():472] updated telemetry
14
+ 2021-07-15 13:29:56,325 INFO MainThread:688983 [wandb_init.py:init():491] communicating current version
15
+ 2021-07-15 13:29:56,958 INFO MainThread:688983 [wandb_init.py:init():496] got version response
16
+ 2021-07-15 13:29:56,959 INFO MainThread:688983 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
17
+ 2021-07-15 13:29:57,133 INFO MainThread:688983 [wandb_init.py:init():529] starting run threads in backend
18
+ 2021-07-15 13:29:58,265 INFO MainThread:688983 [wandb_run.py:_console_start():1623] atexit reg
19
+ 2021-07-15 13:29:58,266 INFO MainThread:688983 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
20
+ 2021-07-15 13:29:58,266 INFO MainThread:688983 [wandb_run.py:_redirect():1502] Redirecting console.
21
+ 2021-07-15 13:29:58,268 INFO MainThread:688983 [wandb_run.py:_redirect():1558] Redirects installed.
22
+ 2021-07-15 13:29:58,268 INFO MainThread:688983 [wandb_init.py:init():554] run started, returning control to user process
23
+ 2021-07-15 13:29:58,274 INFO MainThread:688983 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 3e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 2.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul15_13-29-47_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 50, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 10000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 2000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': './', 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
24
+ 2021-07-15 13:29:58,276 INFO MainThread:688983 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
25
+ 2021-07-15 13:29:58,278 INFO MainThread:688983 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 5000}
26
+ 2021-07-15 13:46:15,841 INFO MainThread:688983 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 255
27
+ 2021-07-15 13:46:15,841 INFO MainThread:688983 [wandb_run.py:_restore():1565] restore
wandb/run-20210715_132956-37l0zsn1/run-37l0zsn1.wandb ADDED
Binary file (64.3 kB). View file
 
wandb/run-20210715_135329-1c6qs9ww/files/config.yaml ADDED
@@ -0,0 +1,301 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ wandb_version: 1
2
+
3
+ _wandb:
4
+ desc: null
5
+ value:
6
+ cli_version: 0.10.33
7
+ framework: huggingface
8
+ huggingface_version: 4.9.0.dev0
9
+ is_jupyter_run: false
10
+ is_kaggle_kernel: false
11
+ python_version: 3.8.10
12
+ t:
13
+ 1:
14
+ - 3
15
+ - 11
16
+ 4: 3.8.10
17
+ 5: 0.10.33
18
+ 6: 4.9.0.dev0
19
+ 8:
20
+ - 5
21
+ adafactor:
22
+ desc: null
23
+ value: false
24
+ adam_beta1:
25
+ desc: null
26
+ value: 0.9
27
+ adam_beta2:
28
+ desc: null
29
+ value: 0.98
30
+ adam_epsilon:
31
+ desc: null
32
+ value: 1.0e-08
33
+ cache_dir:
34
+ desc: null
35
+ value: null
36
+ config_name:
37
+ desc: null
38
+ value: ./
39
+ dataloader_drop_last:
40
+ desc: null
41
+ value: false
42
+ dataloader_num_workers:
43
+ desc: null
44
+ value: 0
45
+ dataloader_pin_memory:
46
+ desc: null
47
+ value: true
48
+ dataset_config_name:
49
+ desc: null
50
+ value: null
51
+ dataset_name:
52
+ desc: null
53
+ value: null
54
+ ddp_find_unused_parameters:
55
+ desc: null
56
+ value: null
57
+ debug:
58
+ desc: null
59
+ value: []
60
+ deepspeed:
61
+ desc: null
62
+ value: null
63
+ disable_tqdm:
64
+ desc: null
65
+ value: false
66
+ do_eval:
67
+ desc: null
68
+ value: false
69
+ do_predict:
70
+ desc: null
71
+ value: false
72
+ do_train:
73
+ desc: null
74
+ value: false
75
+ dtype:
76
+ desc: null
77
+ value: float32
78
+ eval_accumulation_steps:
79
+ desc: null
80
+ value: null
81
+ eval_steps:
82
+ desc: null
83
+ value: 2000
84
+ evaluation_strategy:
85
+ desc: null
86
+ value: IntervalStrategy.NO
87
+ fp16:
88
+ desc: null
89
+ value: false
90
+ fp16_backend:
91
+ desc: null
92
+ value: auto
93
+ fp16_full_eval:
94
+ desc: null
95
+ value: false
96
+ fp16_opt_level:
97
+ desc: null
98
+ value: O1
99
+ gradient_accumulation_steps:
100
+ desc: null
101
+ value: 1
102
+ greater_is_better:
103
+ desc: null
104
+ value: null
105
+ group_by_length:
106
+ desc: null
107
+ value: false
108
+ ignore_data_skip:
109
+ desc: null
110
+ value: false
111
+ label_names:
112
+ desc: null
113
+ value: null
114
+ label_smoothing_factor:
115
+ desc: null
116
+ value: 0.0
117
+ learning_rate:
118
+ desc: null
119
+ value: 3.0e-05
120
+ length_column_name:
121
+ desc: null
122
+ value: length
123
+ line_by_line:
124
+ desc: null
125
+ value: false
126
+ load_best_model_at_end:
127
+ desc: null
128
+ value: false
129
+ local_rank:
130
+ desc: null
131
+ value: -1
132
+ log_level:
133
+ desc: null
134
+ value: -1
135
+ log_level_replica:
136
+ desc: null
137
+ value: -1
138
+ log_on_each_node:
139
+ desc: null
140
+ value: true
141
+ logging_dir:
142
+ desc: null
143
+ value: ./runs/Jul15_13-53-21_t1v-n-f5c06ea1-w-0
144
+ logging_first_step:
145
+ desc: null
146
+ value: false
147
+ logging_steps:
148
+ desc: null
149
+ value: 50
150
+ logging_strategy:
151
+ desc: null
152
+ value: IntervalStrategy.STEPS
153
+ lr_scheduler_type:
154
+ desc: null
155
+ value: SchedulerType.LINEAR
156
+ max_eval_samples:
157
+ desc: null
158
+ value: 5000
159
+ max_grad_norm:
160
+ desc: null
161
+ value: 1.0
162
+ max_seq_length:
163
+ desc: null
164
+ value: 4096
165
+ max_steps:
166
+ desc: null
167
+ value: -1
168
+ metric_for_best_model:
169
+ desc: null
170
+ value: null
171
+ mlm_probability:
172
+ desc: null
173
+ value: 0.15
174
+ model_name_or_path:
175
+ desc: null
176
+ value: null
177
+ model_type:
178
+ desc: null
179
+ value: big_bird
180
+ mp_parameters:
181
+ desc: null
182
+ value: ''
183
+ no_cuda:
184
+ desc: null
185
+ value: false
186
+ num_train_epochs:
187
+ desc: null
188
+ value: 2.0
189
+ output_dir:
190
+ desc: null
191
+ value: ./
192
+ overwrite_cache:
193
+ desc: null
194
+ value: false
195
+ overwrite_output_dir:
196
+ desc: null
197
+ value: true
198
+ pad_to_max_length:
199
+ desc: null
200
+ value: false
201
+ past_index:
202
+ desc: null
203
+ value: -1
204
+ per_device_eval_batch_size:
205
+ desc: null
206
+ value: 1
207
+ per_device_train_batch_size:
208
+ desc: null
209
+ value: 1
210
+ per_gpu_eval_batch_size:
211
+ desc: null
212
+ value: null
213
+ per_gpu_train_batch_size:
214
+ desc: null
215
+ value: null
216
+ prediction_loss_only:
217
+ desc: null
218
+ value: false
219
+ preprocessing_num_workers:
220
+ desc: null
221
+ value: 96
222
+ push_to_hub:
223
+ desc: null
224
+ value: true
225
+ push_to_hub_model_id:
226
+ desc: null
227
+ value: ''
228
+ push_to_hub_organization:
229
+ desc: null
230
+ value: null
231
+ push_to_hub_token:
232
+ desc: null
233
+ value: null
234
+ remove_unused_columns:
235
+ desc: null
236
+ value: true
237
+ report_to:
238
+ desc: null
239
+ value:
240
+ - tensorboard
241
+ - wandb
242
+ resume_from_checkpoint:
243
+ desc: null
244
+ value: ./
245
+ run_name:
246
+ desc: null
247
+ value: ./
248
+ save_on_each_node:
249
+ desc: null
250
+ value: false
251
+ save_steps:
252
+ desc: null
253
+ value: 10000
254
+ save_strategy:
255
+ desc: null
256
+ value: IntervalStrategy.STEPS
257
+ save_total_limit:
258
+ desc: null
259
+ value: 5
260
+ seed:
261
+ desc: null
262
+ value: 42
263
+ sharded_ddp:
264
+ desc: null
265
+ value: []
266
+ skip_memory_metrics:
267
+ desc: null
268
+ value: true
269
+ tokenizer_name:
270
+ desc: null
271
+ value: ./
272
+ tpu_metrics_debug:
273
+ desc: null
274
+ value: false
275
+ tpu_num_cores:
276
+ desc: null
277
+ value: null
278
+ train_ref_file:
279
+ desc: null
280
+ value: null
281
+ use_fast_tokenizer:
282
+ desc: null
283
+ value: true
284
+ use_legacy_prediction_loop:
285
+ desc: null
286
+ value: false
287
+ validation_ref_file:
288
+ desc: null
289
+ value: null
290
+ validation_split_percentage:
291
+ desc: null
292
+ value: 5
293
+ warmup_ratio:
294
+ desc: null
295
+ value: 0.0
296
+ warmup_steps:
297
+ desc: null
298
+ value: 10000
299
+ weight_decay:
300
+ desc: null
301
+ value: 0.0095
wandb/run-20210715_135329-1c6qs9ww/files/output.log ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [13:53:43] - INFO - absl - Restoring checkpoint from ./checkpoint_90000
2
+ tcmalloc: large alloc 1530273792 bytes == 0x9ad7a000 @ 0x7f3feb16d680 0x7f3feb18e824 0x5b9a14 0x50b2ae 0x50cb1b 0x5a6f17 0x5f3010 0x56fd36 0x568d9a 0x5f5b33 0x56aadf 0x568d9a 0x68cdc7 0x67e161 0x67e1df 0x67e281 0x67e627 0x6b6e62 0x6b71ed 0x7f3feaf820b3 0x5f96de
3
+ /home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
4
+ warnings.warn(
5
+ /home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
6
+ warnings.warn(
7
+ Epoch ... (1/2): 0%| | 0/2 [00:00<?, ?it/s]
8
+
9
+
10
+
11
+
12
+
13
+ Training...: 26%|████████████████████████████████████▊ | 90051/352766 [04:22<27:51:51, 2.62it/s]
14
+
15
+
16
+
17
+
18
+ Training...: 26%|████████████████████████████████████▊ | 90101/352766 [04:42<27:17:08, 2.67it/s]
19
+
20
+
21
+
22
+
23
+ Training...: 26%|████████████████████████████████████▊ | 90151/352766 [05:02<27:16:16, 2.67it/s]
24
+
25
+
26
+
27
+
28
+ Training...: 26%|████████████████████████████████████▊ | 90201/352766 [05:22<27:14:35, 2.68it/s]
29
+
30
+
31
+
32
+
33
+ Training...: 26%|████████████████████████████████████▊ | 90251/352766 [05:42<27:01:30, 2.70it/s]
34
+
35
+
36
+
37
+
38
+
39
+ Training...: 26%|████████████████████████████████████▌ | 90302/352766 [06:15<292:20:48, 4.01s/it]
40
+
41
+
42
+
43
+
44
+ Training...: 26%|████████████████████████████████████▉ | 90351/352766 [06:22<26:58:26, 2.70it/s]
45
+
46
+
47
+
48
+
49
+ Training...: 26%|████████████████████████████████████▉ | 90401/352766 [06:42<27:14:02, 2.68it/s]
50
+
51
+
52
+
53
+
54
+ Training...: 26%|████████████████████████████████████▉ | 90451/352766 [07:02<27:03:56, 2.69it/s]
55
+
56
+
57
+
58
+
59
+ Training...: 26%|████████████████████████████████████▉ | 90501/352766 [07:22<27:12:39, 2.68it/s]
60
+
61
+
62
+
63
+
64
+
65
+ Training...: 26%|████████████████████████████████████▋ | 90552/352766 [07:55<291:55:56, 4.01s/it]
66
+
67
+
68
+
69
+
70
+
71
+ Training...: 26%|████████████████████████████████████▋ | 90604/352766 [08:16<163:26:36, 2.24s/it]
72
+
73
+
74
+
75
+
76
+
77
+ Training...: 26%|████████████████████████████████████▋ | 90654/352766 [08:36<163:24:41, 2.24s/it]
78
+
79
+
80
+
81
+
82
+
83
+ Training...: 26%|████████████████████████████████████▊ | 90706/352766 [08:56<101:36:18, 1.40s/it]
84
+
85
+
86
+
87
+
88
+
89
+ Training...: 26%|████████████████���███████████████████▊ | 90752/352766 [09:16<291:41:06, 4.01s/it]
90
+
91
+
92
+
93
+
94
+
95
+ Training...: 26%|████████████████████████████████████▊ | 90802/352766 [09:36<291:45:30, 4.01s/it]
96
+
97
+
98
+
99
+
100
+
101
+ Training...: 26%|████████████████████████████████████▊ | 90852/352766 [09:56<291:42:28, 4.01s/it]
102
+
103
+
104
+
105
+
106
+
107
+ Training...: 26%|████████████████████████████████████▊ | 90906/352766 [10:16<101:52:03, 1.40s/it]
108
+
109
+
110
+
111
+
112
+
113
+ Training...: 26%|████████████████████████████████████▊ | 90956/352766 [10:36<101:26:47, 1.39s/it]
114
+
115
+
116
+
117
+
118
+
119
+ Training...: 26%|█████████████████████████████████████▏ | 91008/352766 [10:56<66:45:48, 1.09it/s]
wandb/run-20210715_135329-1c6qs9ww/files/requirements.txt ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ absl-py==0.13.0
2
+ aiohttp==3.7.4.post0
3
+ astunparse==1.6.3
4
+ async-timeout==3.0.1
5
+ attrs==21.2.0
6
+ cachetools==4.2.2
7
+ certifi==2021.5.30
8
+ chardet==4.0.0
9
+ charset-normalizer==2.0.1
10
+ chex==0.0.8
11
+ click==8.0.1
12
+ configparser==5.0.2
13
+ cycler==0.10.0
14
+ datasets==1.9.1.dev0
15
+ dill==0.3.4
16
+ dm-tree==0.1.6
17
+ docker-pycreds==0.4.0
18
+ filelock==3.0.12
19
+ flatbuffers==1.12
20
+ flax==0.3.4
21
+ fsspec==2021.7.0
22
+ gast==0.4.0
23
+ gitdb==4.0.7
24
+ gitpython==3.1.18
25
+ google-auth-oauthlib==0.4.4
26
+ google-auth==1.32.1
27
+ google-pasta==0.2.0
28
+ grpcio==1.34.1
29
+ h5py==3.1.0
30
+ huggingface-hub==0.0.12
31
+ idna==3.2
32
+ install==1.3.4
33
+ jax==0.2.17
34
+ jaxlib==0.1.68
35
+ joblib==1.0.1
36
+ keras-nightly==2.5.0.dev2021032900
37
+ keras-preprocessing==1.1.2
38
+ kiwisolver==1.3.1
39
+ libtpu-nightly==0.1.dev20210615
40
+ markdown==3.3.4
41
+ matplotlib==3.4.2
42
+ msgpack==1.0.2
43
+ multidict==5.1.0
44
+ multiprocess==0.70.12.2
45
+ numpy==1.19.5
46
+ oauthlib==3.1.1
47
+ opt-einsum==3.3.0
48
+ optax==0.0.9
49
+ packaging==21.0
50
+ pandas==1.3.0
51
+ pathtools==0.1.2
52
+ pillow==8.3.1
53
+ pip==20.0.2
54
+ pkg-resources==0.0.0
55
+ promise==2.3
56
+ protobuf==3.17.3
57
+ psutil==5.8.0
58
+ pyarrow==4.0.1
59
+ pyasn1-modules==0.2.8
60
+ pyasn1==0.4.8
61
+ pyparsing==2.4.7
62
+ python-dateutil==2.8.1
63
+ pytz==2021.1
64
+ pyyaml==5.4.1
65
+ regex==2021.7.6
66
+ requests-oauthlib==1.3.0
67
+ requests==2.26.0
68
+ rsa==4.7.2
69
+ sacremoses==0.0.45
70
+ scipy==1.7.0
71
+ sentry-sdk==1.3.0
72
+ setuptools==44.0.0
73
+ shortuuid==1.0.1
74
+ six==1.15.0
75
+ smmap==4.0.0
76
+ subprocess32==3.5.4
77
+ tensorboard-data-server==0.6.1
78
+ tensorboard-plugin-wit==1.8.0
79
+ tensorboard==2.5.0
80
+ tensorflow-estimator==2.5.0
81
+ tensorflow==2.5.0
82
+ termcolor==1.1.0
83
+ tokenizers==0.10.3
84
+ toolz==0.11.1
85
+ tqdm==4.61.2
86
+ transformers==4.9.0.dev0
87
+ typing-extensions==3.7.4.3
88
+ urllib3==1.26.6
89
+ wandb==0.10.33
90
+ werkzeug==2.0.1
91
+ wheel==0.36.2
92
+ wrapt==1.12.1
93
+ xxhash==2.0.2
94
+ yarl==1.6.3
wandb/run-20210715_135329-1c6qs9ww/files/wandb-metadata.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
3
+ "python": "3.8.10",
4
+ "heartbeatAt": "2021-07-15T13:53:31.167145",
5
+ "startedAt": "2021-07-15T13:53:29.138276",
6
+ "docker": null,
7
+ "cpu_count": 96,
8
+ "cuda": null,
9
+ "args": [
10
+ "--push_to_hub",
11
+ "--output_dir=./",
12
+ "--model_type=big_bird",
13
+ "--config_name=./",
14
+ "--tokenizer_name=./",
15
+ "--max_seq_length=4096",
16
+ "--weight_decay=0.0095",
17
+ "--warmup_steps=10000",
18
+ "--overwrite_output_dir",
19
+ "--adam_beta1=0.9",
20
+ "--adam_beta2=0.98",
21
+ "--logging_steps=50",
22
+ "--eval_steps=2000",
23
+ "--num_train_epochs=2",
24
+ "--preprocessing_num_workers=96",
25
+ "--save_steps=10000",
26
+ "--learning_rate=3e-5",
27
+ "--per_device_train_batch_size=1",
28
+ "--per_device_eval_batch_size=1",
29
+ "--save_total_limit=5",
30
+ "--max_eval_samples=5000",
31
+ "--resume_from_checkpoint=./"
32
+ ],
33
+ "state": "running",
34
+ "program": "./run_mlm_flax_no_accum.py",
35
+ "codePath": "run_mlm_flax_no_accum.py",
36
+ "git": {
37
+ "remote": "https://huggingface.co/flax-community/pino-roberta-base",
38
+ "commit": "b6814517994e86d862ddec99dda42b9dbea5b8b0"
39
+ },
40
+ "email": null,
41
+ "root": "/home/dat/pino-roberta-base",
42
+ "host": "t1v-n-f5c06ea1-w-0",
43
+ "username": "dat",
44
+ "executable": "/home/dat/pino/bin/python"
45
+ }
wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"training_step": 1000, "learning_rate": 2.650622627697885e-05, "train_loss": 2.3977699279785156, "_runtime": 1003, "_timestamp": 1626358212, "_step": 19}
wandb/run-20210715_135329-1c6qs9ww/logs/debug-internal.log ADDED
@@ -0,0 +1,399 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-07-15 13:53:29,847 INFO MainThread:693510 [internal.py:wandb_internal():88] W&B internal server running at pid: 693510, started at: 2021-07-15 13:53:29.847625
2
+ 2021-07-15 13:53:29,850 INFO WriterThread:693510 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/run-1c6qs9ww.wandb
3
+ 2021-07-15 13:53:29,850 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: check_version
4
+ 2021-07-15 13:53:29,851 DEBUG SenderThread:693510 [sender.py:send():179] send: header
5
+ 2021-07-15 13:53:29,851 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: check_version
6
+ 2021-07-15 13:53:29,891 DEBUG SenderThread:693510 [sender.py:send():179] send: run
7
+ 2021-07-15 13:53:30,065 INFO SenderThread:693510 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files
8
+ 2021-07-15 13:53:30,065 INFO SenderThread:693510 [sender.py:_start_run_threads():716] run started: 1c6qs9ww with start time 1626357209
9
+ 2021-07-15 13:53:30,065 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
10
+ 2021-07-15 13:53:30,066 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
11
+ 2021-07-15 13:53:30,066 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: run_start
12
+ 2021-07-15 13:53:31,068 INFO Thread-8 :693510 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
13
+ 2021-07-15 13:53:31,166 DEBUG HandlerThread:693510 [meta.py:__init__():39] meta init
14
+ 2021-07-15 13:53:31,167 DEBUG HandlerThread:693510 [meta.py:__init__():53] meta init done
15
+ 2021-07-15 13:53:31,167 DEBUG HandlerThread:693510 [meta.py:probe():210] probe
16
+ 2021-07-15 13:53:31,168 DEBUG HandlerThread:693510 [meta.py:_setup_git():200] setup git
17
+ 2021-07-15 13:53:31,198 DEBUG HandlerThread:693510 [meta.py:_setup_git():207] setup git done
18
+ 2021-07-15 13:53:31,198 DEBUG HandlerThread:693510 [meta.py:_save_pip():57] save pip
19
+ 2021-07-15 13:53:31,199 DEBUG HandlerThread:693510 [meta.py:_save_pip():71] save pip done
20
+ 2021-07-15 13:53:31,199 DEBUG HandlerThread:693510 [meta.py:probe():252] probe done
21
+ 2021-07-15 13:53:31,202 DEBUG SenderThread:693510 [sender.py:send():179] send: files
22
+ 2021-07-15 13:53:31,202 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now
23
+ 2021-07-15 13:53:31,210 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
24
+ 2021-07-15 13:53:31,211 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
25
+ 2021-07-15 13:53:31,339 DEBUG SenderThread:693510 [sender.py:send():179] send: config
26
+ 2021-07-15 13:53:31,340 DEBUG SenderThread:693510 [sender.py:send():179] send: config
27
+ 2021-07-15 13:53:31,340 DEBUG SenderThread:693510 [sender.py:send():179] send: config
28
+ 2021-07-15 13:53:31,710 INFO Thread-11 :693510 [upload_job.py:push():137] Uploaded file /tmp/tmpbh2gacmiwandb/3ahj6nuh-wandb-metadata.json
29
+ 2021-07-15 13:53:32,067 INFO Thread-8 :693510 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/requirements.txt
30
+ 2021-07-15 13:53:32,067 INFO Thread-8 :693510 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-metadata.json
31
+ 2021-07-15 13:53:32,068 INFO Thread-8 :693510 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
32
+ 2021-07-15 13:53:46,073 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
33
+ 2021-07-15 13:53:46,399 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
34
+ 2021-07-15 13:53:46,399 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
35
+ 2021-07-15 13:53:49,074 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
36
+ 2021-07-15 13:53:59,251 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
37
+ 2021-07-15 13:54:01,078 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/config.yaml
38
+ 2021-07-15 13:54:01,586 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
39
+ 2021-07-15 13:54:01,587 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
40
+ 2021-07-15 13:54:16,718 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
41
+ 2021-07-15 13:54:16,719 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
42
+ 2021-07-15 13:54:29,328 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
43
+ 2021-07-15 13:54:31,849 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
44
+ 2021-07-15 13:54:31,850 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
45
+ 2021-07-15 13:54:46,986 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
46
+ 2021-07-15 13:54:46,986 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
47
+ 2021-07-15 13:54:59,405 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
48
+ 2021-07-15 13:55:02,118 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
49
+ 2021-07-15 13:55:02,118 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
50
+ 2021-07-15 13:55:17,248 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
51
+ 2021-07-15 13:55:17,249 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
52
+ 2021-07-15 13:55:29,470 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
53
+ 2021-07-15 13:55:32,381 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
54
+ 2021-07-15 13:55:32,381 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
55
+ 2021-07-15 13:55:47,513 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
56
+ 2021-07-15 13:55:47,513 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
57
+ 2021-07-15 13:55:59,545 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
58
+ 2021-07-15 13:56:02,648 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
59
+ 2021-07-15 13:56:02,648 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
60
+ 2021-07-15 13:56:17,780 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
61
+ 2021-07-15 13:56:17,780 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
62
+ 2021-07-15 13:56:29,613 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
63
+ 2021-07-15 13:56:32,912 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
64
+ 2021-07-15 13:56:32,912 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
65
+ 2021-07-15 13:56:48,044 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
66
+ 2021-07-15 13:56:48,044 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
67
+ 2021-07-15 13:56:59,677 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
68
+ 2021-07-15 13:57:03,179 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
69
+ 2021-07-15 13:57:03,179 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
70
+ 2021-07-15 13:57:18,310 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
71
+ 2021-07-15 13:57:18,311 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
72
+ 2021-07-15 13:57:29,744 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
73
+ 2021-07-15 13:57:33,446 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
74
+ 2021-07-15 13:57:33,446 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
75
+ 2021-07-15 13:57:48,577 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
76
+ 2021-07-15 13:57:48,577 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
77
+ 2021-07-15 13:57:59,820 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
78
+ 2021-07-15 13:58:03,712 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
79
+ 2021-07-15 13:58:03,712 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
80
+ 2021-07-15 13:58:18,841 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
81
+ 2021-07-15 13:58:18,841 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
82
+ 2021-07-15 13:58:29,899 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
83
+ 2021-07-15 13:58:33,972 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
84
+ 2021-07-15 13:58:33,972 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
85
+ 2021-07-15 13:58:49,102 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
86
+ 2021-07-15 13:58:49,102 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
87
+ 2021-07-15 13:58:59,963 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
88
+ 2021-07-15 13:59:04,236 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
89
+ 2021-07-15 13:59:04,236 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
90
+ 2021-07-15 13:59:17,202 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
91
+ 2021-07-15 13:59:19,407 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
92
+ 2021-07-15 13:59:19,408 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
93
+ 2021-07-15 13:59:30,030 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
94
+ 2021-07-15 13:59:34,559 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
95
+ 2021-07-15 13:59:34,559 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
96
+ 2021-07-15 13:59:49,711 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
97
+ 2021-07-15 13:59:49,712 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
98
+ 2021-07-15 14:00:00,102 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
99
+ 2021-07-15 14:00:04,842 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
100
+ 2021-07-15 14:00:04,842 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
101
+ 2021-07-15 14:00:19,976 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
102
+ 2021-07-15 14:00:19,976 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
103
+ 2021-07-15 14:00:30,175 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
104
+ 2021-07-15 14:00:35,107 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
105
+ 2021-07-15 14:00:35,107 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
106
+ 2021-07-15 14:00:50,245 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
107
+ 2021-07-15 14:00:50,246 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
108
+ 2021-07-15 14:01:00,254 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
109
+ 2021-07-15 14:01:05,377 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
110
+ 2021-07-15 14:01:05,378 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
111
+ 2021-07-15 14:01:19,253 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
112
+ 2021-07-15 14:01:20,513 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
113
+ 2021-07-15 14:01:20,513 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
114
+ 2021-07-15 14:01:30,331 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
115
+ 2021-07-15 14:01:35,686 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
116
+ 2021-07-15 14:01:35,686 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
117
+ 2021-07-15 14:01:50,838 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
118
+ 2021-07-15 14:01:50,838 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
119
+ 2021-07-15 14:02:00,415 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
120
+ 2021-07-15 14:02:05,977 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
121
+ 2021-07-15 14:02:05,977 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
122
+ 2021-07-15 14:02:21,110 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
123
+ 2021-07-15 14:02:21,110 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
124
+ 2021-07-15 14:02:30,499 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
125
+ 2021-07-15 14:02:36,243 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
126
+ 2021-07-15 14:02:36,243 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
127
+ 2021-07-15 14:02:51,378 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
128
+ 2021-07-15 14:02:51,378 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
129
+ 2021-07-15 14:03:00,580 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
130
+ 2021-07-15 14:03:06,510 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
131
+ 2021-07-15 14:03:06,511 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
132
+ 2021-07-15 14:03:21,645 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
133
+ 2021-07-15 14:03:21,645 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
134
+ 2021-07-15 14:03:30,688 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
135
+ 2021-07-15 14:03:34,306 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
136
+ 2021-07-15 14:03:36,307 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
137
+ 2021-07-15 14:03:36,775 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
138
+ 2021-07-15 14:03:36,775 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
139
+ 2021-07-15 14:03:38,307 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
140
+ 2021-07-15 14:03:40,308 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
141
+ 2021-07-15 14:03:51,638 DEBUG SenderThread:693510 [sender.py:send():179] send: history
142
+ 2021-07-15 14:03:51,638 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
143
+ 2021-07-15 14:03:51,639 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
144
+ 2021-07-15 14:03:51,934 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
145
+ 2021-07-15 14:03:51,934 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
146
+ 2021-07-15 14:03:52,313 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
147
+ 2021-07-15 14:03:53,314 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
148
+ 2021-07-15 14:03:54,314 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
149
+ 2021-07-15 14:03:56,315 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
150
+ 2021-07-15 14:03:58,316 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
151
+ 2021-07-15 14:04:00,316 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
152
+ 2021-07-15 14:04:00,764 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
153
+ 2021-07-15 14:04:07,066 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
154
+ 2021-07-15 14:04:07,066 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
155
+ 2021-07-15 14:04:11,676 DEBUG SenderThread:693510 [sender.py:send():179] send: history
156
+ 2021-07-15 14:04:11,676 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
157
+ 2021-07-15 14:04:11,677 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
158
+ 2021-07-15 14:04:12,321 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
159
+ 2021-07-15 14:04:13,321 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
160
+ 2021-07-15 14:04:14,322 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
161
+ 2021-07-15 14:04:16,323 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
162
+ 2021-07-15 14:04:18,324 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
163
+ 2021-07-15 14:04:20,325 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
164
+ 2021-07-15 14:04:22,195 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
165
+ 2021-07-15 14:04:22,196 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
166
+ 2021-07-15 14:04:30,839 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
167
+ 2021-07-15 14:04:31,712 DEBUG SenderThread:693510 [sender.py:send():179] send: history
168
+ 2021-07-15 14:04:31,712 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
169
+ 2021-07-15 14:04:31,713 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
170
+ 2021-07-15 14:04:32,330 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
171
+ 2021-07-15 14:04:33,331 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
172
+ 2021-07-15 14:04:34,331 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
173
+ 2021-07-15 14:04:36,332 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
174
+ 2021-07-15 14:04:37,327 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
175
+ 2021-07-15 14:04:37,327 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
176
+ 2021-07-15 14:04:38,333 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
177
+ 2021-07-15 14:04:40,334 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
178
+ 2021-07-15 14:04:51,749 DEBUG SenderThread:693510 [sender.py:send():179] send: history
179
+ 2021-07-15 14:04:51,749 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
180
+ 2021-07-15 14:04:51,749 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
181
+ 2021-07-15 14:04:52,339 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
182
+ 2021-07-15 14:04:52,479 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
183
+ 2021-07-15 14:04:52,479 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
184
+ 2021-07-15 14:04:53,339 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
185
+ 2021-07-15 14:04:54,340 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
186
+ 2021-07-15 14:04:56,340 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
187
+ 2021-07-15 14:04:58,341 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
188
+ 2021-07-15 14:05:00,342 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
189
+ 2021-07-15 14:05:00,907 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
190
+ 2021-07-15 14:05:07,647 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
191
+ 2021-07-15 14:05:07,648 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
192
+ 2021-07-15 14:05:11,789 DEBUG SenderThread:693510 [sender.py:send():179] send: history
193
+ 2021-07-15 14:05:11,789 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
194
+ 2021-07-15 14:05:11,789 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
195
+ 2021-07-15 14:05:12,347 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
196
+ 2021-07-15 14:05:13,348 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
197
+ 2021-07-15 14:05:14,348 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
198
+ 2021-07-15 14:05:16,349 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
199
+ 2021-07-15 14:05:18,350 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
200
+ 2021-07-15 14:05:20,351 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
201
+ 2021-07-15 14:05:22,781 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
202
+ 2021-07-15 14:05:22,782 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
203
+ 2021-07-15 14:05:30,976 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
204
+ 2021-07-15 14:05:31,824 DEBUG SenderThread:693510 [sender.py:send():179] send: history
205
+ 2021-07-15 14:05:31,824 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
206
+ 2021-07-15 14:05:31,825 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
207
+ 2021-07-15 14:05:32,355 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
208
+ 2021-07-15 14:05:32,356 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
209
+ 2021-07-15 14:05:33,356 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
210
+ 2021-07-15 14:05:34,356 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
211
+ 2021-07-15 14:05:36,357 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
212
+ 2021-07-15 14:05:37,916 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
213
+ 2021-07-15 14:05:37,916 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
214
+ 2021-07-15 14:05:38,358 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
215
+ 2021-07-15 14:05:40,359 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
216
+ 2021-07-15 14:05:51,866 DEBUG SenderThread:693510 [sender.py:send():179] send: history
217
+ 2021-07-15 14:05:51,866 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
218
+ 2021-07-15 14:05:51,866 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
219
+ 2021-07-15 14:05:52,364 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
220
+ 2021-07-15 14:05:53,068 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
221
+ 2021-07-15 14:05:53,068 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
222
+ 2021-07-15 14:05:53,364 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
223
+ 2021-07-15 14:05:54,365 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
224
+ 2021-07-15 14:05:56,366 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
225
+ 2021-07-15 14:05:58,367 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
226
+ 2021-07-15 14:06:00,368 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
227
+ 2021-07-15 14:06:01,051 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
228
+ 2021-07-15 14:06:08,203 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
229
+ 2021-07-15 14:06:08,203 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
230
+ 2021-07-15 14:06:11,904 DEBUG SenderThread:693510 [sender.py:send():179] send: history
231
+ 2021-07-15 14:06:11,904 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
232
+ 2021-07-15 14:06:11,904 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
233
+ 2021-07-15 14:06:12,372 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
234
+ 2021-07-15 14:06:13,373 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
235
+ 2021-07-15 14:06:14,373 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
236
+ 2021-07-15 14:06:16,374 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
237
+ 2021-07-15 14:06:18,375 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
238
+ 2021-07-15 14:06:20,376 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
239
+ 2021-07-15 14:06:23,336 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
240
+ 2021-07-15 14:06:23,336 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
241
+ 2021-07-15 14:06:31,124 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
242
+ 2021-07-15 14:06:31,942 DEBUG SenderThread:693510 [sender.py:send():179] send: history
243
+ 2021-07-15 14:06:31,943 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
244
+ 2021-07-15 14:06:31,943 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
245
+ 2021-07-15 14:06:32,381 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
246
+ 2021-07-15 14:06:33,382 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
247
+ 2021-07-15 14:06:34,382 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
248
+ 2021-07-15 14:06:36,383 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
249
+ 2021-07-15 14:06:38,384 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
250
+ 2021-07-15 14:06:38,467 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
251
+ 2021-07-15 14:06:38,468 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
252
+ 2021-07-15 14:06:40,385 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
253
+ 2021-07-15 14:06:51,987 DEBUG SenderThread:693510 [sender.py:send():179] send: history
254
+ 2021-07-15 14:06:51,987 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
255
+ 2021-07-15 14:06:51,988 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
256
+ 2021-07-15 14:06:52,389 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
257
+ 2021-07-15 14:06:53,390 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
258
+ 2021-07-15 14:06:53,648 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
259
+ 2021-07-15 14:06:53,648 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
260
+ 2021-07-15 14:06:54,390 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
261
+ 2021-07-15 14:06:56,391 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
262
+ 2021-07-15 14:06:58,392 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
263
+ 2021-07-15 14:07:00,393 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
264
+ 2021-07-15 14:07:01,197 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
265
+ 2021-07-15 14:07:08,814 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
266
+ 2021-07-15 14:07:08,814 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
267
+ 2021-07-15 14:07:12,024 DEBUG SenderThread:693510 [sender.py:send():179] send: history
268
+ 2021-07-15 14:07:12,025 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
269
+ 2021-07-15 14:07:12,025 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
270
+ 2021-07-15 14:07:12,397 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
271
+ 2021-07-15 14:07:12,397 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
272
+ 2021-07-15 14:07:13,398 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
273
+ 2021-07-15 14:07:14,398 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
274
+ 2021-07-15 14:07:16,399 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
275
+ 2021-07-15 14:07:18,400 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
276
+ 2021-07-15 14:07:20,400 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
277
+ 2021-07-15 14:07:23,978 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
278
+ 2021-07-15 14:07:23,978 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
279
+ 2021-07-15 14:07:31,275 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
280
+ 2021-07-15 14:07:32,060 DEBUG SenderThread:693510 [sender.py:send():179] send: history
281
+ 2021-07-15 14:07:32,060 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
282
+ 2021-07-15 14:07:32,061 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
283
+ 2021-07-15 14:07:32,406 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
284
+ 2021-07-15 14:07:32,406 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
285
+ 2021-07-15 14:07:33,406 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
286
+ 2021-07-15 14:07:34,406 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
287
+ 2021-07-15 14:07:36,407 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
288
+ 2021-07-15 14:07:38,408 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
289
+ 2021-07-15 14:07:39,111 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
290
+ 2021-07-15 14:07:39,111 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
291
+ 2021-07-15 14:07:40,409 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
292
+ 2021-07-15 14:07:52,095 DEBUG SenderThread:693510 [sender.py:send():179] send: history
293
+ 2021-07-15 14:07:52,095 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
294
+ 2021-07-15 14:07:52,095 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
295
+ 2021-07-15 14:07:52,414 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
296
+ 2021-07-15 14:07:52,414 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
297
+ 2021-07-15 14:07:54,262 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
298
+ 2021-07-15 14:07:54,262 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
299
+ 2021-07-15 14:07:54,414 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
300
+ 2021-07-15 14:07:56,415 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
301
+ 2021-07-15 14:07:58,416 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
302
+ 2021-07-15 14:08:00,417 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
303
+ 2021-07-15 14:08:01,410 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
304
+ 2021-07-15 14:08:09,395 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
305
+ 2021-07-15 14:08:09,395 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
306
+ 2021-07-15 14:08:12,138 DEBUG SenderThread:693510 [sender.py:send():179] send: history
307
+ 2021-07-15 14:08:12,138 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
308
+ 2021-07-15 14:08:12,139 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
309
+ 2021-07-15 14:08:12,422 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
310
+ 2021-07-15 14:08:13,422 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
311
+ 2021-07-15 14:08:14,423 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
312
+ 2021-07-15 14:08:15,423 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
313
+ 2021-07-15 14:08:17,424 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
314
+ 2021-07-15 14:08:19,425 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
315
+ 2021-07-15 14:08:21,426 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
316
+ 2021-07-15 14:08:24,526 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
317
+ 2021-07-15 14:08:24,527 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
318
+ 2021-07-15 14:08:31,498 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
319
+ 2021-07-15 14:08:32,177 DEBUG SenderThread:693510 [sender.py:send():179] send: history
320
+ 2021-07-15 14:08:32,177 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
321
+ 2021-07-15 14:08:32,177 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
322
+ 2021-07-15 14:08:32,431 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
323
+ 2021-07-15 14:08:33,431 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
324
+ 2021-07-15 14:08:34,432 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
325
+ 2021-07-15 14:08:35,432 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
326
+ 2021-07-15 14:08:37,433 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
327
+ 2021-07-15 14:08:39,434 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
328
+ 2021-07-15 14:08:39,657 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
329
+ 2021-07-15 14:08:39,657 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
330
+ 2021-07-15 14:08:41,435 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
331
+ 2021-07-15 14:08:52,218 DEBUG SenderThread:693510 [sender.py:send():179] send: history
332
+ 2021-07-15 14:08:52,218 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
333
+ 2021-07-15 14:08:52,218 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
334
+ 2021-07-15 14:08:52,439 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
335
+ 2021-07-15 14:08:53,440 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
336
+ 2021-07-15 14:08:54,440 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
337
+ 2021-07-15 14:08:54,790 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
338
+ 2021-07-15 14:08:54,790 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
339
+ 2021-07-15 14:08:55,441 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
340
+ 2021-07-15 14:08:57,442 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
341
+ 2021-07-15 14:08:59,443 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
342
+ 2021-07-15 14:09:01,444 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
343
+ 2021-07-15 14:09:01,574 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
344
+ 2021-07-15 14:09:09,928 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
345
+ 2021-07-15 14:09:09,928 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
346
+ 2021-07-15 14:09:12,253 DEBUG SenderThread:693510 [sender.py:send():179] send: history
347
+ 2021-07-15 14:09:12,254 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
348
+ 2021-07-15 14:09:12,254 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
349
+ 2021-07-15 14:09:12,448 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
350
+ 2021-07-15 14:09:13,449 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
351
+ 2021-07-15 14:09:14,449 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
352
+ 2021-07-15 14:09:15,450 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
353
+ 2021-07-15 14:09:17,451 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
354
+ 2021-07-15 14:09:19,452 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
355
+ 2021-07-15 14:09:21,453 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
356
+ 2021-07-15 14:09:25,058 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
357
+ 2021-07-15 14:09:25,058 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
358
+ 2021-07-15 14:09:31,644 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
359
+ 2021-07-15 14:09:32,292 DEBUG SenderThread:693510 [sender.py:send():179] send: history
360
+ 2021-07-15 14:09:32,292 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
361
+ 2021-07-15 14:09:32,293 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
362
+ 2021-07-15 14:09:32,458 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
363
+ 2021-07-15 14:09:33,458 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
364
+ 2021-07-15 14:09:34,459 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
365
+ 2021-07-15 14:09:35,459 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
366
+ 2021-07-15 14:09:37,460 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
367
+ 2021-07-15 14:09:39,461 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
368
+ 2021-07-15 14:09:40,191 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
369
+ 2021-07-15 14:09:40,192 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
370
+ 2021-07-15 14:09:41,462 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
371
+ 2021-07-15 14:09:52,334 DEBUG SenderThread:693510 [sender.py:send():179] send: history
372
+ 2021-07-15 14:09:52,334 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
373
+ 2021-07-15 14:09:52,335 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
374
+ 2021-07-15 14:09:52,466 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
375
+ 2021-07-15 14:09:53,466 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
376
+ 2021-07-15 14:09:54,467 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
377
+ 2021-07-15 14:09:55,325 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
378
+ 2021-07-15 14:09:55,325 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
379
+ 2021-07-15 14:09:55,467 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
380
+ 2021-07-15 14:09:57,468 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
381
+ 2021-07-15 14:09:59,469 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
382
+ 2021-07-15 14:10:01,469 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
383
+ 2021-07-15 14:10:01,719 DEBUG SenderThread:693510 [sender.py:send():179] send: stats
384
+ 2021-07-15 14:10:10,458 DEBUG HandlerThread:693510 [handler.py:handle_request():124] handle_request: stop_status
385
+ 2021-07-15 14:10:10,458 DEBUG SenderThread:693510 [sender.py:send_request():193] send_request: stop_status
386
+ 2021-07-15 14:10:12,382 DEBUG SenderThread:693510 [sender.py:send():179] send: history
387
+ 2021-07-15 14:10:12,383 DEBUG SenderThread:693510 [sender.py:send():179] send: summary
388
+ 2021-07-15 14:10:12,383 INFO SenderThread:693510 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
389
+ 2021-07-15 14:10:12,474 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/wandb-summary.json
390
+ 2021-07-15 14:10:13,474 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
391
+ 2021-07-15 14:10:14,354 WARNING MainThread:693510 [internal.py:wandb_internal():147] Internal process interrupt: 1
392
+ 2021-07-15 14:10:14,475 INFO Thread-8 :693510 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/files/output.log
393
+ 2021-07-15 14:10:14,571 WARNING MainThread:693510 [internal.py:wandb_internal():147] Internal process interrupt: 2
394
+ 2021-07-15 14:10:14,571 ERROR MainThread:693510 [internal.py:wandb_internal():150] Internal process interrupted.
395
+ 2021-07-15 14:10:14,911 INFO SenderThread:693510 [sender.py:finish():945] shutting down sender
396
+ 2021-07-15 14:10:14,912 INFO SenderThread:693510 [dir_watcher.py:finish():282] shutting down directory watcher
397
+ 2021-07-15 14:10:14,912 INFO WriterThread:693510 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/run-1c6qs9ww.wandb
398
+ 2021-07-15 14:10:14,912 INFO HandlerThread:693510 [handler.py:finish():638] shutting down handler
399
+ 2021-07-15 14:10:15,102 INFO MainThread:693510 [internal.py:handle_exit():78] Internal process exited
wandb/run-20210715_135329-1c6qs9ww/logs/debug.log ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-07-15 13:53:29,139 INFO MainThread:692253 [wandb_setup.py:_flush():69] setting env: {}
2
+ 2021-07-15 13:53:29,140 INFO MainThread:692253 [wandb_setup.py:_flush():69] setting login settings: {}
3
+ 2021-07-15 13:53:29,140 INFO MainThread:692253 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/logs/debug.log
4
+ 2021-07-15 13:53:29,140 INFO MainThread:692253 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210715_135329-1c6qs9ww/logs/debug-internal.log
5
+ 2021-07-15 13:53:29,140 INFO MainThread:692253 [wandb_init.py:init():370] calling init triggers
6
+ 2021-07-15 13:53:29,140 INFO MainThread:692253 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
7
+ config: {}
8
+ 2021-07-15 13:53:29,140 INFO MainThread:692253 [wandb_init.py:init():419] starting backend
9
+ 2021-07-15 13:53:29,140 INFO MainThread:692253 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
10
+ 2021-07-15 13:53:29,188 INFO MainThread:692253 [backend.py:ensure_launched():135] starting backend process...
11
+ 2021-07-15 13:53:29,256 INFO MainThread:692253 [backend.py:ensure_launched():139] started backend process with pid: 693510
12
+ 2021-07-15 13:53:29,257 INFO MainThread:692253 [wandb_init.py:init():424] backend started and connected
13
+ 2021-07-15 13:53:29,260 INFO MainThread:692253 [wandb_init.py:init():472] updated telemetry
14
+ 2021-07-15 13:53:29,261 INFO MainThread:692253 [wandb_init.py:init():491] communicating current version
15
+ 2021-07-15 13:53:29,890 INFO MainThread:692253 [wandb_init.py:init():496] got version response
16
+ 2021-07-15 13:53:29,890 INFO MainThread:692253 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
17
+ 2021-07-15 13:53:30,065 INFO MainThread:692253 [wandb_init.py:init():529] starting run threads in backend
18
+ 2021-07-15 13:53:31,205 INFO MainThread:692253 [wandb_run.py:_console_start():1623] atexit reg
19
+ 2021-07-15 13:53:31,206 INFO MainThread:692253 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
20
+ 2021-07-15 13:53:31,206 INFO MainThread:692253 [wandb_run.py:_redirect():1502] Redirecting console.
21
+ 2021-07-15 13:53:31,208 INFO MainThread:692253 [wandb_run.py:_redirect():1558] Redirects installed.
22
+ 2021-07-15 13:53:31,208 INFO MainThread:692253 [wandb_init.py:init():554] run started, returning control to user process
23
+ 2021-07-15 13:53:31,215 INFO MainThread:692253 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 3e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 2.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul15_13-53-21_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 50, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 10000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 2000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': './', 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
24
+ 2021-07-15 13:53:31,216 INFO MainThread:692253 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
25
+ 2021-07-15 13:53:31,218 INFO MainThread:692253 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 5000}
26
+ 2021-07-15 14:10:14,362 INFO MainThread:692253 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 255
27
+ 2021-07-15 14:10:14,363 INFO MainThread:692253 [wandb_run.py:_restore():1565] restore
wandb/run-20210715_135329-1c6qs9ww/run-1c6qs9ww.wandb ADDED
Binary file (46 kB). View file
 
wandb/run-20210715_141143-1vry5yso/files/config.yaml ADDED
@@ -0,0 +1,301 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ wandb_version: 1
2
+
3
+ _wandb:
4
+ desc: null
5
+ value:
6
+ cli_version: 0.10.33
7
+ framework: huggingface
8
+ huggingface_version: 4.9.0.dev0
9
+ is_jupyter_run: false
10
+ is_kaggle_kernel: false
11
+ python_version: 3.8.10
12
+ t:
13
+ 1:
14
+ - 3
15
+ - 11
16
+ 4: 3.8.10
17
+ 5: 0.10.33
18
+ 6: 4.9.0.dev0
19
+ 8:
20
+ - 5
21
+ adafactor:
22
+ desc: null
23
+ value: false
24
+ adam_beta1:
25
+ desc: null
26
+ value: 0.9
27
+ adam_beta2:
28
+ desc: null
29
+ value: 0.98
30
+ adam_epsilon:
31
+ desc: null
32
+ value: 1.0e-08
33
+ cache_dir:
34
+ desc: null
35
+ value: null
36
+ config_name:
37
+ desc: null
38
+ value: ./
39
+ dataloader_drop_last:
40
+ desc: null
41
+ value: false
42
+ dataloader_num_workers:
43
+ desc: null
44
+ value: 0
45
+ dataloader_pin_memory:
46
+ desc: null
47
+ value: true
48
+ dataset_config_name:
49
+ desc: null
50
+ value: null
51
+ dataset_name:
52
+ desc: null
53
+ value: null
54
+ ddp_find_unused_parameters:
55
+ desc: null
56
+ value: null
57
+ debug:
58
+ desc: null
59
+ value: []
60
+ deepspeed:
61
+ desc: null
62
+ value: null
63
+ disable_tqdm:
64
+ desc: null
65
+ value: false
66
+ do_eval:
67
+ desc: null
68
+ value: false
69
+ do_predict:
70
+ desc: null
71
+ value: false
72
+ do_train:
73
+ desc: null
74
+ value: false
75
+ dtype:
76
+ desc: null
77
+ value: float32
78
+ eval_accumulation_steps:
79
+ desc: null
80
+ value: null
81
+ eval_steps:
82
+ desc: null
83
+ value: 2000
84
+ evaluation_strategy:
85
+ desc: null
86
+ value: IntervalStrategy.NO
87
+ fp16:
88
+ desc: null
89
+ value: false
90
+ fp16_backend:
91
+ desc: null
92
+ value: auto
93
+ fp16_full_eval:
94
+ desc: null
95
+ value: false
96
+ fp16_opt_level:
97
+ desc: null
98
+ value: O1
99
+ gradient_accumulation_steps:
100
+ desc: null
101
+ value: 1
102
+ greater_is_better:
103
+ desc: null
104
+ value: null
105
+ group_by_length:
106
+ desc: null
107
+ value: false
108
+ ignore_data_skip:
109
+ desc: null
110
+ value: false
111
+ label_names:
112
+ desc: null
113
+ value: null
114
+ label_smoothing_factor:
115
+ desc: null
116
+ value: 0.0
117
+ learning_rate:
118
+ desc: null
119
+ value: 3.0e-05
120
+ length_column_name:
121
+ desc: null
122
+ value: length
123
+ line_by_line:
124
+ desc: null
125
+ value: false
126
+ load_best_model_at_end:
127
+ desc: null
128
+ value: false
129
+ local_rank:
130
+ desc: null
131
+ value: -1
132
+ log_level:
133
+ desc: null
134
+ value: -1
135
+ log_level_replica:
136
+ desc: null
137
+ value: -1
138
+ log_on_each_node:
139
+ desc: null
140
+ value: true
141
+ logging_dir:
142
+ desc: null
143
+ value: ./runs/Jul15_14-11-35_t1v-n-f5c06ea1-w-0
144
+ logging_first_step:
145
+ desc: null
146
+ value: false
147
+ logging_steps:
148
+ desc: null
149
+ value: 50
150
+ logging_strategy:
151
+ desc: null
152
+ value: IntervalStrategy.STEPS
153
+ lr_scheduler_type:
154
+ desc: null
155
+ value: SchedulerType.LINEAR
156
+ max_eval_samples:
157
+ desc: null
158
+ value: 5000
159
+ max_grad_norm:
160
+ desc: null
161
+ value: 1.0
162
+ max_seq_length:
163
+ desc: null
164
+ value: 4096
165
+ max_steps:
166
+ desc: null
167
+ value: -1
168
+ metric_for_best_model:
169
+ desc: null
170
+ value: null
171
+ mlm_probability:
172
+ desc: null
173
+ value: 0.15
174
+ model_name_or_path:
175
+ desc: null
176
+ value: null
177
+ model_type:
178
+ desc: null
179
+ value: big_bird
180
+ mp_parameters:
181
+ desc: null
182
+ value: ''
183
+ no_cuda:
184
+ desc: null
185
+ value: false
186
+ num_train_epochs:
187
+ desc: null
188
+ value: 2.0
189
+ output_dir:
190
+ desc: null
191
+ value: ./
192
+ overwrite_cache:
193
+ desc: null
194
+ value: false
195
+ overwrite_output_dir:
196
+ desc: null
197
+ value: true
198
+ pad_to_max_length:
199
+ desc: null
200
+ value: false
201
+ past_index:
202
+ desc: null
203
+ value: -1
204
+ per_device_eval_batch_size:
205
+ desc: null
206
+ value: 1
207
+ per_device_train_batch_size:
208
+ desc: null
209
+ value: 1
210
+ per_gpu_eval_batch_size:
211
+ desc: null
212
+ value: null
213
+ per_gpu_train_batch_size:
214
+ desc: null
215
+ value: null
216
+ prediction_loss_only:
217
+ desc: null
218
+ value: false
219
+ preprocessing_num_workers:
220
+ desc: null
221
+ value: 96
222
+ push_to_hub:
223
+ desc: null
224
+ value: true
225
+ push_to_hub_model_id:
226
+ desc: null
227
+ value: ''
228
+ push_to_hub_organization:
229
+ desc: null
230
+ value: null
231
+ push_to_hub_token:
232
+ desc: null
233
+ value: null
234
+ remove_unused_columns:
235
+ desc: null
236
+ value: true
237
+ report_to:
238
+ desc: null
239
+ value:
240
+ - tensorboard
241
+ - wandb
242
+ resume_from_checkpoint:
243
+ desc: null
244
+ value: ./
245
+ run_name:
246
+ desc: null
247
+ value: ./
248
+ save_on_each_node:
249
+ desc: null
250
+ value: false
251
+ save_steps:
252
+ desc: null
253
+ value: 10000
254
+ save_strategy:
255
+ desc: null
256
+ value: IntervalStrategy.STEPS
257
+ save_total_limit:
258
+ desc: null
259
+ value: 5
260
+ seed:
261
+ desc: null
262
+ value: 42
263
+ sharded_ddp:
264
+ desc: null
265
+ value: []
266
+ skip_memory_metrics:
267
+ desc: null
268
+ value: true
269
+ tokenizer_name:
270
+ desc: null
271
+ value: ./
272
+ tpu_metrics_debug:
273
+ desc: null
274
+ value: false
275
+ tpu_num_cores:
276
+ desc: null
277
+ value: null
278
+ train_ref_file:
279
+ desc: null
280
+ value: null
281
+ use_fast_tokenizer:
282
+ desc: null
283
+ value: true
284
+ use_legacy_prediction_loop:
285
+ desc: null
286
+ value: false
287
+ validation_ref_file:
288
+ desc: null
289
+ value: null
290
+ validation_split_percentage:
291
+ desc: null
292
+ value: 5
293
+ warmup_ratio:
294
+ desc: null
295
+ value: 0.0
296
+ warmup_steps:
297
+ desc: null
298
+ value: 10000
299
+ weight_decay:
300
+ desc: null
301
+ value: 0.0095
wandb/run-20210715_141143-1vry5yso/files/output.log ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ [14:11:58] - INFO - absl - Restoring checkpoint from ./checkpoint_90000
3
+ tcmalloc: large alloc 1530273792 bytes == 0x9aa8a000 @ 0x7f287c292680 0x7f287c2b3824 0x5b9a14 0x50b2ae 0x50cb1b 0x5a6f17 0x5f3010 0x56fd36 0x568d9a 0x5f5b33 0x56aadf 0x568d9a 0x68cdc7 0x67e161 0x67e1df 0x67e281 0x67e627 0x6b6e62 0x6b71ed 0x7f287c0a70b3 0x5f96de
4
+ /home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:386: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
5
+ warnings.warn(
6
+ /home/dat/pino/lib/python3.8/site-packages/jax/lib/xla_bridge.py:373: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
7
+ warnings.warn(
8
+ Epoch ... (1/2): 0%| | 0/2 [00:00<?, ?it/s]
9
+
10
+ Traceback (most recent call last):███████████████████▉ | 90002/352766 [02:07<9322:46:40, 127.73s/it]
11
+ File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
12
+ self.run()
13
+ File "/usr/lib/python3.8/threading.py", line 870, in run
14
+ self._target(*self._args, **self._kwargs)
15
+ File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/wandb_run.py", line 183, in check_network_status
16
+ status_response = self._interface.communicate_network_status()
17
+ File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 755, in communicate_network_status
18
+ resp = self._communicate(req, timeout=timeout, local=True)
19
+ File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 545, in _communicate
20
+ return self._communicate_async(rec, local=local).get(timeout=timeout)
21
+ File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 550, in _communicate_async
22
+ raise Exception("The wandb backend process has shutdown")
23
+ Exception: The wandb backend process has shutdown
24
+ Exception in thread Thread-3:
25
+ Traceback (most recent call last):
26
+ File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
27
+ self.run()
28
+ File "/usr/lib/python3.8/threading.py", line 870, in run
29
+ self._target(*self._args, **self._kwargs)
30
+ File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/wandb_run.py", line 201, in check_status
31
+ status_response = self._interface.communicate_stop_status()
32
+ File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 743, in communicate_stop_status
33
+ resp = self._communicate(req, timeout=timeout, local=True)
34
+ File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 545, in _communicate
35
+ return self._communicate_async(rec, local=local).get(timeout=timeout)
36
+ File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 550, in _communicate_async
37
+ raise Exception("The wandb backend process has shutdown")
38
+ Exception: The wandb backend process has shutdown
39
+ Training...: 26%|███████████████████████████████████▋ | 90002/352766 [04:22<19182:56:09, 262.82s/it]
40
+ Epoch ... (1/2): 0%| | 0/2 [09:51<?, ?it/s]
41
+ Traceback (most recent call last):
42
+ File "./run_mlm_flax_no_accum.py", line 701, in <module>
43
+ state, train_metric, dropout_rngs = p_train_step(state, model_inputs, dropout_rngs)
44
+ File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/traceback_util.py", line 183, in reraise_with_filtered_traceback
45
+ return fun(*args, **kwargs)
46
+ File "/home/dat/pino/lib/python3.8/site-packages/jax/_src/api.py", line 1669, in f_pmapped
47
+ out = pxla.xla_pmap(
48
+ File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1620, in bind
49
+ return call_bind(self, fun, *args, **params)
50
+ File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1551, in call_bind
51
+ outs = primitive.process(top_trace, fun, tracers, params)
52
+ File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 1623, in process
53
+ return trace.process_map(self, fun, tracers, params)
54
+ File "/home/dat/pino/lib/python3.8/site-packages/jax/core.py", line 606, in process_call
55
+ return primitive.impl(f, *tracers, **params)
56
+ File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 624, in xla_pmap_impl
57
+ compiled_fun, fingerprint = parallel_callable(fun, backend, axis_name, axis_size,
58
+ File "/home/dat/pino/lib/python3.8/site-packages/jax/linear_util.py", line 262, in memoized_fun
59
+ ans = call(fun, *args)
60
+ File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/pxla.py", line 906, in parallel_callable
61
+ compiled = xla.backend_compile(backend, built, compile_options)
62
+ File "/home/dat/pino/lib/python3.8/site-packages/jax/interpreters/xla.py", line 360, in backend_compile
63
+ return backend.compile(built_c, compile_options=options)
64
+ KeyboardInterrupt
wandb/run-20210715_141143-1vry5yso/files/requirements.txt ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ absl-py==0.13.0
2
+ aiohttp==3.7.4.post0
3
+ astunparse==1.6.3
4
+ async-timeout==3.0.1
5
+ attrs==21.2.0
6
+ cachetools==4.2.2
7
+ certifi==2021.5.30
8
+ chardet==4.0.0
9
+ charset-normalizer==2.0.1
10
+ chex==0.0.8
11
+ click==8.0.1
12
+ configparser==5.0.2
13
+ cycler==0.10.0
14
+ datasets==1.9.1.dev0
15
+ dill==0.3.4
16
+ dm-tree==0.1.6
17
+ docker-pycreds==0.4.0
18
+ filelock==3.0.12
19
+ flatbuffers==1.12
20
+ flax==0.3.4
21
+ fsspec==2021.7.0
22
+ gast==0.4.0
23
+ gitdb==4.0.7
24
+ gitpython==3.1.18
25
+ google-auth-oauthlib==0.4.4
26
+ google-auth==1.32.1
27
+ google-pasta==0.2.0
28
+ grpcio==1.34.1
29
+ h5py==3.1.0
30
+ huggingface-hub==0.0.12
31
+ idna==3.2
32
+ install==1.3.4
33
+ jax==0.2.17
34
+ jaxlib==0.1.68
35
+ joblib==1.0.1
36
+ keras-nightly==2.5.0.dev2021032900
37
+ keras-preprocessing==1.1.2
38
+ kiwisolver==1.3.1
39
+ libtpu-nightly==0.1.dev20210615
40
+ markdown==3.3.4
41
+ matplotlib==3.4.2
42
+ msgpack==1.0.2
43
+ multidict==5.1.0
44
+ multiprocess==0.70.12.2
45
+ numpy==1.19.5
46
+ oauthlib==3.1.1
47
+ opt-einsum==3.3.0
48
+ optax==0.0.9
49
+ packaging==21.0
50
+ pandas==1.3.0
51
+ pathtools==0.1.2
52
+ pillow==8.3.1
53
+ pip==20.0.2
54
+ pkg-resources==0.0.0
55
+ promise==2.3
56
+ protobuf==3.17.3
57
+ psutil==5.8.0
58
+ pyarrow==4.0.1
59
+ pyasn1-modules==0.2.8
60
+ pyasn1==0.4.8
61
+ pyparsing==2.4.7
62
+ python-dateutil==2.8.1
63
+ pytz==2021.1
64
+ pyyaml==5.4.1
65
+ regex==2021.7.6
66
+ requests-oauthlib==1.3.0
67
+ requests==2.26.0
68
+ rsa==4.7.2
69
+ sacremoses==0.0.45
70
+ scipy==1.7.0
71
+ sentry-sdk==1.3.0
72
+ setuptools==44.0.0
73
+ shortuuid==1.0.1
74
+ six==1.15.0
75
+ smmap==4.0.0
76
+ subprocess32==3.5.4
77
+ tensorboard-data-server==0.6.1
78
+ tensorboard-plugin-wit==1.8.0
79
+ tensorboard==2.5.0
80
+ tensorflow-estimator==2.5.0
81
+ tensorflow==2.5.0
82
+ termcolor==1.1.0
83
+ tokenizers==0.10.3
84
+ toolz==0.11.1
85
+ tqdm==4.61.2
86
+ transformers==4.9.0.dev0
87
+ typing-extensions==3.7.4.3
88
+ urllib3==1.26.6
89
+ wandb==0.10.33
90
+ werkzeug==2.0.1
91
+ wheel==0.36.2
92
+ wrapt==1.12.1
93
+ xxhash==2.0.2
94
+ yarl==1.6.3
wandb/run-20210715_141143-1vry5yso/files/wandb-metadata.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29",
3
+ "python": "3.8.10",
4
+ "heartbeatAt": "2021-07-15T14:11:46.095724",
5
+ "startedAt": "2021-07-15T14:11:43.787979",
6
+ "docker": null,
7
+ "cpu_count": 96,
8
+ "cuda": null,
9
+ "args": [
10
+ "--push_to_hub",
11
+ "--output_dir=./",
12
+ "--model_type=big_bird",
13
+ "--config_name=./",
14
+ "--tokenizer_name=./",
15
+ "--max_seq_length=4096",
16
+ "--weight_decay=0.0095",
17
+ "--warmup_steps=10000",
18
+ "--overwrite_output_dir",
19
+ "--adam_beta1=0.9",
20
+ "--adam_beta2=0.98",
21
+ "--logging_steps=50",
22
+ "--eval_steps=2000",
23
+ "--num_train_epochs=2",
24
+ "--preprocessing_num_workers=96",
25
+ "--save_steps=10000",
26
+ "--learning_rate=3e-5",
27
+ "--per_device_train_batch_size=1",
28
+ "--per_device_eval_batch_size=1",
29
+ "--save_total_limit=5",
30
+ "--max_eval_samples=5000",
31
+ "--resume_from_checkpoint=./"
32
+ ],
33
+ "state": "running",
34
+ "program": "./run_mlm_flax_no_accum.py",
35
+ "codePath": "run_mlm_flax_no_accum.py",
36
+ "git": {
37
+ "remote": "https://huggingface.co/flax-community/pino-roberta-base",
38
+ "commit": "b6814517994e86d862ddec99dda42b9dbea5b8b0"
39
+ },
40
+ "email": null,
41
+ "root": "/home/dat/pino-roberta-base",
42
+ "host": "t1v-n-f5c06ea1-w-0",
43
+ "username": "dat",
44
+ "executable": "/home/dat/pino/bin/python"
45
+ }
wandb/run-20210715_141143-1vry5yso/files/wandb-summary.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {}
wandb/run-20210715_141143-1vry5yso/logs/debug-internal.log ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-07-15 14:11:44,509 INFO MainThread:696553 [internal.py:wandb_internal():88] W&B internal server running at pid: 696553, started at: 2021-07-15 14:11:44.509350
2
+ 2021-07-15 14:11:44,511 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: check_version
3
+ 2021-07-15 14:11:44,512 INFO WriterThread:696553 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/run-1vry5yso.wandb
4
+ 2021-07-15 14:11:44,513 DEBUG SenderThread:696553 [sender.py:send():179] send: header
5
+ 2021-07-15 14:11:44,513 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: check_version
6
+ 2021-07-15 14:11:44,549 DEBUG SenderThread:696553 [sender.py:send():179] send: run
7
+ 2021-07-15 14:11:44,551 INFO SenderThread:696553 [sender.py:_maybe_setup_resume():413] checking resume status for wandb/hf-flax-pino-roberta/1vry5yso
8
+ 2021-07-15 14:11:44,848 INFO SenderThread:696553 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files
9
+ 2021-07-15 14:11:44,849 INFO SenderThread:696553 [sender.py:_start_run_threads():716] run started: 1vry5yso with start time 1626358303
10
+ 2021-07-15 14:11:44,849 DEBUG SenderThread:696553 [sender.py:send():179] send: summary
11
+ 2021-07-15 14:11:44,849 INFO SenderThread:696553 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
12
+ 2021-07-15 14:11:44,850 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: run_start
13
+ 2021-07-15 14:11:45,906 INFO Thread-8 :696553 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/wandb-summary.json
14
+ 2021-07-15 14:11:46,095 DEBUG HandlerThread:696553 [meta.py:__init__():39] meta init
15
+ 2021-07-15 14:11:46,095 DEBUG HandlerThread:696553 [meta.py:__init__():53] meta init done
16
+ 2021-07-15 14:11:46,095 DEBUG HandlerThread:696553 [meta.py:probe():210] probe
17
+ 2021-07-15 14:11:46,097 DEBUG HandlerThread:696553 [meta.py:_setup_git():200] setup git
18
+ 2021-07-15 14:11:46,132 DEBUG HandlerThread:696553 [meta.py:_setup_git():207] setup git done
19
+ 2021-07-15 14:11:46,132 DEBUG HandlerThread:696553 [meta.py:_save_pip():57] save pip
20
+ 2021-07-15 14:11:46,132 DEBUG HandlerThread:696553 [meta.py:_save_pip():71] save pip done
21
+ 2021-07-15 14:11:46,132 DEBUG HandlerThread:696553 [meta.py:probe():252] probe done
22
+ 2021-07-15 14:11:46,136 DEBUG SenderThread:696553 [sender.py:send():179] send: files
23
+ 2021-07-15 14:11:46,136 INFO SenderThread:696553 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now
24
+ 2021-07-15 14:11:46,144 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
25
+ 2021-07-15 14:11:46,144 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
26
+ 2021-07-15 14:11:46,273 DEBUG SenderThread:696553 [sender.py:send():179] send: config
27
+ 2021-07-15 14:11:46,274 DEBUG SenderThread:696553 [sender.py:send():179] send: config
28
+ 2021-07-15 14:11:46,274 DEBUG SenderThread:696553 [sender.py:send():179] send: config
29
+ 2021-07-15 14:11:46,589 INFO Thread-11 :696553 [upload_job.py:push():137] Uploaded file /tmp/tmp6ba_u0b3wandb/11b0hrsr-wandb-metadata.json
30
+ 2021-07-15 14:11:46,906 INFO Thread-8 :696553 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/requirements.txt
31
+ 2021-07-15 14:11:46,906 INFO Thread-8 :696553 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/wandb-metadata.json
32
+ 2021-07-15 14:11:46,907 INFO Thread-8 :696553 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/output.log
33
+ 2021-07-15 14:12:01,364 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
34
+ 2021-07-15 14:12:01,364 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
35
+ 2021-07-15 14:12:01,912 INFO Thread-8 :696553 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/output.log
36
+ 2021-07-15 14:12:03,914 INFO Thread-8 :696553 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/output.log
37
+ 2021-07-15 14:12:14,183 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
38
+ 2021-07-15 14:12:15,919 INFO Thread-8 :696553 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/config.yaml
39
+ 2021-07-15 14:12:16,527 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
40
+ 2021-07-15 14:12:16,527 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
41
+ 2021-07-15 14:12:31,659 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
42
+ 2021-07-15 14:12:31,660 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
43
+ 2021-07-15 14:12:44,260 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
44
+ 2021-07-15 14:12:46,795 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
45
+ 2021-07-15 14:12:46,795 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
46
+ 2021-07-15 14:13:01,927 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
47
+ 2021-07-15 14:13:01,928 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
48
+ 2021-07-15 14:13:14,337 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
49
+ 2021-07-15 14:13:17,060 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
50
+ 2021-07-15 14:13:17,060 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
51
+ 2021-07-15 14:13:32,199 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
52
+ 2021-07-15 14:13:32,199 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
53
+ 2021-07-15 14:13:44,410 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
54
+ 2021-07-15 14:13:47,335 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
55
+ 2021-07-15 14:13:47,335 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
56
+ 2021-07-15 14:14:02,468 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
57
+ 2021-07-15 14:14:02,468 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
58
+ 2021-07-15 14:14:14,489 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
59
+ 2021-07-15 14:14:17,599 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
60
+ 2021-07-15 14:14:17,600 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
61
+ 2021-07-15 14:14:32,741 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
62
+ 2021-07-15 14:14:32,742 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
63
+ 2021-07-15 14:14:44,568 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
64
+ 2021-07-15 14:14:47,874 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
65
+ 2021-07-15 14:14:47,875 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
66
+ 2021-07-15 14:15:03,008 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
67
+ 2021-07-15 14:15:03,008 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
68
+ 2021-07-15 14:15:14,644 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
69
+ 2021-07-15 14:15:18,144 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
70
+ 2021-07-15 14:15:18,145 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
71
+ 2021-07-15 14:15:33,277 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
72
+ 2021-07-15 14:15:33,278 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
73
+ 2021-07-15 14:15:44,719 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
74
+ 2021-07-15 14:15:48,414 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
75
+ 2021-07-15 14:15:48,414 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
76
+ 2021-07-15 14:16:03,552 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
77
+ 2021-07-15 14:16:03,552 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
78
+ 2021-07-15 14:16:14,795 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
79
+ 2021-07-15 14:16:18,684 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
80
+ 2021-07-15 14:16:18,685 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
81
+ 2021-07-15 14:16:33,998 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
82
+ 2021-07-15 14:16:33,998 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
83
+ 2021-07-15 14:16:44,868 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
84
+ 2021-07-15 14:16:49,137 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
85
+ 2021-07-15 14:16:49,138 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
86
+ 2021-07-15 14:17:04,268 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
87
+ 2021-07-15 14:17:04,268 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
88
+ 2021-07-15 14:17:14,945 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
89
+ 2021-07-15 14:17:19,402 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
90
+ 2021-07-15 14:17:19,402 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
91
+ 2021-07-15 14:17:32,049 INFO Thread-8 :696553 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/output.log
92
+ 2021-07-15 14:17:34,571 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
93
+ 2021-07-15 14:17:34,572 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
94
+ 2021-07-15 14:17:45,021 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
95
+ 2021-07-15 14:17:49,721 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
96
+ 2021-07-15 14:17:49,722 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
97
+ 2021-07-15 14:18:04,866 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
98
+ 2021-07-15 14:18:04,866 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
99
+ 2021-07-15 14:18:15,099 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
100
+ 2021-07-15 14:18:19,998 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
101
+ 2021-07-15 14:18:19,998 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
102
+ 2021-07-15 14:18:35,130 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
103
+ 2021-07-15 14:18:35,130 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
104
+ 2021-07-15 14:18:45,179 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
105
+ 2021-07-15 14:18:50,263 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
106
+ 2021-07-15 14:18:50,263 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
107
+ 2021-07-15 14:19:05,399 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
108
+ 2021-07-15 14:19:05,400 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
109
+ 2021-07-15 14:19:15,259 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
110
+ 2021-07-15 14:19:20,534 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
111
+ 2021-07-15 14:19:20,534 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
112
+ 2021-07-15 14:19:35,668 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
113
+ 2021-07-15 14:19:35,668 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
114
+ 2021-07-15 14:19:40,101 INFO Thread-8 :696553 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/files/output.log
115
+ 2021-07-15 14:19:45,340 DEBUG SenderThread:696553 [sender.py:send():179] send: stats
116
+ 2021-07-15 14:19:50,825 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
117
+ 2021-07-15 14:19:50,826 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
118
+ 2021-07-15 14:20:05,974 DEBUG HandlerThread:696553 [handler.py:handle_request():124] handle_request: stop_status
119
+ 2021-07-15 14:20:05,975 DEBUG SenderThread:696553 [sender.py:send_request():193] send_request: stop_status
120
+ 2021-07-15 14:20:10,445 WARNING MainThread:696553 [internal.py:wandb_internal():147] Internal process interrupt: 1
121
+ 2021-07-15 14:20:10,628 WARNING MainThread:696553 [internal.py:wandb_internal():147] Internal process interrupt: 2
122
+ 2021-07-15 14:20:10,628 ERROR MainThread:696553 [internal.py:wandb_internal():150] Internal process interrupted.
123
+ 2021-07-15 14:20:10,822 INFO SenderThread:696553 [sender.py:finish():945] shutting down sender
124
+ 2021-07-15 14:20:10,823 INFO SenderThread:696553 [dir_watcher.py:finish():282] shutting down directory watcher
125
+ 2021-07-15 14:20:10,947 INFO MainThread:696553 [internal.py:handle_exit():78] Internal process exited
wandb/run-20210715_141143-1vry5yso/logs/debug.log ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-07-15 14:11:43,789 INFO MainThread:695296 [wandb_setup.py:_flush():69] setting env: {}
2
+ 2021-07-15 14:11:43,789 INFO MainThread:695296 [wandb_setup.py:_flush():69] setting login settings: {}
3
+ 2021-07-15 14:11:43,789 INFO MainThread:695296 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/logs/debug.log
4
+ 2021-07-15 14:11:43,789 INFO MainThread:695296 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210715_141143-1vry5yso/logs/debug-internal.log
5
+ 2021-07-15 14:11:43,790 INFO MainThread:695296 [wandb_init.py:init():370] calling init triggers
6
+ 2021-07-15 14:11:43,790 INFO MainThread:695296 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
7
+ config: {}
8
+ 2021-07-15 14:11:43,790 INFO MainThread:695296 [wandb_init.py:init():419] starting backend
9
+ 2021-07-15 14:11:43,790 INFO MainThread:695296 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
10
+ 2021-07-15 14:11:43,866 INFO MainThread:695296 [backend.py:ensure_launched():135] starting backend process...
11
+ 2021-07-15 14:11:43,921 INFO MainThread:695296 [backend.py:ensure_launched():139] started backend process with pid: 696553
12
+ 2021-07-15 14:11:43,923 INFO MainThread:695296 [wandb_init.py:init():424] backend started and connected
13
+ 2021-07-15 14:11:43,926 INFO MainThread:695296 [wandb_init.py:init():472] updated telemetry
14
+ 2021-07-15 14:11:43,927 INFO MainThread:695296 [wandb_init.py:init():491] communicating current version
15
+ 2021-07-15 14:11:44,548 INFO MainThread:695296 [wandb_init.py:init():496] got version response
16
+ 2021-07-15 14:11:44,548 INFO MainThread:695296 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
17
+ 2021-07-15 14:11:44,849 INFO MainThread:695296 [wandb_init.py:init():529] starting run threads in backend
18
+ 2021-07-15 14:11:46,139 INFO MainThread:695296 [wandb_run.py:_console_start():1623] atexit reg
19
+ 2021-07-15 14:11:46,140 INFO MainThread:695296 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT
20
+ 2021-07-15 14:11:46,141 INFO MainThread:695296 [wandb_run.py:_redirect():1502] Redirecting console.
21
+ 2021-07-15 14:11:46,142 INFO MainThread:695296 [wandb_run.py:_redirect():1558] Redirects installed.
22
+ 2021-07-15 14:11:46,143 INFO MainThread:695296 [wandb_init.py:init():554] run started, returning control to user process
23
+ 2021-07-15 14:11:46,148 INFO MainThread:695296 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 3e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 2.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul15_14-11-35_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 50, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 10000, 'save_total_limit': 5, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 2000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': './', 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''}
24
+ 2021-07-15 14:11:46,150 INFO MainThread:695296 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
25
+ 2021-07-15 14:11:46,152 INFO MainThread:695296 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 5000}
26
+ 2021-07-15 14:21:53,633 INFO MainThread:695296 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 255
27
+ 2021-07-15 14:21:53,634 INFO MainThread:695296 [wandb_run.py:_restore():1565] restore
28
+ 2021-07-15 14:21:56,528 INFO MainThread:695296 [wandb_run.py:_restore():1565] restore
wandb/run-20210715_141143-1vry5yso/run-1vry5yso.wandb ADDED
Binary file (8.12 kB). View file
 
wandb/run-20210715_142215-1vry5yso/files/config.yaml ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ wandb_version: 1
2
+
3
+ _wandb:
4
+ desc: null
5
+ value:
6
+ cli_version: 0.10.33
7
+ framework: huggingface
8
+ huggingface_version: 4.9.0.dev0
9
+ is_jupyter_run: false
10
+ is_kaggle_kernel: false
11
+ python_version: 3.8.10
12
+ t:
13
+ 1:
14
+ - 3
15
+ - 11
16
+ 4: 3.8.10
17
+ 5: 0.10.33
18
+ 6: 4.9.0.dev0
19
+ 8:
20
+ - 5
wandb/run-20210715_142215-1vry5yso/logs/debug-internal.log ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-07-15 14:22:16,240 INFO MainThread:698894 [internal.py:wandb_internal():88] W&B internal server running at pid: 698894, started at: 2021-07-15 14:22:16.240324
2
+ 2021-07-15 14:22:16,242 DEBUG HandlerThread:698894 [handler.py:handle_request():124] handle_request: check_version
3
+ 2021-07-15 14:22:16,242 INFO WriterThread:698894 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210715_142215-1vry5yso/run-1vry5yso.wandb
4
+ 2021-07-15 14:22:16,243 DEBUG SenderThread:698894 [sender.py:send():179] send: header
5
+ 2021-07-15 14:22:16,244 DEBUG SenderThread:698894 [sender.py:send_request():193] send_request: check_version
6
+ 2021-07-15 14:22:16,280 DEBUG SenderThread:698894 [sender.py:send():179] send: run
7
+ 2021-07-15 14:22:16,282 INFO SenderThread:698894 [sender.py:_maybe_setup_resume():413] checking resume status for wandb/hf-flax-pino-roberta/1vry5yso
8
+ 2021-07-15 14:22:16,566 ERROR SenderThread:698894 [internal_api.py:execute():130] 409 response executing GraphQL.
9
+ 2021-07-15 14:22:16,567 ERROR SenderThread:698894 [internal_api.py:execute():131] {"errors":[{"message":"Error 1062: Duplicate entry '370253-1vry5yso' for key 'PRIMARY'","path":["upsertBucket"]}],"data":{"upsertBucket":null}}
10
+ 2021-07-15 14:22:17,890 ERROR SenderThread:698894 [internal_api.py:execute():130] 409 response executing GraphQL.
11
+ 2021-07-15 14:22:17,890 ERROR SenderThread:698894 [internal_api.py:execute():131] {"errors":[{"message":"Error 1062: Duplicate entry '370253-1vry5yso' for key 'PRIMARY'","path":["upsertBucket"]}],"data":{"upsertBucket":null}}
12
+ 2021-07-15 14:22:20,498 ERROR SenderThread:698894 [internal_api.py:execute():130] 409 response executing GraphQL.
13
+ 2021-07-15 14:22:20,498 ERROR SenderThread:698894 [internal_api.py:execute():131] {"errors":[{"message":"Error 1062: Duplicate entry '370253-1vry5yso' for key 'PRIMARY'","path":["upsertBucket"]}],"data":{"upsertBucket":null}}
14
+ 2021-07-15 14:22:20,499 ERROR SenderThread:698894 [retry.py:__call__():126] Retry attempt failed:
15
+ Traceback (most recent call last):
16
+ File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/lib/retry.py", line 102, in __call__
17
+ result = self._call_fn(*args, **kwargs)
18
+ File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/internal/internal_api.py", line 133, in execute
19
+ six.reraise(*sys.exc_info())
20
+ File "/home/dat/pino/lib/python3.8/site-packages/six.py", line 703, in reraise
21
+ raise value
22
+ File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/internal/internal_api.py", line 127, in execute
23
+ return self.client.execute(*args, **kwargs)
24
+ File "/home/dat/pino/lib/python3.8/site-packages/wandb/vendor/gql-0.2.0/gql/client.py", line 52, in execute
25
+ result = self._get_result(document, *args, **kwargs)
26
+ File "/home/dat/pino/lib/python3.8/site-packages/wandb/vendor/gql-0.2.0/gql/client.py", line 60, in _get_result
27
+ return self.transport.execute(document, *args, **kwargs)
28
+ File "/home/dat/pino/lib/python3.8/site-packages/wandb/vendor/gql-0.2.0/gql/transport/requests.py", line 39, in execute
29
+ request.raise_for_status()
30
+ File "/home/dat/pino/lib/python3.8/site-packages/requests/models.py", line 953, in raise_for_status
31
+ raise HTTPError(http_error_msg, response=self)
32
+ requests.exceptions.HTTPError: 409 Client Error: Conflict for url: https://api.wandb.ai/graphql
33
+ 2021-07-15 14:22:25,627 ERROR SenderThread:698894 [internal_api.py:execute():130] 409 response executing GraphQL.
34
+ 2021-07-15 14:22:25,627 ERROR SenderThread:698894 [internal_api.py:execute():131] {"errors":[{"message":"Error 1062: Duplicate entry '370253-1vry5yso' for key 'PRIMARY'","path":["upsertBucket"]}],"data":{"upsertBucket":null}}
35
+ 2021-07-15 14:22:33,972 ERROR SenderThread:698894 [internal_api.py:execute():130] 409 response executing GraphQL.
36
+ 2021-07-15 14:22:33,972 ERROR SenderThread:698894 [internal_api.py:execute():131] {"errors":[{"message":"Error 1062: Duplicate entry '370253-1vry5yso' for key 'PRIMARY'","path":["upsertBucket"]}],"data":{"upsertBucket":null}}
37
+ 2021-07-15 14:22:46,281 DEBUG HandlerThread:698894 [handler.py:handle_request():124] handle_request: shutdown
38
+ 2021-07-15 14:22:46,281 INFO HandlerThread:698894 [handler.py:finish():638] shutting down handler
39
+ 2021-07-15 14:22:46,286 INFO WriterThread:698894 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210715_142215-1vry5yso/run-1vry5yso.wandb
40
+ 2021-07-15 14:22:52,665 ERROR SenderThread:698894 [internal_api.py:execute():130] 409 response executing GraphQL.
41
+ 2021-07-15 14:22:52,665 ERROR SenderThread:698894 [internal_api.py:execute():131] {"errors":[{"message":"Error 1062: Duplicate entry '370253-1vry5yso' for key 'PRIMARY'","path":["upsertBucket"]}],"data":{"upsertBucket":null}}
42
+ 2021-07-15 14:23:25,999 ERROR SenderThread:698894 [internal_api.py:execute():130] 409 response executing GraphQL.
43
+ 2021-07-15 14:23:25,999 ERROR SenderThread:698894 [internal_api.py:execute():131] {"errors":[{"message":"Error 1062: Duplicate entry '370253-1vry5yso' for key 'PRIMARY'","path":["upsertBucket"]}],"data":{"upsertBucket":null}}
44
+ 2021-07-15 14:24:44,355 ERROR SenderThread:698894 [internal_api.py:execute():130] 409 response executing GraphQL.
45
+ 2021-07-15 14:24:44,355 ERROR SenderThread:698894 [internal_api.py:execute():131] {"errors":[{"message":"Error 1062: Duplicate entry '370253-1vry5yso' for key 'PRIMARY'","path":["upsertBucket"]}],"data":{"upsertBucket":null}}
46
+ 2021-07-15 14:27:09,800 INFO MainThread:698894 [internal.py:handle_exit():78] Internal process exited
wandb/run-20210715_142215-1vry5yso/logs/debug.log ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2021-07-15 14:22:15,517 INFO MainThread:697638 [wandb_setup.py:_flush():69] setting env: {}
2
+ 2021-07-15 14:22:15,517 INFO MainThread:697638 [wandb_setup.py:_flush():69] setting login settings: {}
3
+ 2021-07-15 14:22:15,517 INFO MainThread:697638 [wandb_init.py:_log_setup():337] Logging user logs to /home/dat/pino-roberta-base/wandb/run-20210715_142215-1vry5yso/logs/debug.log
4
+ 2021-07-15 14:22:15,517 INFO MainThread:697638 [wandb_init.py:_log_setup():338] Logging internal logs to /home/dat/pino-roberta-base/wandb/run-20210715_142215-1vry5yso/logs/debug-internal.log
5
+ 2021-07-15 14:22:15,517 INFO MainThread:697638 [wandb_init.py:init():370] calling init triggers
6
+ 2021-07-15 14:22:15,517 INFO MainThread:697638 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
7
+ config: {}
8
+ 2021-07-15 14:22:15,517 INFO MainThread:697638 [wandb_init.py:init():419] starting backend
9
+ 2021-07-15 14:22:15,518 INFO MainThread:697638 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
10
+ 2021-07-15 14:22:15,565 INFO MainThread:697638 [backend.py:ensure_launched():135] starting backend process...
11
+ 2021-07-15 14:22:15,610 INFO MainThread:697638 [backend.py:ensure_launched():139] started backend process with pid: 698894
12
+ 2021-07-15 14:22:15,612 INFO MainThread:697638 [wandb_init.py:init():424] backend started and connected
13
+ 2021-07-15 14:22:15,615 INFO MainThread:697638 [wandb_init.py:init():472] updated telemetry
14
+ 2021-07-15 14:22:15,616 INFO MainThread:697638 [wandb_init.py:init():491] communicating current version
15
+ 2021-07-15 14:22:16,279 INFO MainThread:697638 [wandb_init.py:init():496] got version response
16
+ 2021-07-15 14:22:16,279 INFO MainThread:697638 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
17
+ 2021-07-15 14:22:46,280 INFO MainThread:697638 [interface.py:_communicate_run():657] couldn't get run from backend
18
+ 2021-07-15 14:22:46,280 ERROR MainThread:697638 [wandb_init.py:init():508] backend process timed out
19
+ 2021-07-15 14:22:46,280 ERROR MainThread:697638 [wandb_init.py:init():517] encountered error: Error communicating with wandb process
20
+ try: wandb.init(settings=wandb.Settings(start_method='fork'))
21
+ or: wandb.init(settings=wandb.Settings(start_method='thread'))
22
+ For more info see: https://docs.wandb.ai/library/init#init-start-error
23
+ 2021-07-15 14:27:09,566 WARNING MainThread:697638 [wandb_init.py:init():781] interrupted
24
+ Traceback (most recent call last):
25
+ File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/wandb_init.py", line 761, in init
26
+ run = wi.init()
27
+ File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/wandb_init.py", line 520, in init
28
+ backend.cleanup()
29
+ File "/home/dat/pino/lib/python3.8/site-packages/wandb/sdk/backend/backend.py", line 168, in cleanup
30
+ self.wandb_process.join()
31
+ File "/usr/lib/python3.8/multiprocessing/process.py", line 149, in join
32
+ res = self._popen.wait(timeout)
33
+ File "/usr/lib/python3.8/multiprocessing/popen_fork.py", line 47, in wait
34
+ return self.poll(os.WNOHANG if timeout == 0.0 else 0)
35
+ File "/usr/lib/python3.8/multiprocessing/popen_fork.py", line 27, in poll
36
+ pid, sts = os.waitpid(self.pid, flag)
37
+ KeyboardInterrupt
wandb/run-20210715_142215-1vry5yso/run-1vry5yso.wandb ADDED
Binary file (194 Bytes). View file
 
wandb/wandb-resume.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"run_id": "1vry5yso"}