update pt model

events.out.tfevents.1626474829.t1v-n-f5c06ea1-w-0.798495.3.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f059fcaed778fbca8e5101b541e74ed0f069df98b386e1c0671d14fe84f987f3
-size 13537318
+oid sha256:ccdb7f11327d987ca59d1c9b41d2e6654ad39f0d1ac33ba9eae0c6b0e271ce3e
+size 13552428

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b790aa29e6afdb9ee10d9eb4a2b45f5db49b0f6ebfac95e8c220af4c5c68954f
+oid sha256:cbd6001a3f6c1af1bbbb0597fb1513f3a7507286611b0800c408d926e2c4b5e3
 size 512555623

wandb/run-20210716_223350-8eukt20m/files/output.log

@@ -10662,3 +10662,21 @@ Training...: 89999it [10:24:09,  2.75it/s]████████████
 [09:05:03] - INFO - absl - Saved checkpoint at checkpoint_420000█████████████████████████████████| 500/500 [00:59<00:00,  7.90it/s]
 [09:05:04] - INFO - huggingface_hub.repository - git version 2.25.1
 git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
+[09:05:05] - DEBUG - huggingface_hub.repository - [Repository] is a valid git repo
+[09:06:13] - INFO - huggingface_hub.repository - Uploading LFS objects: 100% (3/3), 2.1 GB | 48 MB/s, done.
+
+
+
+
+
+Training...: 90052it [10:26:52,  2.22s/it]
+
+
+
+
+
+Training...: 90102it [10:27:12,  2.22s/it]
+
+
+
+

wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json

@@ -1 +1 @@
-{"training_step": 420000, "learning_rate": 2.1220957933110185e-05, "train_loss": 1.9748303890228271, "_runtime": 37865, "_timestamp": 1626512695, "_step": 1808, "eval_step": 420000, "eval_accuracy": 0.6525571346282959, "eval_loss": 1.7464253902435303}
+{"training_step": 420100, "learning_rate": 2.1218815163592808e-05, "train_loss": 1.9333391189575195, "_runtime": 37984, "_timestamp": 1626512814, "_step": 1810, "eval_step": 420000, "eval_accuracy": 0.6525571346282959, "eval_loss": 1.7464253902435303}

wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log

@@ -23290,3 +23290,64 @@
 2021-07-17 09:05:07,856 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
 2021-07-17 09:05:10,945 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
 2021-07-17 09:05:10,945 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 09:05:23,481 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-17 09:05:26,075 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 09:05:26,076 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 09:05:41,206 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 09:05:41,207 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 09:05:53,552 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-17 09:05:56,339 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 09:05:56,503 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 09:06:11,635 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 09:06:11,636 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 09:06:15,886 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 09:06:17,886 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 09:06:19,887 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 09:06:21,888 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 09:06:23,623 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-17 09:06:23,889 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 09:06:26,767 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 09:06:26,767 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 09:06:34,948 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-17 09:06:34,949 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-17 09:06:34,949 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-17 09:06:35,894 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 09:06:35,895 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-17 09:06:37,895 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 09:06:39,896 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 09:06:41,897 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 09:06:41,900 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 09:06:41,900 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 09:06:43,898 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 09:06:53,696 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-17 09:06:54,996 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-17 09:06:54,997 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-17 09:06:54,997 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-17 09:06:55,904 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 09:06:55,904 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-17 09:06:57,036 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 09:06:57,036 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 09:06:57,905 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 09:06:59,906 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 09:07:01,907 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 09:07:03,908 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 09:07:05,086 WARNING MainThread:799749 [internal.py:wandb_internal():147] Internal process interrupt: 1
+2021-07-17 09:07:05,580 WARNING MainThread:799749 [internal.py:wandb_internal():147] Internal process interrupt: 2
+2021-07-17 09:07:05,580 ERROR   MainThread:799749 [internal.py:wandb_internal():150] Internal process interrupted.
+2021-07-17 09:07:05,977 INFO    SenderThread:799749 [sender.py:finish():945] shutting down sender
+2021-07-17 09:07:05,977 INFO    SenderThread:799749 [dir_watcher.py:finish():282] shutting down directory watcher
+2021-07-17 09:07:05,979 INFO    HandlerThread:799749 [handler.py:finish():638] shutting down handler
+2021-07-17 09:07:06,379 INFO    WriterThread:799749 [datastore.py:close():288] close: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/run-8eukt20m.wandb
+2021-07-17 09:07:06,909 INFO    SenderThread:799749 [dir_watcher.py:finish():312] scan: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files
+2021-07-17 09:07:06,910 INFO    SenderThread:799749 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/requirements.txt requirements.txt
+2021-07-17 09:07:06,910 INFO    SenderThread:799749 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log output.log
+2021-07-17 09:07:06,910 INFO    SenderThread:799749 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-metadata.json wandb-metadata.json
+2021-07-17 09:07:06,910 INFO    SenderThread:799749 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/config.yaml config.yaml
+2021-07-17 09:07:06,910 INFO    SenderThread:799749 [dir_watcher.py:finish():318] scan save: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json wandb-summary.json
+2021-07-17 09:07:06,911 INFO    SenderThread:799749 [file_pusher.py:finish():177] shutting down file pusher
+2021-07-17 09:07:06,911 INFO    SenderThread:799749 [file_pusher.py:join():182] waiting for file pusher
+2021-07-17 09:07:07,386 INFO    Thread-14 :799749 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/config.yaml
+2021-07-17 09:07:07,400 INFO    Thread-12 :799749 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/requirements.txt
+2021-07-17 09:07:07,408 INFO    Thread-15 :799749 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-17 09:07:07,614 INFO    Thread-13 :799749 [upload_job.py:push():137] Uploaded file /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 09:07:08,201 INFO    MainThread:799749 [internal.py:handle_exit():78] Internal process exited

wandb/run-20210716_223350-8eukt20m/logs/debug.log

@@ -24,3 +24,5 @@ config: {}
 2021-07-16 22:33:52,807 INFO    MainThread:798495 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 1, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 3e-05, 'weight_decay': 0.0095, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 4.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 10000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul16_22-33-42_t1v-n-f5c06ea1-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 50, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 15000, 'save_total_limit': 50, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 10000, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': './', 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': '', '_n_gpu': 0, '__cached__setup_devices': 'cpu'}
 2021-07-16 22:33:52,809 INFO    MainThread:798495 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'big_bird', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'}
 2021-07-16 22:33:52,811 INFO    MainThread:798495 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': None, 'dataset_config_name': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 4096, 'preprocessing_num_workers': 96, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False, 'max_eval_samples': 4000}
+2021-07-17 09:07:05,456 INFO    MainThread:798495 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 255
+2021-07-17 09:07:05,456 INFO    MainThread:798495 [wandb_run.py:_restore():1565] restore