diff --git a/events.out.tfevents.1625836684.t1v-n-112df4a9-w-0.38631.3.v2 b/events.out.tfevents.1625836684.t1v-n-112df4a9-w-0.38631.3.v2 new file mode 100644 index 0000000000000000000000000000000000000000..ec5323508966889ef50cb2b358d0ebf025655bf6 --- /dev/null +++ b/events.out.tfevents.1625836684.t1v-n-112df4a9-w-0.38631.3.v2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1198c8afc6504eecba539da0dee931919669a28655dbd66157e980a6606dbd50 +size 11274 diff --git a/flax_model.msgpack b/flax_model.msgpack index c6966c9753e2c2e66d1c98c8da2678ffabe7b54b..6b859fe11525d61b6506df8b2dcf21699abbd7c9 100644 --- a/flax_model.msgpack +++ b/flax_model.msgpack @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:76d55c57630984430801c3b6d5cc75703bd2a61ef6bb8f9d20f7c6150f2dfbd3 +oid sha256:f1f30bd95331d281308f6cb94e74bec0d82ebc06684501d536c8d38952e09abb size 498796983 diff --git a/run.log b/run.log index 0d9dd2bb6a374136dccd1deb0df713f52d8333f4..586ecf24b220d0db237a9804856af165bce8f28d 100644 --- a/run.log +++ b/run.log @@ -1,4 +1,5 @@ -[10:39:50] - INFO - __main__ - Training/evaluation parameters TrainingArguments( +2021-07-09 13:17:54.997560: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory +[13:17:56] - INFO - __main__ - Training/evaluation parameters TrainingArguments( _n_gpu=-1, adafactor=False, adam_beta1=0.9, @@ -34,7 +35,7 @@ local_rank=-1, log_level=-1, log_level_replica=-1, log_on_each_node=True, -logging_dir=./runs/Jul09_10-39-50_t1v-n-112df4a9-w-0, +logging_dir=./runs/Jul09_13-17-56_t1v-n-112df4a9-w-0, logging_first_step=False, logging_steps=500, logging_strategy=IntervalStrategy.STEPS, @@ -73,326 +74,181 @@ warmup_ratio=0.0, warmup_steps=1000, weight_decay=0.0, ) -[10:39:50] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443 -[10:39:50] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0 -[10:39:50] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443 -[10:39:50] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0 -[10:39:50] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443 -[10:39:50] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0 -[10:39:50] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2) -[10:39:50] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443 -[10:39:51] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0 -[10:39:51] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443 -[10:39:51] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0 -[10:39:51] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443 -[10:39:51] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0 -[10:39:51] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2) -[10:39:51] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443 -[10:39:51] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0 -[10:39:51] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443 -[10:39:51] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0 -[10:39:51] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443 -[10:39:51] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0 -[10:39:51] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2) -[10:39:51] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-0f52086e7b10d7e8.arrow -[10:39:51] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-a39e5f5a5c6c69fc.arrow -[10:39:51] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-e4d3282a2dd50fa0.arrow -[10:39:51] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-b9a3aa9913be3b34.arrow -[10:39:51] - INFO - absl - Starting the local TPU driver. -[10:39:51] - INFO - absl - Unable to initialize backend 'tpu_driver': Not found: Unable to find driver in registry given worker: local:// -[10:39:51] - INFO - absl - Unable to initialize backend 'gpu': Not found: Could not find registered platform with name: "cuda". Available platform names are: Interpreter Host TPU -[10:39:55] - WARNING - __main__ - Unable to display metrics through TensorBoard because some package are not installed: No module named 'tensorflow' +[13:17:56] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443 +[13:17:56] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0 +[13:17:56] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443 +[13:17:56] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0 +[13:17:56] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443 +[13:17:56] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0 +[13:17:56] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2) +[13:17:56] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443 +[13:17:57] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0 +[13:17:57] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443 +[13:17:57] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0 +[13:17:57] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443 +[13:17:57] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0 +[13:17:57] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2) +[13:17:57] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443 +[13:17:57] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0 +[13:17:57] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443 +[13:17:57] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0 +[13:17:57] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443 +[13:17:57] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0 +[13:17:57] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2) +[13:17:57] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-0f52086e7b10d7e8.arrow +[13:17:57] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-a39e5f5a5c6c69fc.arrow + 0%| | 0/5 [00:00&1 | tee run.log + --push_to_hub 2>&1 | tee run.log \ No newline at end of file diff --git a/run_mlm_flax.py b/run_mlm_flax.py index 0a9f959f20676735c23a3b25ae55660b9d9fc1d6..4282560dac4063ace3e8eb814f827b6ccb23037b 100755 --- a/run_mlm_flax.py +++ b/run_mlm_flax.py @@ -431,7 +431,8 @@ if __name__ == "__main__": total_length = len(concatenated_examples[list(examples.keys())[0]]) # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can # customize this part to your needs. - total_length = (total_length // max_seq_length) * max_seq_length + if total_length >= max_seq_length: + total_length = (total_length // max_seq_length) * max_seq_length # Split by chunks of max_len. result = { k: [t[i : i + max_seq_length] for i in range(0, total_length, max_seq_length)] @@ -457,6 +458,7 @@ if __name__ == "__main__": if has_tensorboard and jax.process_index() == 0: try: from flax.metrics.tensorboard import SummaryWriter + summary_writer = SummaryWriter(log_dir=Path(training_args.output_dir)) except ImportError as ie: has_tensorboard = False @@ -477,7 +479,14 @@ if __name__ == "__main__": rng = jax.random.PRNGKey(training_args.seed) dropout_rngs = jax.random.split(rng, jax.local_device_count()) - model = FlaxAutoModelForMaskedLM.from_config(config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype)) + if model_args.model_name_or_path: + model = FlaxAutoModelForMaskedLM.from_pretrained( + model_args.model_name_or_path, config=config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype) + ) + else: + model = FlaxAutoModelForMaskedLM.from_config( + config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype) + ) # Store some constant num_epochs = int(training_args.num_train_epochs) @@ -654,7 +663,6 @@ if __name__ == "__main__": # Save metrics if has_tensorboard and jax.process_index() == 0: - cur_step = epoch * (len(tokenized_datasets["train"]) // train_batch_size) write_eval_metric(summary_writer, eval_metrics, cur_step) if cur_step % training_args.save_steps == 0 and cur_step > 0: diff --git a/test_mr_tokenizer.py b/test_mr_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..077a51f1a16d119b44d5bf9b347b5671db450a60 --- /dev/null +++ b/test_mr_tokenizer.py @@ -0,0 +1,8 @@ +from tokenizers import Tokenizer + +tokenizer = Tokenizer.from_file("tokenizer.json") + +text = "गेल्या अनेक दिवसांपासून केंद्र सरकरा विरुद्ध ट्विटर इंडिया यांच्यामध्ये सरकारच्या नव्या मार्गदर्शक सूचना किंवा नियमावलीवरून वाद सुरू आहे." + +output = tokenizer.encode(text) +print(tokenizer.decode(output.ids)) \ No newline at end of file diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log index 0831265b0724374eccb9c4a3d1f3d1c4340aaff2..cf98c3b8536a11dc6f6415ee7cabd0c5a2b99abf 120000 --- a/wandb/debug-internal.log +++ b/wandb/debug-internal.log @@ -1 +1 @@ -run-20210709_103433-d7n9baaw/logs/debug-internal.log \ No newline at end of file +run-20210709_125049-89spqu1d/logs/debug-internal.log \ No newline at end of file diff --git a/wandb/debug.log b/wandb/debug.log index 505c35c036e77dbfe35d84812762d7e09bdb1e68..b46a7c2f390e314b90985c739d6861c1999c837a 120000 --- a/wandb/debug.log +++ b/wandb/debug.log @@ -1 +1 @@ -run-20210709_103433-d7n9baaw/logs/debug.log \ No newline at end of file +run-20210709_125049-89spqu1d/logs/debug.log \ No newline at end of file diff --git a/wandb/latest-run b/wandb/latest-run index 152e9e2a23af4b3f6ad9e7e5566a604e719d681a..7bf034cd67aafbb99ba9d055e1c9f0d8c4e3e5c3 120000 --- a/wandb/latest-run +++ b/wandb/latest-run @@ -1 +1 @@ -run-20210709_103433-d7n9baaw \ No newline at end of file +run-20210709_125049-89spqu1d \ No newline at end of file diff --git a/wandb/run-20210709_111555-275xd40l/files/config.yaml b/wandb/run-20210709_111555-275xd40l/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07909a1cb2de14eeee62be4583ecc56b60e954ea --- /dev/null +++ b/wandb/run-20210709_111555-275xd40l/files/config.yaml @@ -0,0 +1,305 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 11 + 2: + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 +adafactor: + desc: null + value: false +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.999 +adam_epsilon: + desc: null + value: 1.0e-08 +cache_dir: + desc: null + value: null +config_name: + desc: null + value: ./ +dataloader_drop_last: + desc: null + value: false +dataloader_num_workers: + desc: null + value: 0 +dataloader_pin_memory: + desc: null + value: true +dataset_config_name: + desc: null + value: unshuffled_deduplicated_als +dataset_name: + desc: null + value: oscar +ddp_find_unused_parameters: + desc: null + value: null +debug: + desc: null + value: [] +deepspeed: + desc: null + value: null +disable_tqdm: + desc: null + value: false +do_eval: + desc: null + value: false +do_predict: + desc: null + value: false +do_train: + desc: null + value: false +dtype: + desc: null + value: float32 +eval_accumulation_steps: + desc: null + value: null +eval_steps: + desc: null + value: 500 +evaluation_strategy: + desc: null + value: IntervalStrategy.NO +fp16: + desc: null + value: false +fp16_backend: + desc: null + value: auto +fp16_full_eval: + desc: null + value: false +fp16_opt_level: + desc: null + value: O1 +gradient_accumulation_steps: + desc: null + value: 1 +greater_is_better: + desc: null + value: null +group_by_length: + desc: null + value: false +ignore_data_skip: + desc: null + value: false +label_names: + desc: null + value: null +label_smoothing_factor: + desc: null + value: 0.0 +learning_rate: + desc: null + value: 0.0003 +length_column_name: + desc: null + value: length +line_by_line: + desc: null + value: false +load_best_model_at_end: + desc: null + value: false +local_rank: + desc: null + value: -1 +log_level: + desc: null + value: -1 +log_level_replica: + desc: null + value: -1 +log_on_each_node: + desc: null + value: true +logging_dir: + desc: null + value: ./runs/Jul09_11-15-50_t1v-n-112df4a9-w-0 +logging_first_step: + desc: null + value: false +logging_steps: + desc: null + value: 500 +logging_strategy: + desc: null + value: IntervalStrategy.STEPS +lr_scheduler_type: + desc: null + value: SchedulerType.LINEAR +max_grad_norm: + desc: null + value: 1.0 +max_seq_length: + desc: null + value: 128 +max_steps: + desc: null + value: -1 +metric_for_best_model: + desc: null + value: null +mlm_probability: + desc: null + value: 0.15 +model_name_or_path: + desc: null + value: null +model_type: + desc: null + value: roberta +mp_parameters: + desc: null + value: '' +no_cuda: + desc: null + value: false +num_train_epochs: + desc: null + value: 8.0 +output_dir: + desc: null + value: ./ +overwrite_cache: + desc: null + value: false +overwrite_output_dir: + desc: null + value: true +pad_to_max_length: + desc: null + value: false +past_index: + desc: null + value: -1 +per_device_eval_batch_size: + desc: null + value: 4 +per_device_train_batch_size: + desc: null + value: 4 +per_gpu_eval_batch_size: + desc: null + value: null +per_gpu_train_batch_size: + desc: null + value: null +prediction_loss_only: + desc: null + value: false +preprocessing_num_workers: + desc: null + value: null +push_to_hub: + desc: null + value: false +push_to_hub_model_id: + desc: null + value: '' +push_to_hub_organization: + desc: null + value: null +push_to_hub_token: + desc: null + value: null +remove_unused_columns: + desc: null + value: true +report_to: + desc: null + value: + - tensorboard + - wandb +resume_from_checkpoint: + desc: null + value: null +run_name: + desc: null + value: ./ +save_on_each_node: + desc: null + value: false +save_steps: + desc: null + value: 500 +save_strategy: + desc: null + value: IntervalStrategy.STEPS +save_total_limit: + desc: null + value: null +seed: + desc: null + value: 42 +sharded_ddp: + desc: null + value: [] +skip_memory_metrics: + desc: null + value: true +tokenizer_name: + desc: null + value: ./ +tpu_metrics_debug: + desc: null + value: false +tpu_num_cores: + desc: null + value: null +train_file: + desc: null + value: null +train_ref_file: + desc: null + value: null +use_fast_tokenizer: + desc: null + value: true +use_legacy_prediction_loop: + desc: null + value: false +validation_file: + desc: null + value: null +validation_ref_file: + desc: null + value: null +validation_split_percentage: + desc: null + value: 5 +warmup_ratio: + desc: null + value: 0.0 +warmup_steps: + desc: null + value: 1000 +weight_decay: + desc: null + value: 0.0 diff --git a/wandb/run-20210709_111555-275xd40l/files/output.log b/wandb/run-20210709_111555-275xd40l/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e3f0fa4a46f6817520ddc8a5aed36e4e8bf67d4c --- /dev/null +++ b/wandb/run-20210709_111555-275xd40l/files/output.log @@ -0,0 +1,33 @@ + +[11:15:56] - WARNING - __main__ - Unable to display metrics through TensorBoard because some package are not installed: No module named 'tensorflow' +/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/8): 0%| | 0/8 [00:00>, args: ('https://api.wandb.ai/files/nipunsadvilkar/hf-flax-robert-base-mr/2pc90wcd/file_stream',), kwargs: {'json': {'files': {'wandb-events.jsonl': {'offset': 1, 'content': ['{"system.cpu": 4.69, "system.memory": 2.63, "system.disk": 29.2, "system.proc.memory.availableMB": 333586.39, "system.proc.memory.rssMB": 72.37, "system.proc.memory.percent": 0.02, "system.proc.cpu.threads": 12.0, "system.network.sent": 2873699, "system.network.recv": 4435580, "_wandb": true, "_timestamp": 1625833687, "_runtime": 58}']}}}} +2021-07-09 12:28:10,019 DEBUG HandlerThread:33395 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:28:10,019 DEBUG SenderThread:33395 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:28:25,052 DEBUG HandlerThread:33395 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:28:25,052 DEBUG SenderThread:33395 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:28:28,818 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log +2021-07-09 12:28:30,819 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log +2021-07-09 12:28:32,820 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log +2021-07-09 12:28:34,821 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log +2021-07-09 12:28:36,822 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log +2021-07-09 12:28:38,069 DEBUG SenderThread:33395 [sender.py:send():179] send: stats +2021-07-09 12:28:38,823 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log +2021-07-09 12:28:40,086 DEBUG HandlerThread:33395 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:28:40,086 DEBUG SenderThread:33395 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:28:40,824 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log +2021-07-09 12:28:42,825 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log +2021-07-09 12:28:43,425 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:43,425 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:43,426 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:43,826 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/wandb-summary.json +2021-07-09 12:28:43,934 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:43,936 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:43,949 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:43,949 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:43,949 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:43,955 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:43,955 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:43,955 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:43,963 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:43,983 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:43,983 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:43,986 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:43,986 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:43,987 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:43,989 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:43,990 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:43,990 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,011 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,013 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,013 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,034 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,034 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,034 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,042 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,044 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,051 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,072 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,072 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,072 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,072 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,072 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,072 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,073 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,073 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,073 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,074 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,074 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,074 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,074 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,074 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,074 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,075 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,075 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,075 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,075 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,076 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,076 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,076 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,076 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,076 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,077 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,077 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,077 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,077 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,077 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,077 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,078 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,078 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,078 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,079 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,079 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,079 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,092 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,092 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,092 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,093 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,093 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,093 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,093 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,093 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,094 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,094 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,094 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,094 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,094 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,095 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,095 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,095 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,095 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,095 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,096 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,096 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,096 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,096 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,096 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,096 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,097 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,097 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,097 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,097 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,098 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,098 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,098 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,098 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,098 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,099 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,099 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,099 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,099 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,099 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,099 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,100 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,100 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,100 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,100 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,100 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,100 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,101 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,101 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,101 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,101 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,101 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,101 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,102 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,102 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,102 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,102 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,102 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,102 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,103 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,103 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,103 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,103 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,103 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,103 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,104 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,104 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,104 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,104 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,104 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,105 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,105 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,105 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,105 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,105 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,106 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,106 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,106 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,106 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,106 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,107 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,107 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,107 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,107 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,107 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,108 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,108 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,108 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,108 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,109 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,109 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,109 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,109 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,109 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,109 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,110 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,110 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,110 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,110 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,110 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,110 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,111 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,111 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,111 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,111 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,111 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,111 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,112 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,112 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,112 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,112 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,112 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,112 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,113 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,113 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,113 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,113 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,113 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,114 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,114 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,114 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,114 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,114 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,114 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,115 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,115 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,115 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,115 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,115 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,116 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,116 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,116 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,116 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,116 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,117 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,117 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,117 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,117 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,118 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,118 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,118 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,118 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,118 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,119 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,119 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,119 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,119 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,119 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,119 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,120 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,120 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,120 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,120 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,120 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,120 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,121 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,121 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,121 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,121 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,122 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,122 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,122 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,122 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,122 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,122 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,123 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,123 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,123 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,123 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,123 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,123 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,124 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,124 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,124 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,124 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,124 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,124 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,124 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,125 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,125 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,125 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,125 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,125 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,126 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,126 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,126 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,126 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,126 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,127 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,127 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,127 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,127 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,127 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,127 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,128 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,128 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,128 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,128 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,128 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,129 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,129 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,129 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,129 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,129 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,129 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,130 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,130 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,130 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,130 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,130 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,130 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,130 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,131 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,131 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,131 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,131 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,131 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,132 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,132 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,132 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,132 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,132 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,133 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,133 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,133 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,133 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,133 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,134 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,134 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,134 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,134 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,134 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,134 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,135 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,135 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,135 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,136 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,136 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,136 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,136 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,136 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,136 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,137 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,137 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,137 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,137 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,138 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,138 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,138 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,138 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,138 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,139 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,139 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,139 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,139 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,139 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,139 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,140 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,140 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,140 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,140 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,140 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,141 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,141 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,141 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,141 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,141 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,142 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,142 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,142 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,142 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,142 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,143 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,143 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,143 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,143 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,143 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,143 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,144 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,144 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,144 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,144 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,144 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,144 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,145 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,145 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,145 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,145 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,145 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,145 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,146 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,146 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,146 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,146 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,146 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,146 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,147 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,147 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,147 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,147 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,147 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,147 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,148 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,148 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,148 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,148 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,148 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,149 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,149 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,149 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,149 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,149 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,150 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,150 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,150 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,150 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,150 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,150 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,150 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,151 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,151 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,151 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,151 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,151 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,152 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,152 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,152 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,152 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,152 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,152 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,153 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,153 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,153 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,153 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,153 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,153 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,154 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,154 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,154 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,154 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,154 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,154 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,155 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,155 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,155 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,155 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,155 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,155 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,156 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,156 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,156 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,156 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,156 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,157 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,157 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,157 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,157 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,157 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,158 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,158 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,158 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,158 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,159 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,159 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,159 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,159 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,159 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,159 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,160 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,160 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,160 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,160 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,160 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,160 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,161 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,161 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,161 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,161 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,161 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,161 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,162 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,162 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,162 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,162 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,162 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,162 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,163 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,163 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,163 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,163 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,163 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,164 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,164 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,164 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,164 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,164 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,165 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,165 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,165 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,165 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,165 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,165 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,166 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,166 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,166 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,166 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,166 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,167 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,167 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,167 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,167 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,167 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,167 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,168 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,168 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,168 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,168 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,168 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,169 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,169 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,169 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,169 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,169 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,170 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,170 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,170 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,170 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,170 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,170 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,171 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,171 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,171 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,171 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,171 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,172 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,172 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,172 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,172 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,172 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,172 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,173 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,173 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,173 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,173 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,173 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,173 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,174 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,174 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,174 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,174 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,174 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,174 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,175 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,175 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,175 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,175 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,175 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,176 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,176 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,176 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,176 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,176 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,177 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,177 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,177 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,177 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,177 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,178 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,178 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,178 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,179 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,179 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,179 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,179 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,179 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,179 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,180 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,180 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,180 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,180 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,180 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,180 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,181 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,181 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,181 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,181 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,181 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,182 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,182 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,182 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,182 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,182 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,183 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,183 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,183 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,183 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,183 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,183 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,184 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,184 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,184 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,184 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,184 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,184 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,185 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,185 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,185 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,185 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,185 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,186 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,186 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,186 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,186 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,186 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,186 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,187 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,187 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,187 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,187 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,187 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,188 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,188 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,188 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,188 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,188 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,189 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,189 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,189 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,189 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,189 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,190 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,190 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,190 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,190 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,190 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,190 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,191 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,191 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,191 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,191 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,191 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,192 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,192 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,192 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,192 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,192 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,192 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,193 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,193 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,193 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,193 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,193 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,194 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,194 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,194 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,194 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,194 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,195 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,195 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,195 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,195 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,195 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,195 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,196 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,196 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,196 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,196 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,196 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,196 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,197 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,197 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,197 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,197 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,197 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,198 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,198 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,198 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,198 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,198 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,199 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,199 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,199 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,199 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,199 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,200 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,200 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,200 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,200 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,200 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,200 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,201 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,201 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,201 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,201 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,201 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,201 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,202 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,202 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,202 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,202 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,203 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,203 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,203 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,203 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,203 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,204 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,204 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,204 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,204 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,204 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,204 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,205 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,205 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,205 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,205 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,205 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,205 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,206 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,206 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,206 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,207 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,207 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,207 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,207 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,207 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,208 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,208 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,208 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,208 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,209 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,209 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,209 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,209 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,209 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,209 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,210 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,210 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,210 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,210 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,210 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,210 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,211 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,211 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,211 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,211 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,211 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,211 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,212 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,212 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,212 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,212 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,212 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,213 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,213 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,213 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,213 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,213 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,214 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,214 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,214 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,214 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,214 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,214 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,215 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,215 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,215 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,215 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,215 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,216 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,216 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,216 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,216 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,216 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,216 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,217 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,217 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,217 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,217 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,217 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,217 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,218 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,218 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,218 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,218 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,218 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,219 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,219 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,219 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,219 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,219 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,220 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,220 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,220 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,220 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,220 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,221 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,221 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,221 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,221 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,221 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,221 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,222 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,222 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,222 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,222 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,222 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,222 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,223 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,223 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,223 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,223 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,223 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,223 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,224 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,224 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,224 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,224 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,224 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,225 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,225 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,225 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,225 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,225 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,226 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,226 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,226 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,226 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,226 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,226 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,227 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,227 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,227 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,227 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,227 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,228 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,228 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,228 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,228 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,228 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,228 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,229 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,229 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,229 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,229 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,229 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,229 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,230 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,230 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,230 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,230 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,230 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,231 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,231 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,231 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,231 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,231 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,232 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,232 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,232 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,232 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,232 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,233 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,233 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,233 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,233 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,233 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,233 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,234 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,234 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,234 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,234 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,234 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,234 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,235 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,235 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,235 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,235 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,235 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,235 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,236 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,236 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,236 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,236 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,237 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,237 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,237 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,237 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,237 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,238 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,238 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,238 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,238 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,238 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,238 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,239 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,239 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,239 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,239 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,239 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,240 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,240 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,240 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,240 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,240 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,241 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,241 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,241 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,241 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,241 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,242 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,242 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,242 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,242 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,242 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,242 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,243 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,243 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,243 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,243 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,243 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,243 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,244 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,244 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,244 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,244 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,244 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,245 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,245 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,245 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,245 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,245 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,246 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,246 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,246 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,246 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,246 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,247 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,247 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,247 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,247 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,247 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,247 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,248 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,248 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:44,248 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:44,248 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:44,832 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/wandb-summary.json +2021-07-09 12:28:44,832 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log +2021-07-09 12:28:46,827 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log +2021-07-09 12:28:48,205 DEBUG SenderThread:33395 [sender.py:send():179] send: history +2021-07-09 12:28:48,206 DEBUG SenderThread:33395 [sender.py:send():179] send: summary +2021-07-09 12:28:48,208 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:28:48,828 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/wandb-summary.json +2021-07-09 12:28:48,829 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log +2021-07-09 12:28:50,492 DEBUG HandlerThread:33395 [handler.py:handle_request():124] handle_request: shutdown +2021-07-09 12:28:50,492 INFO HandlerThread:33395 [handler.py:finish():638] shutting down handler +2021-07-09 12:28:50,829 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log +2021-07-09 12:28:51,491 INFO SenderThread:33395 [sender.py:finish():945] shutting down sender +2021-07-09 12:28:51,492 INFO WriterThread:33395 [datastore.py:close():288] close: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/run-2pc90wcd.wandb +2021-07-09 12:28:51,492 INFO SenderThread:33395 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-09 12:28:51,830 INFO SenderThread:33395 [dir_watcher.py:finish():312] scan: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files +2021-07-09 12:28:51,830 INFO SenderThread:33395 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/requirements.txt requirements.txt +2021-07-09 12:28:51,830 INFO SenderThread:33395 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log output.log +2021-07-09 12:28:51,831 INFO SenderThread:33395 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/wandb-metadata.json wandb-metadata.json +2021-07-09 12:28:51,831 INFO SenderThread:33395 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/config.yaml config.yaml +2021-07-09 12:28:51,831 INFO SenderThread:33395 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/wandb-summary.json wandb-summary.json +2021-07-09 12:28:51,831 INFO SenderThread:33395 [file_pusher.py:finish():177] shutting down file pusher +2021-07-09 12:28:51,831 INFO SenderThread:33395 [file_pusher.py:join():182] waiting for file pusher +2021-07-09 12:28:52,029 INFO Thread-14 :33395 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/config.yaml +2021-07-09 12:28:52,038 INFO Thread-15 :33395 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/wandb-summary.json +2021-07-09 12:28:52,039 INFO Thread-12 :33395 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/requirements.txt +2021-07-09 12:28:52,064 INFO Thread-13 :33395 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log +2021-07-09 12:28:52,398 INFO MainThread:33395 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210709_122709-2pc90wcd/logs/debug.log b/wandb/run-20210709_122709-2pc90wcd/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0e7c0d4633f788492ef1369797462bf162f7d09c --- /dev/null +++ b/wandb/run-20210709_122709-2pc90wcd/logs/debug.log @@ -0,0 +1,53 @@ +2021-07-09 12:27:09,053 INFO MainThread:32339 [wandb_setup.py:_flush():69] Unhandled environment var: WANDB_WATCH +2021-07-09 12:27:09,053 INFO MainThread:32339 [wandb_setup.py:_flush():69] setting env: {'project': 'hf-flax-robert-base-mr', 'entity': 'nipunsadvilkar'} +2021-07-09 12:27:09,053 INFO MainThread:32339 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-09 12:27:09,053 INFO MainThread:32339 [wandb_init.py:_log_setup():337] Logging user logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/logs/debug.log +2021-07-09 12:27:09,053 INFO MainThread:32339 [wandb_init.py:_log_setup():338] Logging internal logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/logs/debug-internal.log +2021-07-09 12:27:09,053 INFO MainThread:32339 [wandb_init.py:init():370] calling init triggers +2021-07-09 12:27:09,053 INFO MainThread:32339 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-09 12:27:09,053 INFO MainThread:32339 [wandb_init.py:init():419] starting backend +2021-07-09 12:27:09,053 INFO MainThread:32339 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-09 12:27:09,087 INFO MainThread:32339 [backend.py:ensure_launched():135] starting backend process... +2021-07-09 12:27:09,119 INFO MainThread:32339 [backend.py:ensure_launched():139] started backend process with pid: 33395 +2021-07-09 12:27:09,122 INFO MainThread:32339 [wandb_init.py:init():424] backend started and connected +2021-07-09 12:27:09,124 INFO MainThread:32339 [wandb_init.py:init():472] updated telemetry +2021-07-09 12:27:09,125 INFO MainThread:32339 [wandb_init.py:init():491] communicating current version +2021-07-09 12:27:09,709 INFO MainThread:32339 [wandb_init.py:init():496] got version response +2021-07-09 12:27:09,709 INFO MainThread:32339 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-09 12:27:09,783 INFO MainThread:32339 [wandb_init.py:init():529] starting run threads in backend +2021-07-09 12:27:09,838 INFO MainThread:32339 [wandb_run.py:_console_start():1623] atexit reg +2021-07-09 12:27:09,839 INFO MainThread:32339 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-09 12:27:09,839 INFO MainThread:32339 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-09 12:27:09,841 INFO MainThread:32339 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-09 12:27:09,841 INFO MainThread:32339 [wandb_init.py:init():554] run started, returning control to user process +2021-07-09 12:27:09,842 INFO MainThread:32339 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 4, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 0.0003, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 8.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul09_12-27-03_t1v-n-112df4a9-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 500, 'save_total_limit': None, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-09 12:27:09,843 INFO MainThread:32339 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'roberta', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'} +2021-07-09 12:27:09,844 INFO MainThread:32339 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': 'oscar', 'dataset_config_name': 'unshuffled_deduplicated_als', 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 128, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False} +2021-07-09 12:28:48,239 INFO MainThread:32339 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1 +2021-07-09 12:28:48,240 INFO MainThread:32339 [wandb_run.py:_restore():1565] restore +2021-07-09 12:28:52,518 ERROR MainThread:32339 [wandb_run.py:_atexit_cleanup():1611] Problem finishing run +Traceback (most recent call last): + File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/wandb_run.py", line 1601, in _atexit_cleanup + self._on_finish() + File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/wandb_run.py", line 1737, in _on_finish + self.history._flush() + File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/wandb_history.py", line 59, in _flush + self._callback(row=self._data, step=self._step) + File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/wandb_run.py", line 931, in _history_callback + self._backend.interface.publish_history( + File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 231, in publish_history + item.value_json = json_dumps_safer_history(v) # type: ignore + File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/util.py", line 667, in json_dumps_safer_history + return json.dumps(obj, cls=WandBHistoryJSONEncoder, **kwargs) + File "/usr/lib/python3.8/json/__init__.py", line 234, in dumps + return cls( + File "/usr/lib/python3.8/json/encoder.py", line 199, in encode + chunks = self.iterencode(o, _one_shot=True) + File "/usr/lib/python3.8/json/encoder.py", line 257, in iterencode + return _iterencode(o, 0) + File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/util.py", line 634, in default + return json.JSONEncoder.default(self, obj) + File "/usr/lib/python3.8/json/encoder.py", line 179, in default + raise TypeError(f'Object of type {o.__class__.__name__} ' +TypeError: Object of type DeviceArray is not JSON serializable diff --git a/wandb/run-20210709_122709-2pc90wcd/run-2pc90wcd.wandb b/wandb/run-20210709_122709-2pc90wcd/run-2pc90wcd.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b6fd6dbce5ac3f1398c13f8358db8df85cf51a35 Binary files /dev/null and b/wandb/run-20210709_122709-2pc90wcd/run-2pc90wcd.wandb differ diff --git a/wandb/run-20210709_124353-3pjvcule/files/config.yaml b/wandb/run-20210709_124353-3pjvcule/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab3d37a013838fc169594a1929bc84ac6663fbb1 --- /dev/null +++ b/wandb/run-20210709_124353-3pjvcule/files/config.yaml @@ -0,0 +1,19 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + cli_version: 0.10.33 + framework: huggingface + huggingface_version: 4.9.0.dev0 + is_jupyter_run: false + is_kaggle_kernel: false + python_version: 3.8.10 + t: + 1: + - 11 + 4: 3.8.10 + 5: 0.10.33 + 6: 4.9.0.dev0 + 8: + - 5 diff --git a/wandb/run-20210709_124353-3pjvcule/files/output.log b/wandb/run-20210709_124353-3pjvcule/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..67fb84fd3ef63d3314f3818ae5b2ed6874545251 --- /dev/null +++ b/wandb/run-20210709_124353-3pjvcule/files/output.log @@ -0,0 +1,5 @@ +/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. + warnings.warn( +/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. + warnings.warn( +Epoch ... (1/8): 0%| | 0/8 [00:00 /home/nipunsadvilkar/sample/roberta-base-mr/run_mlm_flax.py(647)() + 646 import ipdb; ipdb.set_trace() +--> 647 train_metrics = [] + 648 + + +ipdb> train_metrics +[{'learning_rate': ShardedDeviceArray([0.0001278, 0.0001278, 0.0001278, 0.0001278, 0.0001278, + 0.0001278, 0.0001278, 0.0001278], dtype=float32), 'loss': ShardedDeviceArray([7.9980755, 7.9980755, 7.9980755, 7.9980755, 7.9980755, + 7.9980755, 7.9980755, 7.9980755], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001281, 0.0001281, 0.0001281, 0.0001281, 0.0001281, + 0.0001281, 0.0001281, 0.0001281], dtype=float32), 'loss': ShardedDeviceArray([7.7923646, 7.7923646, 7.7923646, 7.7923646, 7.7923646, + 7.7923646, 7.7923646, 7.7923646], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001284, 0.0001284, 0.0001284, 0.0001284, 0.0001284, + 0.0001284, 0.0001284, 0.0001284], dtype=float32), 'loss': ShardedDeviceArray([7.851894, 7.851894, 7.851894, 7.851894, 7.851894, + 7.851894, 7.851894, 7.851894], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001287, 0.0001287, 0.0001287, 0.0001287, 0.0001287, + 0.0001287, 0.0001287, 0.0001287], dtype=float32), 'loss': ShardedDeviceArray([7.9031653, 7.9031653, 7.9031653, 7.9031653, 7.9031653, + 7.9031653, 7.9031653, 7.9031653], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.000129, 0.000129, 0.000129, 0.000129, 0.000129, + 0.000129, 0.000129, 0.000129], dtype=float32), 'loss': ShardedDeviceArray([7.9961405, 7.9961405, 7.9961405, 7.9961405, 7.9961405, + 7.9961405, 7.9961405, 7.9961405], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001293, 0.0001293, 0.0001293, 0.0001293, 0.0001293, + 0.0001293, 0.0001293, 0.0001293], dtype=float32), 'loss': ShardedDeviceArray([7.9092264, 7.9092264, 7.9092264, 7.9092264, 7.9092264, + 7.9092264, 7.9092264, 7.9092264], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001296, 0.0001296, 0.0001296, 0.0001296, 0.0001296, + 0.0001296, 0.0001296, 0.0001296], dtype=float32), 'loss': ShardedDeviceArray([7.845132, 7.845132, 7.845132, 7.845132, 7.845132, + 7.845132, 7.845132, 7.845132], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001299, 0.0001299, 0.0001299, 0.0001299, 0.0001299, + 0.0001299, 0.0001299, 0.0001299], dtype=float32), 'loss': ShardedDeviceArray([7.8732557, 7.8732557, 7.8732557, 7.8732557, 7.8732557, + 7.8732557, 7.8732557, 7.8732557], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001302, 0.0001302, 0.0001302, 0.0001302, 0.0001302, + 0.0001302, 0.0001302, 0.0001302], dtype=float32), 'loss': ShardedDeviceArray([7.875514, 7.875514, 7.875514, 7.875514, 7.875514, + 7.875514, 7.875514, 7.875514], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001305, 0.0001305, 0.0001305, 0.0001305, 0.0001305, + 0.0001305, 0.0001305, 0.0001305], dtype=float32), 'loss': ShardedDeviceArray([7.7459865, 7.7459865, 7.7459865, 7.7459865, 7.7459865, + 7.7459865, 7.7459865, 7.7459865], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001308, 0.0001308, 0.0001308, 0.0001308, 0.0001308, + 0.0001308, 0.0001308, 0.0001308], dtype=float32), 'loss': ShardedDeviceArray([8.150806, 8.150806, 8.150806, 8.150806, 8.150806, + 8.150806, 8.150806, 8.150806], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001311, 0.0001311, 0.0001311, 0.0001311, 0.0001311, + 0.0001311, 0.0001311, 0.0001311], dtype=float32), 'loss': ShardedDeviceArray([7.893339, 7.893339, 7.893339, 7.893339, 7.893339, + 7.893339, 7.893339, 7.893339], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001314, 0.0001314, 0.0001314, 0.0001314, 0.0001314, + 0.0001314, 0.0001314, 0.0001314], dtype=float32), 'loss': ShardedDeviceArray([8.129753, 8.129753, 8.129753, 8.129753, 8.129753, + 8.129753, 8.129753, 8.129753], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001317, 0.0001317, 0.0001317, 0.0001317, 0.0001317, + 0.0001317, 0.0001317, 0.0001317], dtype=float32), 'loss': ShardedDeviceArray([7.964561, 7.964561, 7.964561, 7.964561, 7.964561, + 7.964561, 7.964561, 7.964561], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.000132, 0.000132, 0.000132, 0.000132, 0.000132, + 0.000132, 0.000132, 0.000132], dtype=float32), 'loss': ShardedDeviceArray([7.915361, 7.915361, 7.915361, 7.915361, 7.915361, + 7.915361, 7.915361, 7.915361], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001323, 0.0001323, 0.0001323, 0.0001323, 0.0001323, + 0.0001323, 0.0001323, 0.0001323], dtype=float32), 'loss': ShardedDeviceArray([8.008739, 8.008739, 8.008739, 8.008739, 8.008739, + 8.008739, 8.008739, 8.008739], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001326, 0.0001326, 0.0001326, 0.0001326, 0.0001326, + 0.0001326, 0.0001326, 0.0001326], dtype=float32), 'loss': ShardedDeviceArray([7.92212, 7.92212, 7.92212, 7.92212, 7.92212, 7.92212, + 7.92212, 7.92212], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001329, 0.0001329, 0.0001329, 0.0001329, 0.0001329, + 0.0001329, 0.0001329, 0.0001329], dtype=float32), 'loss': ShardedDeviceArray([7.9835634, 7.9835634, 7.9835634, 7.9835634, 7.9835634, + 7.9835634, 7.9835634, 7.9835634], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001332, 0.0001332, 0.0001332, 0.0001332, 0.0001332, + 0.0001332, 0.0001332, 0.0001332], dtype=float32), 'loss': ShardedDeviceArray([7.877103, 7.877103, 7.877103, 7.877103, 7.877103, + 7.877103, 7.877103, 7.877103], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001335, 0.0001335, 0.0001335, 0.0001335, 0.0001335, + 0.0001335, 0.0001335, 0.0001335], dtype=float32), 'loss': ShardedDeviceArray([8.125843, 8.125843, 8.125843, 8.125843, 8.125843, + 8.125843, 8.125843, 8.125843], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001338, 0.0001338, 0.0001338, 0.0001338, 0.0001338, + 0.0001338, 0.0001338, 0.0001338], dtype=float32), 'loss': ShardedDeviceArray([7.8731213, 7.8731213, 7.8731213, 7.8731213, 7.8731213, + 7.8731213, 7.8731213, 7.8731213], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001341, 0.0001341, 0.0001341, 0.0001341, 0.0001341, + 0.0001341, 0.0001341, 0.0001341], dtype=float32), 'loss': ShardedDeviceArray([7.9708652, 7.9708652, 7.9708652, 7.9708652, 7.9708652, + 7.9708652, 7.9708652, 7.9708652], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001344, 0.0001344, 0.0001344, 0.0001344, 0.0001344, + 0.0001344, 0.0001344, 0.0001344], dtype=float32), 'loss': ShardedDeviceArray([7.783667, 7.783667, 7.783667, 7.783667, 7.783667, + 7.783667, 7.783667, 7.783667], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001347, 0.0001347, 0.0001347, 0.0001347, 0.0001347, + 0.0001347, 0.0001347, 0.0001347], dtype=float32), 'loss': ShardedDeviceArray([7.688542, 7.688542, 7.688542, 7.688542, 7.688542, + 7.688542, 7.688542, 7.688542], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.000135, 0.000135, 0.000135, 0.000135, 0.000135, + 0.000135, 0.000135, 0.000135], dtype=float32), 'loss': ShardedDeviceArray([7.810261, 7.810261, 7.810261, 7.810261, 7.810261, + 7.810261, 7.810261, 7.810261], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001353, 0.0001353, 0.0001353, 0.0001353, 0.0001353, + 0.0001353, 0.0001353, 0.0001353], dtype=float32), 'loss': ShardedDeviceArray([7.789603, 7.789603, 7.789603, 7.789603, 7.789603, + 7.789603, 7.789603, 7.789603], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001356, 0.0001356, 0.0001356, 0.0001356, 0.0001356, + 0.0001356, 0.0001356, 0.0001356], dtype=float32), 'loss': ShardedDeviceArray([7.976757, 7.976757, 7.976757, 7.976757, 7.976757, + 7.976757, 7.976757, 7.976757], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001359, 0.0001359, 0.0001359, 0.0001359, 0.0001359, + 0.0001359, 0.0001359, 0.0001359], dtype=float32), 'loss': ShardedDeviceArray([8.04048, 8.04048, 8.04048, 8.04048, 8.04048, 8.04048, + 8.04048, 8.04048], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001362, 0.0001362, 0.0001362, 0.0001362, 0.0001362, + 0.0001362, 0.0001362, 0.0001362], dtype=float32), 'loss': ShardedDeviceArray([7.758219, 7.758219, 7.758219, 7.758219, 7.758219, + 7.758219, 7.758219, 7.758219], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001365, 0.0001365, 0.0001365, 0.0001365, 0.0001365, + 0.0001365, 0.0001365, 0.0001365], dtype=float32), 'loss': ShardedDeviceArray([8.146248, 8.146248, 8.146248, 8.146248, 8.146248, + 8.146248, 8.146248, 8.146248], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001368, 0.0001368, 0.0001368, 0.0001368, 0.0001368, + 0.0001368, 0.0001368, 0.0001368], dtype=float32), 'loss': ShardedDeviceArray([7.768535, 7.768535, 7.768535, 7.768535, 7.768535, + 7.768535, 7.768535, 7.768535], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001371, 0.0001371, 0.0001371, 0.0001371, 0.0001371, + 0.0001371, 0.0001371, 0.0001371], dtype=float32), 'loss': ShardedDeviceArray([7.689828, 7.689828, 7.689828, 7.689828, 7.689828, + 7.689828, 7.689828, 7.689828], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001374, 0.0001374, 0.0001374, 0.0001374, 0.0001374, + 0.0001374, 0.0001374, 0.0001374], dtype=float32), 'loss': ShardedDeviceArray([7.796749, 7.796749, 7.796749, 7.796749, 7.796749, + 7.796749, 7.796749, 7.796749], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001377, 0.0001377, 0.0001377, 0.0001377, 0.0001377, + 0.0001377, 0.0001377, 0.0001377], dtype=float32), 'loss': ShardedDeviceArray([8.086693, 8.086693, 8.086693, 8.086693, 8.086693, + 8.086693, 8.086693, 8.086693], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.000138, 0.000138, 0.000138, 0.000138, 0.000138, + 0.000138, 0.000138, 0.000138], dtype=float32), 'loss': ShardedDeviceArray([8.044639, 8.044639, 8.044639, 8.044639, 8.044639, + 8.044639, 8.044639, 8.044639], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001383, 0.0001383, 0.0001383, 0.0001383, 0.0001383, + 0.0001383, 0.0001383, 0.0001383], dtype=float32), 'loss': ShardedDeviceArray([7.971658, 7.971658, 7.971658, 7.971658, 7.971658, + 7.971658, 7.971658, 7.971658], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001386, 0.0001386, 0.0001386, 0.0001386, 0.0001386, + 0.0001386, 0.0001386, 0.0001386], dtype=float32), 'loss': ShardedDeviceArray([7.994483, 7.994483, 7.994483, 7.994483, 7.994483, + 7.994483, 7.994483, 7.994483], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001389, 0.0001389, 0.0001389, 0.0001389, 0.0001389, + 0.0001389, 0.0001389, 0.0001389], dtype=float32), 'loss': ShardedDeviceArray([8.119253, 8.119253, 8.119253, 8.119253, 8.119253, + 8.119253, 8.119253, 8.119253], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001392, 0.0001392, 0.0001392, 0.0001392, 0.0001392, + 0.0001392, 0.0001392, 0.0001392], dtype=float32), 'loss': ShardedDeviceArray([7.9448185, 7.9448185, 7.9448185, 7.9448185, 7.9448185, + 7.9448185, 7.9448185, 7.9448185], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001395, 0.0001395, 0.0001395, 0.0001395, 0.0001395, + 0.0001395, 0.0001395, 0.0001395], dtype=float32), 'loss': ShardedDeviceArray([8.125984, 8.125984, 8.125984, 8.125984, 8.125984, + 8.125984, 8.125984, 8.125984], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001398, 0.0001398, 0.0001398, 0.0001398, 0.0001398, + 0.0001398, 0.0001398, 0.0001398], dtype=float32), 'loss': ShardedDeviceArray([7.8050995, 7.8050995, 7.8050995, 7.8050995, 7.8050995, + 7.8050995, 7.8050995, 7.8050995], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001401, 0.0001401, 0.0001401, 0.0001401, 0.0001401, + 0.0001401, 0.0001401, 0.0001401], dtype=float32), 'loss': ShardedDeviceArray([7.962894, 7.962894, 7.962894, 7.962894, 7.962894, + 7.962894, 7.962894, 7.962894], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001404, 0.0001404, 0.0001404, 0.0001404, 0.0001404, + 0.0001404, 0.0001404, 0.0001404], dtype=float32), 'loss': ShardedDeviceArray([7.7137384, 7.7137384, 7.7137384, 7.7137384, 7.7137384, + 7.7137384, 7.7137384, 7.7137384], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001407, 0.0001407, 0.0001407, 0.0001407, 0.0001407, + 0.0001407, 0.0001407, 0.0001407], dtype=float32), 'loss': ShardedDeviceArray([7.7208614, 7.7208614, 7.7208614, 7.7208614, 7.7208614, + 7.7208614, 7.7208614, 7.7208614], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.000141, 0.000141, 0.000141, 0.000141, 0.000141, + 0.000141, 0.000141, 0.000141], dtype=float32), 'loss': ShardedDeviceArray([7.9821005, 7.9821005, 7.9821005, 7.9821005, 7.9821005, + 7.9821005, 7.9821005, 7.9821005], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001413, 0.0001413, 0.0001413, 0.0001413, 0.0001413, + 0.0001413, 0.0001413, 0.0001413], dtype=float32), 'loss': ShardedDeviceArray([7.8411956, 7.8411956, 7.8411956, 7.8411956, 7.8411956, + 7.8411956, 7.8411956, 7.8411956], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001416, 0.0001416, 0.0001416, 0.0001416, 0.0001416, + 0.0001416, 0.0001416, 0.0001416], dtype=float32), 'loss': ShardedDeviceArray([8.111786, 8.111786, 8.111786, 8.111786, 8.111786, + 8.111786, 8.111786, 8.111786], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001419, 0.0001419, 0.0001419, 0.0001419, 0.0001419, + 0.0001419, 0.0001419, 0.0001419], dtype=float32), 'loss': ShardedDeviceArray([7.899773, 7.899773, 7.899773, 7.899773, 7.899773, + 7.899773, 7.899773, 7.899773], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001422, 0.0001422, 0.0001422, 0.0001422, 0.0001422, + 0.0001422, 0.0001422, 0.0001422], dtype=float32), 'loss': ShardedDeviceArray([7.9648595, 7.9648595, 7.9648595, 7.9648595, 7.9648595, + 7.9648595, 7.9648595, 7.9648595], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001425, 0.0001425, 0.0001425, 0.0001425, 0.0001425, + 0.0001425, 0.0001425, 0.0001425], dtype=float32), 'loss': ShardedDeviceArray([7.791326, 7.791326, 7.791326, 7.791326, 7.791326, + 7.791326, 7.791326, 7.791326], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001428, 0.0001428, 0.0001428, 0.0001428, 0.0001428, + 0.0001428, 0.0001428, 0.0001428], dtype=float32), 'loss': ShardedDeviceArray([7.6367683, 7.6367683, 7.6367683, 7.6367683, 7.6367683, + 7.6367683, 7.6367683, 7.6367683], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001431, 0.0001431, 0.0001431, 0.0001431, 0.0001431, + 0.0001431, 0.0001431, 0.0001431], dtype=float32), 'loss': ShardedDeviceArray([8.109274, 8.109274, 8.109274, 8.109274, 8.109274, + 8.109274, 8.109274, 8.109274], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001434, 0.0001434, 0.0001434, 0.0001434, 0.0001434, + 0.0001434, 0.0001434, 0.0001434], dtype=float32), 'loss': ShardedDeviceArray([7.957328, 7.957328, 7.957328, 7.957328, 7.957328, + 7.957328, 7.957328, 7.957328], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001437, 0.0001437, 0.0001437, 0.0001437, 0.0001437, + 0.0001437, 0.0001437, 0.0001437], dtype=float32), 'loss': ShardedDeviceArray([8.124847, 8.124847, 8.124847, 8.124847, 8.124847, + 8.124847, 8.124847, 8.124847], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.000144, 0.000144, 0.000144, 0.000144, 0.000144, + 0.000144, 0.000144, 0.000144], dtype=float32), 'loss': ShardedDeviceArray([8.072102, 8.072102, 8.072102, 8.072102, 8.072102, + 8.072102, 8.072102, 8.072102], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001443, 0.0001443, 0.0001443, 0.0001443, 0.0001443, + 0.0001443, 0.0001443, 0.0001443], dtype=float32), 'loss': ShardedDeviceArray([8.054686, 8.054686, 8.054686, 8.054686, 8.054686, + 8.054686, 8.054686, 8.054686], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001446, 0.0001446, 0.0001446, 0.0001446, 0.0001446, + 0.0001446, 0.0001446, 0.0001446], dtype=float32), 'loss': ShardedDeviceArray([7.875123, 7.875123, 7.875123, 7.875123, 7.875123, + 7.875123, 7.875123, 7.875123], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001449, 0.0001449, 0.0001449, 0.0001449, 0.0001449, + 0.0001449, 0.0001449, 0.0001449], dtype=float32), 'loss': ShardedDeviceArray([7.6778483, 7.6778483, 7.6778483, 7.6778483, 7.6778483, + 7.6778483, 7.6778483, 7.6778483], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001452, 0.0001452, 0.0001452, 0.0001452, 0.0001452, + 0.0001452, 0.0001452, 0.0001452], dtype=float32), 'loss': ShardedDeviceArray([7.8097315, 7.8097315, 7.8097315, 7.8097315, 7.8097315, + 7.8097315, 7.8097315, 7.8097315], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001455, 0.0001455, 0.0001455, 0.0001455, 0.0001455, + 0.0001455, 0.0001455, 0.0001455], dtype=float32), 'loss': ShardedDeviceArray([7.7626286, 7.7626286, 7.7626286, 7.7626286, 7.7626286, + 7.7626286, 7.7626286, 7.7626286], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001458, 0.0001458, 0.0001458, 0.0001458, 0.0001458, + 0.0001458, 0.0001458, 0.0001458], dtype=float32), 'loss': ShardedDeviceArray([7.825801, 7.825801, 7.825801, 7.825801, 7.825801, + 7.825801, 7.825801, 7.825801], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001461, 0.0001461, 0.0001461, 0.0001461, 0.0001461, + 0.0001461, 0.0001461, 0.0001461], dtype=float32), 'loss': ShardedDeviceArray([7.911273, 7.911273, 7.911273, 7.911273, 7.911273, + 7.911273, 7.911273, 7.911273], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001464, 0.0001464, 0.0001464, 0.0001464, 0.0001464, + 0.0001464, 0.0001464, 0.0001464], dtype=float32), 'loss': ShardedDeviceArray([7.7781672, 7.7781672, 7.7781672, 7.7781672, 7.7781672, + 7.7781672, 7.7781672, 7.7781672], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001467, 0.0001467, 0.0001467, 0.0001467, 0.0001467, + 0.0001467, 0.0001467, 0.0001467], dtype=float32), 'loss': ShardedDeviceArray([7.740696, 7.740696, 7.740696, 7.740696, 7.740696, + 7.740696, 7.740696, 7.740696], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.000147, 0.000147, 0.000147, 0.000147, 0.000147, + 0.000147, 0.000147, 0.000147], dtype=float32), 'loss': ShardedDeviceArray([7.7913523, 7.7913523, 7.7913523, 7.7913523, 7.7913523, + 7.7913523, 7.7913523, 7.7913523], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001473, 0.0001473, 0.0001473, 0.0001473, 0.0001473, + 0.0001473, 0.0001473, 0.0001473], dtype=float32), 'loss': ShardedDeviceArray([8.044626, 8.044626, 8.044626, 8.044626, 8.044626, + 8.044626, 8.044626, 8.044626], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001476, 0.0001476, 0.0001476, 0.0001476, 0.0001476, + 0.0001476, 0.0001476, 0.0001476], dtype=float32), 'loss': ShardedDeviceArray([8.033394, 8.033394, 8.033394, 8.033394, 8.033394, + 8.033394, 8.033394, 8.033394], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001479, 0.0001479, 0.0001479, 0.0001479, 0.0001479, + 0.0001479, 0.0001479, 0.0001479], dtype=float32), 'loss': ShardedDeviceArray([7.95217, 7.95217, 7.95217, 7.95217, 7.95217, 7.95217, + 7.95217, 7.95217], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001482, 0.0001482, 0.0001482, 0.0001482, 0.0001482, + 0.0001482, 0.0001482, 0.0001482], dtype=float32), 'loss': ShardedDeviceArray([7.8844614, 7.8844614, 7.8844614, 7.8844614, 7.8844614, + 7.8844614, 7.8844614, 7.8844614], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001485, 0.0001485, 0.0001485, 0.0001485, 0.0001485, + 0.0001485, 0.0001485, 0.0001485], dtype=float32), 'loss': ShardedDeviceArray([8.034703, 8.034703, 8.034703, 8.034703, 8.034703, + 8.034703, 8.034703, 8.034703], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001488, 0.0001488, 0.0001488, 0.0001488, 0.0001488, + 0.0001488, 0.0001488, 0.0001488], dtype=float32), 'loss': ShardedDeviceArray([7.847809, 7.847809, 7.847809, 7.847809, 7.847809, + 7.847809, 7.847809, 7.847809], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001491, 0.0001491, 0.0001491, 0.0001491, 0.0001491, + 0.0001491, 0.0001491, 0.0001491], dtype=float32), 'loss': ShardedDeviceArray([7.815194, 7.815194, 7.815194, 7.815194, 7.815194, + 7.815194, 7.815194, 7.815194], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001494, 0.0001494, 0.0001494, 0.0001494, 0.0001494, + 0.0001494, 0.0001494, 0.0001494], dtype=float32), 'loss': ShardedDeviceArray([7.7018757, 7.7018757, 7.7018757, 7.7018757, 7.7018757, + 7.7018757, 7.7018757, 7.7018757], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001497, 0.0001497, 0.0001497, 0.0001497, 0.0001497, + 0.0001497, 0.0001497, 0.0001497], dtype=float32), 'loss': ShardedDeviceArray([7.874889, 7.874889, 7.874889, 7.874889, 7.874889, + 7.874889, 7.874889, 7.874889], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, + 0.00015, 0.00015], dtype=float32), 'loss': ShardedDeviceArray([8.018753, 8.018753, 8.018753, 8.018753, 8.018753, + 8.018753, 8.018753, 8.018753], dtype=float32)}] +ipdb> train_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +Evaluating ...: 0%| | 0/10 [00:00 tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics +ipdb> tytrain_metrics + +Step... (500 | Loss: 8.205772399902344, Acc: 0.0773010179400444): 38%|██████████████████████████████████████▎ | 3/8 [13:50<23:03, 276.78s/it] \ No newline at end of file diff --git a/wandb/run-20210709_125049-89spqu1d/files/requirements.txt b/wandb/run-20210709_125049-89spqu1d/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..58a6f2b0a62dcb843c1f379c734d6bbe7a841939 --- /dev/null +++ b/wandb/run-20210709_125049-89spqu1d/files/requirements.txt @@ -0,0 +1,98 @@ +absl-py==0.13.0 +aiohttp==3.7.4.post0 +async-timeout==3.0.1 +attrs==21.2.0 +backcall==0.2.0 +cachetools==4.2.2 +certifi==2021.5.30 +chardet==4.0.0 +chex==0.0.8 +click==8.0.1 +configparser==5.0.2 +cycler==0.10.0 +datasets==1.9.1.dev0 +decorator==5.0.9 +dill==0.3.4 +dm-tree==0.1.6 +docker-pycreds==0.4.0 +filelock==3.0.12 +flatbuffers==2.0 +flax==0.3.4 +fsspec==2021.6.1 +gitdb==4.0.7 +gitpython==3.1.18 +google-auth-oauthlib==0.4.4 +google-auth==1.32.1 +grpcio==1.38.1 +huggingface-hub==0.0.12 +idna==2.10 +ipdb==0.13.9 +ipython-genutils==0.2.0 +ipython==7.25.0 +jax==0.2.16 +jaxlib==0.1.68 +jedi==0.18.0 +joblib==1.0.1 +kiwisolver==1.3.1 +libtpu-nightly==0.1.dev20210615 +markdown==3.3.4 +matplotlib-inline==0.1.2 +matplotlib==3.4.2 +msgpack==1.0.2 +multidict==5.1.0 +multiprocess==0.70.12.2 +numpy==1.21.0 +oauthlib==3.1.1 +opt-einsum==3.3.0 +optax==0.0.9 +packaging==21.0 +pandas==1.3.0 +parso==0.8.2 +pathtools==0.1.2 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==8.3.1 +pip==20.0.2 +pkg-resources==0.0.0 +promise==2.3 +prompt-toolkit==3.0.19 +protobuf==3.17.3 +psutil==5.8.0 +ptyprocess==0.7.0 +pyarrow==4.0.1 +pyasn1-modules==0.2.8 +pyasn1==0.4.8 +pygments==2.9.0 +pyparsing==2.4.7 +python-dateutil==2.8.1 +pytz==2021.1 +pyyaml==5.4.1 +regex==2021.7.6 +requests-oauthlib==1.3.0 +requests==2.25.1 +rsa==4.7.2 +sacremoses==0.0.45 +scipy==1.7.0 +sentry-sdk==1.3.0 +setuptools==44.0.0 +shortuuid==1.0.1 +six==1.16.0 +smmap==4.0.0 +subprocess32==3.5.4 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.0 +tensorboard==2.5.0 +tokenizers==0.10.3 +toml==0.10.2 +toolz==0.11.1 +tqdm==4.61.2 +traitlets==5.0.5 +transformers==4.9.0.dev0 +typing-extensions==3.10.0.0 +urllib3==1.26.6 +wandb==0.10.33 +wcwidth==0.2.5 +werkzeug==2.0.1 +wheel==0.36.2 +xxhash==2.0.2 +yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210709_125049-89spqu1d/files/wandb-metadata.json b/wandb/run-20210709_125049-89spqu1d/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..343e746eb949e4a003723a56039d89462a2413d7 --- /dev/null +++ b/wandb/run-20210709_125049-89spqu1d/files/wandb-metadata.json @@ -0,0 +1,36 @@ +{ + "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", + "python": "3.8.10", + "heartbeatAt": "2021-07-09T12:50:50.129806", + "startedAt": "2021-07-09T12:50:49.337388", + "docker": null, + "cpu_count": 96, + "cuda": null, + "args": [ + "--output_dir=./", + "--model_type=roberta", + "--config_name=./", + "--tokenizer_name=./", + "--dataset_name=oscar", + "--dataset_config_name=unshuffled_deduplicated_als", + "--max_seq_length=128", + "--per_device_train_batch_size=4", + "--per_device_eval_batch_size=4", + "--learning_rate=3e-4", + "--warmup_steps=1000", + "--overwrite_output_dir", + "--num_train_epochs=8" + ], + "state": "running", + "program": "run_mlm_flax.py", + "codePath": "run_mlm_flax.py", + "git": { + "remote": "https://huggingface.co/flax-community/roberta-base-mr", + "commit": "faa64e8a79301832a56c78b2b293675039070bb0" + }, + "email": "nipunsadvilkar@gmail.com", + "root": "/home/nipunsadvilkar/sample/roberta-base-mr", + "host": "t1v-n-112df4a9-w-0", + "username": "nipunsadvilkar", + "executable": "/home/nipunsadvilkar/roberta_mr_env/bin/python" +} diff --git a/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json b/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..de417d3d096fef51acb6d8e33bdf6ebbe65a223f --- /dev/null +++ b/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json @@ -0,0 +1 @@ +{"train_time": 2.8953051567077637, "_runtime": 96, "_timestamp": 1625835145, "_step": 302, "training_learning_rate": {"_type": "histogram", "values": [3, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 3], "bins": [0.00012780001270584762, 0.00012849376071244478, 0.00012918750871904194, 0.0001298812567256391, 0.00013057500473223627, 0.00013126876729074866, 0.00013196251529734582, 0.00013265626330394298, 0.00013335001131054014, 0.0001340437593171373, 0.00013473750732373446, 0.00013543125533033162, 0.000136125017888844, 0.00013681876589544117, 0.00013751251390203834, 0.0001382062619086355, 0.00013890000991523266, 0.00013959375792182982, 0.00014028750592842698, 0.00014098125393502414, 0.0001416750019416213, 0.0001423687645001337, 0.00014306251250673085, 0.00014375626051332802, 0.00014445000851992518, 0.00014514375652652234, 0.0001458375045331195, 0.00014653125253971666, 0.00014722501509822905, 0.0001479187631048262, 0.00014861251111142337, 0.00014930625911802053, 0.0001500000071246177]}, "train_step": 500, "train_learning_rate": 0.0001500000071246177, "training_loss": {"_type": "histogram", "values": [1, 0, 1, 2, 2, 1, 2, 2, 2, 6, 3, 2, 2, 2, 6, 2, 3, 3, 0, 3, 5, 3, 3, 2, 2, 3, 1, 1, 1, 2, 5, 2], "bins": [7.636768341064453, 7.65283203125, 7.668895721435547, 7.684959411621094, 7.701023101806641, 7.7170867919921875, 7.733150482177734, 7.749214172363281, 7.765277862548828, 7.781341552734375, 7.797405242919922, 7.813468933105469, 7.829532623291016, 7.8455963134765625, 7.861660003662109, 7.877723693847656, 7.893787384033203, 7.90985107421875, 7.925914764404297, 7.941978454589844, 7.958042144775391, 7.9741058349609375, 7.990169525146484, 8.006233215332031, 8.022296905517578, 8.038360595703125, 8.054424285888672, 8.070487976074219, 8.086551666259766, 8.102615356445312, 8.11867904663086, 8.134742736816406, 8.150806427001953]}, "train_loss": 8.018753051757812} \ No newline at end of file diff --git a/wandb/run-20210709_125049-89spqu1d/logs/debug-internal.log b/wandb/run-20210709_125049-89spqu1d/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..fd36e5980d25ac99cbdc8bf1ad372c41f372b929 --- /dev/null +++ b/wandb/run-20210709_125049-89spqu1d/logs/debug-internal.log @@ -0,0 +1,1240 @@ +2021-07-09 12:50:49,995 INFO MainThread:37232 [internal.py:wandb_internal():88] W&B internal server running at pid: 37232, started at: 2021-07-09 12:50:49.995101 +2021-07-09 12:50:49,997 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: check_version +2021-07-09 12:50:49,997 INFO WriterThread:37232 [datastore.py:open_for_write():80] open: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/run-89spqu1d.wandb +2021-07-09 12:50:49,999 DEBUG SenderThread:37232 [sender.py:send():179] send: header +2021-07-09 12:50:49,999 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: check_version +2021-07-09 12:50:50,064 DEBUG SenderThread:37232 [sender.py:send():179] send: run +2021-07-09 12:50:50,126 INFO SenderThread:37232 [dir_watcher.py:__init__():168] watching files in: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files +2021-07-09 12:50:50,126 INFO SenderThread:37232 [sender.py:_start_run_threads():716] run started: 89spqu1d with start time 1625835049 +2021-07-09 12:50:50,126 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:50:50,126 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:50:50,128 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: run_start +2021-07-09 12:50:50,129 DEBUG HandlerThread:37232 [meta.py:__init__():39] meta init +2021-07-09 12:50:50,129 DEBUG HandlerThread:37232 [meta.py:__init__():53] meta init done +2021-07-09 12:50:50,129 DEBUG HandlerThread:37232 [meta.py:probe():210] probe +2021-07-09 12:50:50,131 DEBUG HandlerThread:37232 [meta.py:_setup_git():200] setup git +2021-07-09 12:50:50,151 DEBUG HandlerThread:37232 [meta.py:_setup_git():207] setup git done +2021-07-09 12:50:50,151 DEBUG HandlerThread:37232 [meta.py:_save_pip():57] save pip +2021-07-09 12:50:50,152 DEBUG HandlerThread:37232 [meta.py:_save_pip():71] save pip done +2021-07-09 12:50:50,152 DEBUG HandlerThread:37232 [meta.py:probe():252] probe done +2021-07-09 12:50:50,183 DEBUG SenderThread:37232 [sender.py:send():179] send: files +2021-07-09 12:50:50,184 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-09 12:50:50,188 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:50:50,188 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:50:50,216 DEBUG SenderThread:37232 [sender.py:send():179] send: config +2021-07-09 12:50:50,217 DEBUG SenderThread:37232 [sender.py:send():179] send: config +2021-07-09 12:50:50,217 DEBUG SenderThread:37232 [sender.py:send():179] send: config +2021-07-09 12:50:50,392 INFO Thread-11 :37232 [upload_job.py:push():137] Uploaded file /tmp/tmpicd15h30wandb/9d69te0o-wandb-metadata.json +2021-07-09 12:50:51,127 INFO Thread-8 :37232 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json +2021-07-09 12:50:51,127 INFO Thread-8 :37232 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:50:51,128 INFO Thread-8 :37232 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/requirements.txt +2021-07-09 12:50:51,128 INFO Thread-8 :37232 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/wandb-metadata.json +2021-07-09 12:50:57,128 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:50:59,129 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:51:05,235 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:51:05,235 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:51:18,211 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:51:20,275 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:51:20,285 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:51:21,139 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/config.yaml +2021-07-09 12:51:35,312 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:51:35,313 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:51:48,297 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:51:50,344 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:51:50,344 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:52:05,424 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:52:05,424 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:52:11,160 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:52:13,161 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:52:15,161 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:52:17,162 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:52:18,380 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:52:19,163 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:52:20,457 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:52:20,457 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:52:21,164 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:52:23,164 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:52:25,165 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:52:25,427 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,428 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,428 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,863 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,864 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,868 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,869 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,869 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,875 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,875 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,875 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,880 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,880 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,880 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,886 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,887 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,887 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,889 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,889 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,889 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,893 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,893 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,893 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,896 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,896 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,896 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,901 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,901 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,901 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,904 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,904 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,904 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,910 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,910 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,910 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,914 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,915 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,915 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,916 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,916 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,917 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,920 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,920 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,920 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,925 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,925 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,925 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,927 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,927 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,927 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,930 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,930 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,930 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,933 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,934 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,934 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,935 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,935 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,936 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,938 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,938 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,938 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,941 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,942 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,942 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,947 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,948 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,948 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,951 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,951 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,951 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,956 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,956 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,957 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,961 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,962 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,962 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,965 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,966 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,966 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,968 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,968 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,968 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,977 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,979 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,979 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,988 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,989 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,989 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,991 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,991 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,991 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:25,997 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:25,997 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:25,997 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,007 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,011 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,011 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,012 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,012 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,012 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,013 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,013 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,013 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,013 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,013 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,013 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,014 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,014 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,014 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,015 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,015 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,015 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,015 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,015 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,016 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,016 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,016 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,016 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,017 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,017 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,017 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,017 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,017 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,018 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,019 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,019 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,019 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,020 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,020 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,020 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,021 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,021 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,021 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,021 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,022 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,022 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,022 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,022 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,022 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,023 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,023 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,023 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,023 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,023 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,023 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,024 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,024 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,024 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,024 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,024 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,025 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,025 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,025 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,025 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,026 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,026 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,026 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,026 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,027 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,027 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,027 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,027 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,027 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,028 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,028 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,028 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,028 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,029 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,029 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,029 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,029 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,029 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,029 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,030 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,030 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,030 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,030 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,030 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,031 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,031 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,031 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,031 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,031 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,031 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,032 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,032 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,032 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,032 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,032 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,032 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,033 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,033 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,033 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,034 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,034 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,034 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,034 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,034 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,034 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,035 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,035 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,035 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,035 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,036 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,036 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,036 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,036 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,036 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,037 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,037 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,037 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,037 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,037 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,037 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,038 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,038 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,038 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,038 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,038 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,039 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,039 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,039 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,039 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,039 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,040 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,040 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,040 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,040 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,040 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,041 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,041 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,041 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,041 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,041 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,041 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,042 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,042 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,042 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,042 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,042 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,042 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,043 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,043 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,043 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,043 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,043 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,043 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,044 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,044 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,044 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,045 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,045 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,045 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,045 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,045 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,045 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,046 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,046 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,046 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,046 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,047 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,047 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,047 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,047 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,047 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,048 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,048 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,048 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,048 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,048 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,048 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,049 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,049 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,049 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,049 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,049 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,049 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,050 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,050 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,050 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,050 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,050 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,050 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,051 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,051 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,051 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,051 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,052 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,052 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,052 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,052 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,052 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,052 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,053 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,053 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,053 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,053 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,053 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,053 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,054 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,054 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,054 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,054 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,054 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,055 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,055 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,055 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,055 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,055 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,055 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,056 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,056 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,056 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,056 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,056 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,056 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,057 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,057 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,057 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,058 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,058 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,058 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,058 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,058 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,058 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,059 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,059 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,059 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,059 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,060 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,060 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,060 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,060 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,060 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,060 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,061 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,061 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,061 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,061 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,061 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,062 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,062 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,062 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,062 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,062 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,062 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,063 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,063 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,063 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,063 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,063 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,063 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,064 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,064 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,064 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,064 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,065 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,065 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,065 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,065 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,065 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,066 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,066 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,066 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,066 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,066 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,066 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,067 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,067 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,067 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,067 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,067 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,067 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,068 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,068 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,068 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,068 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,069 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,069 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,069 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,069 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,069 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,069 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,070 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,070 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,070 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,070 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,070 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,071 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,071 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,071 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,071 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,071 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,071 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,072 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,072 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,072 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,072 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,072 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,073 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,073 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,073 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,073 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,074 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,074 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,074 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,074 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,074 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,074 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,075 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,075 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,075 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,075 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,075 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,076 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,076 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,076 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,076 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,076 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,077 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,077 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,077 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,077 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,077 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,078 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,078 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,078 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,078 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,078 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,078 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,079 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,079 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,079 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,079 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,079 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,079 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,080 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,080 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,080 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,080 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,080 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,081 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,081 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,081 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,081 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,081 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,082 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,082 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,082 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,082 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,082 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,082 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,083 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,083 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,083 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,083 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,083 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,084 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,084 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,084 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,084 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,084 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,084 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,085 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,085 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,085 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,085 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,085 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,085 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,086 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,086 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,086 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,086 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,087 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,087 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,087 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,087 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,087 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,088 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,088 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,088 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,088 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,088 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,088 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,089 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,089 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,089 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,089 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,089 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,090 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,090 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,090 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,090 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,091 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,091 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,091 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,091 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,091 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,091 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,092 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,092 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,092 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,092 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,092 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,092 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,093 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,093 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,093 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,093 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,093 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,093 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,094 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,094 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,094 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,094 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,094 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,095 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,095 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,095 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,095 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,095 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,096 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,096 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,096 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,096 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,096 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,097 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,097 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,097 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,097 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,097 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,097 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,098 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,098 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,098 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,098 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,098 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,099 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,099 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,099 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,099 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,099 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,100 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,100 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,100 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,100 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,100 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,101 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,101 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,101 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,101 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,101 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,101 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,102 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,102 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,102 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,102 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,102 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,102 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,103 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,103 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,103 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,103 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,104 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,104 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,104 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,104 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,104 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,105 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,105 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,105 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,105 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,105 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,105 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,106 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,106 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,106 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,106 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,106 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,107 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,107 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,107 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,107 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,107 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,108 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,108 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,108 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,108 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,108 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,109 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,109 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,109 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,109 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,109 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,109 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,110 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,110 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,110 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,110 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,110 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,110 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,111 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,111 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,111 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,111 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,112 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,112 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,112 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,112 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,112 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,113 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,113 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,113 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,113 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,113 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,113 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,114 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,114 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,114 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,114 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,114 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,114 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,115 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,115 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,115 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,115 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,115 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,116 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,116 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,116 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,116 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,117 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,117 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,117 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,117 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,117 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,117 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,118 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,118 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,118 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,118 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,119 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,119 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,119 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,119 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,119 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,120 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,120 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,120 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,120 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,120 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,120 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,121 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,121 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,121 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,121 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,121 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,122 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,122 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,122 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,122 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,122 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,122 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,123 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,123 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,123 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,123 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,124 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,124 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,124 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,124 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,124 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,124 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,125 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,125 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,125 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,125 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,125 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,125 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,126 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,126 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,126 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,126 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,126 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,127 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,127 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,127 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,127 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,127 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,128 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,128 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,128 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,128 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,128 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,129 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,129 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,129 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,129 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,129 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,129 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,130 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,130 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,130 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,130 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,130 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,131 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,131 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,131 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,131 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,131 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,132 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,132 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,132 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,132 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,132 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,133 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,133 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,133 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,133 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,133 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,133 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,134 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,134 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,134 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,134 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,134 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,134 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,135 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,135 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,135 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,135 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,135 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,136 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,136 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,136 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,136 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,137 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,137 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,137 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,137 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,137 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,137 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,138 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,138 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,138 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,138 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,139 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,139 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,139 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,139 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,139 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,140 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,140 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,140 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,140 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,140 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,140 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,141 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,141 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,141 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,141 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,141 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,141 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,142 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,142 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,142 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,142 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,142 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,143 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,143 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,143 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,143 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,143 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,144 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,144 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,144 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,144 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,144 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,145 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,145 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,145 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,145 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,145 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,145 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,146 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,146 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,146 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,146 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,146 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,147 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,147 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,147 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,147 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,147 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,148 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,148 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,148 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,148 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,148 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,149 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,149 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,149 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,149 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,149 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,149 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,150 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,150 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,150 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,150 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,150 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,150 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,151 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,151 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,151 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,151 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,151 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,151 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,152 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,152 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,152 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,152 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,152 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,153 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,153 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,153 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,153 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,153 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,154 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,154 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,154 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,154 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,154 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,155 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,155 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,155 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,155 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,155 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,155 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,156 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,156 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,156 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,156 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,156 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,157 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,157 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,157 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,157 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,158 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,158 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,158 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,158 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,158 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,158 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,159 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,159 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,159 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,159 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,159 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,160 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,160 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,160 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,160 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,160 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,161 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,161 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,161 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,161 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,161 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,162 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,162 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,162 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,162 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,162 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,162 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,163 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,163 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,163 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,163 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,163 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,164 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,164 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,164 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,164 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,164 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,165 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,165 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,165 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,165 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,165 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,166 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,166 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,166 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,167 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json +2021-07-09 12:52:26,167 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,167 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,167 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,167 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,168 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,168 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,168 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,168 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,168 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,169 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,169 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,169 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,169 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:26,169 DEBUG SenderThread:37232 [sender.py:send():179] send: history +2021-07-09 12:52:26,169 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 12:52:26,170 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 12:52:27,167 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json +2021-07-09 12:52:27,167 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:52:35,492 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:52:35,493 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:52:47,174 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:52:48,463 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:52:50,524 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:52:50,524 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:53:05,552 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:53:05,552 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:53:18,538 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:53:20,604 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:53:20,604 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:53:35,634 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:53:35,634 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:53:48,616 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:53:50,666 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:53:50,666 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:54:05,699 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:54:05,699 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:54:18,691 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:54:20,751 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:54:20,751 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:54:35,779 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:54:35,780 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:54:48,764 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:54:50,813 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:54:50,814 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:55:05,851 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:55:05,851 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:55:18,841 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:55:20,914 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:55:20,915 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:55:35,953 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:55:35,953 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:55:48,918 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:55:50,986 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:55:50,986 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:56:06,018 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:56:06,018 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:56:18,994 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:56:21,056 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:56:21,056 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:56:36,088 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:56:36,088 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:56:49,071 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:56:51,132 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:56:51,132 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:57:06,165 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:57:06,165 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:57:19,148 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:57:21,197 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:57:21,197 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:57:36,225 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:57:36,225 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:57:49,225 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:57:51,255 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:57:51,255 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:58:06,285 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:58:06,285 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:58:19,306 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:58:21,317 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:58:21,318 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:58:36,347 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:58:36,347 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:58:49,390 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:58:51,377 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:58:51,377 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:59:06,621 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:59:06,621 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:59:19,470 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:59:21,650 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:59:21,650 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:59:36,678 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:59:36,679 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:59:39,320 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:59:47,323 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:59:49,324 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:59:49,552 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 12:59:51,325 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:59:51,706 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 12:59:51,706 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 12:59:53,326 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:59:55,326 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:59:57,327 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 12:59:59,328 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:00:01,329 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:00:06,735 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:00:06,735 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:00:19,335 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:00:19,632 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 13:00:21,336 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:00:21,767 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:00:21,767 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:00:25,337 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:00:27,338 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:00:35,340 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:00:36,800 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:00:36,800 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:00:37,341 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:00:39,342 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:00:41,342 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:00:43,343 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:00:45,344 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:00:49,345 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:00:49,713 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 13:00:51,346 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:00:51,828 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:00:51,829 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:00:53,347 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:00:55,347 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:00:57,348 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:01:01,349 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:01:03,350 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:01:05,351 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:01:06,861 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:01:06,862 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:01:13,353 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:01:17,354 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:01:19,355 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:01:19,798 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 13:01:21,356 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:01:21,894 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:01:21,894 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:01:25,357 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:01:33,360 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:01:35,360 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:01:36,923 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:01:36,924 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:01:49,879 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 13:01:51,961 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:01:51,962 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:01:57,368 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:02:01,369 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:02:06,997 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:02:06,997 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:02:19,963 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 13:02:22,026 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:02:22,027 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:02:37,057 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:02:37,057 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:02:50,044 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 13:02:52,086 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:02:52,086 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:03:07,119 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:03:07,119 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:03:20,125 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 13:03:22,227 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:03:22,228 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:03:37,258 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:03:37,258 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:03:50,206 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 13:03:52,293 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:03:52,293 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:04:03,410 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:04:07,321 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:04:07,321 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:04:07,412 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:04:11,413 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:04:13,414 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:04:15,415 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:04:17,416 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:04:19,416 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:04:20,288 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 13:04:21,417 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:04:22,355 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:04:22,355 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:04:37,385 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status +2021-07-09 13:04:37,385 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status +2021-07-09 13:04:47,426 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:04:48,276 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-09 13:04:48,277 DEBUG SenderThread:37232 [sender.py:send():179] send: telemetry +2021-07-09 13:04:48,277 DEBUG SenderThread:37232 [sender.py:send():179] send: exit +2021-07-09 13:04:48,277 INFO SenderThread:37232 [sender.py:send_exit():287] handling exit code: 0 +2021-07-09 13:04:48,277 INFO SenderThread:37232 [sender.py:send_exit():295] send defer +2021-07-09 13:04:48,277 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit +2021-07-09 13:04:48,278 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer +2021-07-09 13:04:48,278 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 0 +2021-07-09 13:04:48,278 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer +2021-07-09 13:04:48,278 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 0 +2021-07-09 13:04:48,278 INFO SenderThread:37232 [sender.py:transition_state():308] send defer: 1 +2021-07-09 13:04:48,278 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer +2021-07-09 13:04:48,278 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 1 +2021-07-09 13:04:48,366 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer +2021-07-09 13:04:48,366 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 1 +2021-07-09 13:04:48,367 INFO SenderThread:37232 [sender.py:transition_state():308] send defer: 2 +2021-07-09 13:04:48,367 DEBUG SenderThread:37232 [sender.py:send():179] send: stats +2021-07-09 13:04:48,367 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer +2021-07-09 13:04:48,367 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 2 +2021-07-09 13:04:48,367 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer +2021-07-09 13:04:48,368 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 2 +2021-07-09 13:04:48,368 INFO SenderThread:37232 [sender.py:transition_state():308] send defer: 3 +2021-07-09 13:04:48,368 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer +2021-07-09 13:04:48,368 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 3 +2021-07-09 13:04:48,368 DEBUG SenderThread:37232 [sender.py:send():179] send: summary +2021-07-09 13:04:48,369 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-09 13:04:48,369 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer +2021-07-09 13:04:48,369 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 3 +2021-07-09 13:04:48,369 INFO SenderThread:37232 [sender.py:transition_state():308] send defer: 4 +2021-07-09 13:04:48,369 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer +2021-07-09 13:04:48,370 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 4 +2021-07-09 13:04:48,370 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer +2021-07-09 13:04:48,370 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 4 +2021-07-09 13:04:48,379 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-09 13:04:48,438 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json +2021-07-09 13:04:48,439 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:04:48,439 INFO SenderThread:37232 [sender.py:transition_state():308] send defer: 5 +2021-07-09 13:04:48,439 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit +2021-07-09 13:04:48,440 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer +2021-07-09 13:04:48,440 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 5 +2021-07-09 13:04:48,440 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer +2021-07-09 13:04:48,440 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 5 +2021-07-09 13:04:48,440 INFO SenderThread:37232 [dir_watcher.py:finish():282] shutting down directory watcher +2021-07-09 13:04:48,541 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-09 13:04:49,439 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/config.yaml +2021-07-09 13:04:49,439 INFO SenderThread:37232 [dir_watcher.py:finish():312] scan: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files +2021-07-09 13:04:49,440 INFO SenderThread:37232 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/requirements.txt requirements.txt +2021-07-09 13:04:49,440 INFO SenderThread:37232 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log output.log +2021-07-09 13:04:49,440 INFO SenderThread:37232 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/wandb-metadata.json wandb-metadata.json +2021-07-09 13:04:49,440 INFO SenderThread:37232 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/config.yaml config.yaml +2021-07-09 13:04:49,440 INFO SenderThread:37232 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json wandb-summary.json +2021-07-09 13:04:49,440 INFO SenderThread:37232 [sender.py:transition_state():308] send defer: 6 +2021-07-09 13:04:49,441 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit +2021-07-09 13:04:49,444 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer +2021-07-09 13:04:49,444 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 6 +2021-07-09 13:04:49,446 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer +2021-07-09 13:04:49,446 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 6 +2021-07-09 13:04:49,446 INFO SenderThread:37232 [file_pusher.py:finish():177] shutting down file pusher +2021-07-09 13:04:49,542 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-09 13:04:49,542 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit +2021-07-09 13:04:49,643 INFO Thread-14 :37232 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/config.yaml +2021-07-09 13:04:49,644 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-09 13:04:49,644 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit +2021-07-09 13:04:49,685 INFO Thread-12 :37232 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/requirements.txt +2021-07-09 13:04:49,687 INFO Thread-15 :37232 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json +2021-07-09 13:04:49,745 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-09 13:04:49,745 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit +2021-07-09 13:04:49,822 INFO Thread-13 :37232 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log +2021-07-09 13:04:49,846 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-09 13:04:49,847 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit +2021-07-09 13:04:49,948 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-09 13:04:49,948 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit +2021-07-09 13:04:50,022 INFO Thread-7 :37232 [sender.py:transition_state():308] send defer: 7 +2021-07-09 13:04:50,023 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer +2021-07-09 13:04:50,023 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 7 +2021-07-09 13:04:50,023 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer +2021-07-09 13:04:50,023 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 7 +2021-07-09 13:04:50,050 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-09 13:04:50,227 INFO SenderThread:37232 [sender.py:transition_state():308] send defer: 8 +2021-07-09 13:04:50,227 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit +2021-07-09 13:04:50,228 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer +2021-07-09 13:04:50,228 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 8 +2021-07-09 13:04:50,228 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer +2021-07-09 13:04:50,228 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 8 +2021-07-09 13:04:50,228 INFO SenderThread:37232 [sender.py:transition_state():308] send defer: 9 +2021-07-09 13:04:50,229 DEBUG SenderThread:37232 [sender.py:send():179] send: final +2021-07-09 13:04:50,229 DEBUG SenderThread:37232 [sender.py:send():179] send: footer +2021-07-09 13:04:50,229 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer +2021-07-09 13:04:50,229 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 9 +2021-07-09 13:04:50,229 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer +2021-07-09 13:04:50,229 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 9 +2021-07-09 13:04:50,329 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit +2021-07-09 13:04:50,329 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit +2021-07-09 13:04:50,329 INFO SenderThread:37232 [file_pusher.py:join():182] waiting for file pusher +2021-07-09 13:04:50,330 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: get_summary +2021-07-09 13:04:50,331 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: sampled_history +2021-07-09 13:04:50,332 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: shutdown +2021-07-09 13:04:50,332 INFO HandlerThread:37232 [handler.py:finish():638] shutting down handler +2021-07-09 13:04:51,229 INFO WriterThread:37232 [datastore.py:close():288] close: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/run-89spqu1d.wandb +2021-07-09 13:04:51,330 INFO SenderThread:37232 [sender.py:finish():945] shutting down sender +2021-07-09 13:04:51,330 INFO SenderThread:37232 [file_pusher.py:finish():177] shutting down file pusher +2021-07-09 13:04:51,330 INFO SenderThread:37232 [file_pusher.py:join():182] waiting for file pusher +2021-07-09 13:04:51,332 INFO MainThread:37232 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210709_125049-89spqu1d/logs/debug.log b/wandb/run-20210709_125049-89spqu1d/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f16e46ce17a910c434a714b562ac800ef11bccd2 --- /dev/null +++ b/wandb/run-20210709_125049-89spqu1d/logs/debug.log @@ -0,0 +1,114 @@ +2021-07-09 12:50:49,339 INFO MainThread:36178 [wandb_setup.py:_flush():69] Unhandled environment var: WANDB_WATCH +2021-07-09 12:50:49,339 INFO MainThread:36178 [wandb_setup.py:_flush():69] setting env: {'project': 'hf-flax-robert-base-mr', 'entity': 'nipunsadvilkar'} +2021-07-09 12:50:49,339 INFO MainThread:36178 [wandb_setup.py:_flush():69] setting login settings: {} +2021-07-09 12:50:49,339 INFO MainThread:36178 [wandb_init.py:_log_setup():337] Logging user logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/logs/debug.log +2021-07-09 12:50:49,339 INFO MainThread:36178 [wandb_init.py:_log_setup():338] Logging internal logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/logs/debug-internal.log +2021-07-09 12:50:49,339 INFO MainThread:36178 [wandb_init.py:init():370] calling init triggers +2021-07-09 12:50:49,339 INFO MainThread:36178 [wandb_init.py:init():375] wandb.init called with sweep_config: {} +config: {} +2021-07-09 12:50:49,339 INFO MainThread:36178 [wandb_init.py:init():419] starting backend +2021-07-09 12:50:49,339 INFO MainThread:36178 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2021-07-09 12:50:49,373 INFO MainThread:36178 [backend.py:ensure_launched():135] starting backend process... +2021-07-09 12:50:49,407 INFO MainThread:36178 [backend.py:ensure_launched():139] started backend process with pid: 37232 +2021-07-09 12:50:49,409 INFO MainThread:36178 [wandb_init.py:init():424] backend started and connected +2021-07-09 12:50:49,412 INFO MainThread:36178 [wandb_init.py:init():472] updated telemetry +2021-07-09 12:50:49,413 INFO MainThread:36178 [wandb_init.py:init():491] communicating current version +2021-07-09 12:50:50,062 INFO MainThread:36178 [wandb_init.py:init():496] got version response +2021-07-09 12:50:50,062 INFO MainThread:36178 [wandb_init.py:init():504] communicating run to backend with 30 second timeout +2021-07-09 12:50:50,127 INFO MainThread:36178 [wandb_init.py:init():529] starting run threads in backend +2021-07-09 12:50:50,186 INFO MainThread:36178 [wandb_run.py:_console_start():1623] atexit reg +2021-07-09 12:50:50,187 INFO MainThread:36178 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT +2021-07-09 12:50:50,187 INFO MainThread:36178 [wandb_run.py:_redirect():1502] Redirecting console. +2021-07-09 12:50:50,189 INFO MainThread:36178 [wandb_run.py:_redirect():1558] Redirects installed. +2021-07-09 12:50:50,189 INFO MainThread:36178 [wandb_init.py:init():554] run started, returning control to user process +2021-07-09 12:50:50,190 INFO MainThread:36178 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 4, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 0.0003, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 8.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul09_12-50-44_t1v-n-112df4a9-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 500, 'save_total_limit': None, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} +2021-07-09 12:50:50,191 INFO MainThread:36178 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'roberta', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'} +2021-07-09 12:50:50,192 INFO MainThread:36178 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': 'oscar', 'dataset_config_name': 'unshuffled_deduplicated_als', 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 128, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False} +2021-07-09 13:04:45,688 INFO MainThread:36178 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 0 +2021-07-09 13:04:45,689 INFO MainThread:36178 [wandb_run.py:_restore():1565] restore +2021-07-09 13:04:48,278 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1200 + total_bytes: 1200 +} + +2021-07-09 13:04:48,440 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 1 +} +pusher_stats { + uploaded_bytes: 1200 + total_bytes: 1200 +} + +2021-07-09 13:04:49,441 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 3 +} +pusher_stats { + uploaded_bytes: 1200 + total_bytes: 30159 +} + +2021-07-09 13:04:49,543 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 36687 + total_bytes: 36687 +} + +2021-07-09 13:04:49,644 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 36687 + total_bytes: 36687 +} + +2021-07-09 13:04:49,746 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 36687 + total_bytes: 36687 +} + +2021-07-09 13:04:49,847 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 36687 + total_bytes: 36687 +} + +2021-07-09 13:04:49,949 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 36687 + total_bytes: 36687 +} + +2021-07-09 13:04:50,228 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 36687 + total_bytes: 36687 +} + +2021-07-09 13:04:50,330 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true +exit_result { +} +file_counts { + wandb_count: 5 +} +pusher_stats { + uploaded_bytes: 36687 + total_bytes: 36687 +} + +2021-07-09 13:04:51,404 INFO MainThread:36178 [wandb_run.py:_show_summary():1870] rendering summary +2021-07-09 13:04:51,404 INFO MainThread:36178 [wandb_run.py:_show_history():1908] rendering history +2021-07-09 13:04:51,405 INFO MainThread:36178 [wandb_run.py:_show_files():1937] logging synced files diff --git a/wandb/run-20210709_125049-89spqu1d/run-89spqu1d.wandb b/wandb/run-20210709_125049-89spqu1d/run-89spqu1d.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ac90648e0c3543667d562c273ff4b3a5eb496bff Binary files /dev/null and b/wandb/run-20210709_125049-89spqu1d/run-89spqu1d.wandb differ