Saving weights and logs of step 500

.gitattributes

@@ -14,3 +14,4 @@
 *.pb filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:76d55c57630984430801c3b6d5cc75703bd2a61ef6bb8f9d20f7c6150f2dfbd3
+oid sha256:f1f30bd95331d281308f6cb94e74bec0d82ebc06684501d536c8d38952e09abb
 size 498796983

run.log

@@ -1,4 +1,4 @@
-[16:38:48] - INFO - __main__ - Training/evaluation parameters TrainingArguments(
+[10:39:50] - INFO - __main__ - Training/evaluation parameters TrainingArguments(
 _n_gpu=-1,
 adafactor=False,
 adam_beta1=0.9,
@@ -34,7 +34,7 @@ local_rank=-1,
 log_level=-1,
 log_level_replica=-1,
 log_on_each_node=True,
-logging_dir=./runs/Jul08_16-38-48_t1v-n-112df4a9-w-0,
+logging_dir=./runs/Jul09_10-39-50_t1v-n-112df4a9-w-0,
 logging_first_step=False,
 logging_steps=500,
 logging_strategy=IntervalStrategy.STEPS,
@@ -56,9 +56,9 @@ push_to_hub_model_id=flax-community/roberta-base-mr,
 push_to_hub_organization=None,
 push_to_hub_token=vdIAyRvCACJNslYtyLHufmNDnUIyknPzUgVDMFiXqJoulvMqjoubonLJzXOJQJczWfRMJumVaMFjGSFVnQAMdswvZkzNIthKrxBeARBXfqnIwjABkKpCbjGEgnkjpjKi,
 remove_unused_columns=True,
-report_to=[],
+report_to=['wandb'],
 resume_from_checkpoint=None,
-run_name=./,
+run_name=hf-flax-robert-base-mr,
 save_on_each_node=False,
 save_steps=500,
 save_strategy=IntervalStrategy.STEPS,
@@ -73,326 +73,183 @@ warmup_ratio=0.0,
 warmup_steps=1000,
 weight_decay=0.0,
 )
-[16:38:48] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443
-[16:38:48] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0
-[16:38:48] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
-[16:38:48] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0
-[16:38:48] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
-[16:38:49] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0
-[16:38:49] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2)
-[16:38:49] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443
-[16:38:49] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0
-[16:38:49] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
-[16:38:49] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0
-[16:38:49] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
-[16:38:49] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0
-[16:38:49] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2)
-[16:38:49] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443
-[16:38:49] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0
-[16:38:49] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
-[16:38:49] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0
-[16:38:49] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
-[16:38:49] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0
-[16:38:49] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2)
-[16:38:49] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-0f52086e7b10d7e8.arrow
-[16:38:49] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-a39e5f5a5c6c69fc.arrow
-[16:38:49] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-e4d3282a2dd50fa0.arrow
-[16:38:49] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-b9a3aa9913be3b34.arrow
-[16:38:49] - WARNING - __main__ - Unable to display metrics through TensorBoard because the package is not installed: Please run pip install tensorboard to enable.
-[16:38:49] - INFO - absl - Starting the local TPU driver.
-[16:38:49] - INFO - absl - Unable to initialize backend 'tpu_driver': Not found: Unable to find driver in registry given worker: local://
-[16:38:49] - INFO - absl - Unable to initialize backend 'gpu': Not found: Could not find registered platform with name: "cuda". Available platform names are: TPU Host Interpreter
+[10:39:50] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443
+[10:39:50] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0
+[10:39:50] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
+[10:39:50] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0
+[10:39:50] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
+[10:39:50] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0
+[10:39:50] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2)
+[10:39:50] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443
+[10:39:51] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0
+[10:39:51] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
+[10:39:51] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0
+[10:39:51] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
+[10:39:51] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0
+[10:39:51] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2)
+[10:39:51] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443
+[10:39:51] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/oscar/oscar.py HTTP/1.1" 404 0
+[10:39:51] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
+[10:39:51] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/oscar.py HTTP/1.1" 200 0
+[10:39:51] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): raw.githubusercontent.com:443
+[10:39:51] - DEBUG - urllib3.connectionpool - https://raw.githubusercontent.com:443 "HEAD /huggingface/datasets/master/datasets/oscar/dataset_infos.json HTTP/1.1" 200 0
+[10:39:51] - WARNING - datasets.builder - Reusing dataset oscar (/home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2)
+[10:39:51] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-0f52086e7b10d7e8.arrow
+[10:39:51] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-a39e5f5a5c6c69fc.arrow
+[10:39:51] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-e4d3282a2dd50fa0.arrow
+[10:39:51] - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/nipunsadvilkar/.cache/huggingface/datasets/oscar/unshuffled_deduplicated_als/1.0.0/84838bd49d2295f62008383b05620571535451d84545037bb94d6f3501651df2/cache-b9a3aa9913be3b34.arrow
+[10:39:51] - INFO - absl - Starting the local TPU driver.
+[10:39:51] - INFO - absl - Unable to initialize backend 'tpu_driver': Not found: Unable to find driver in registry given worker: local://
+[10:39:51] - INFO - absl - Unable to initialize backend 'gpu': Not found: Could not find registered platform with name: "cuda". Available platform names are: Interpreter Host TPU
+[10:39:55] - WARNING - __main__ - Unable to display metrics through TensorBoard because some package are not installed: No module named 'tensorflow'
 /home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
   warnings.warn(
 /home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
   warnings.warn(
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
                                                              
-
                                                             
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                              
+
                                                             
+
 
 
 
 
-
+
 
-
-[16:40:34] - INFO - huggingface_hub.repository - git version 2.25.1
+
+[10:41:35] - INFO - huggingface_hub.repository - git version 2.25.1
 git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
-[16:40:34] - DEBUG - huggingface_hub.repository - [Repository] is a valid git repo
+[10:41:35] - DEBUG - huggingface_hub.repository - [Repository] is a valid git repo
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
                                                                                                               
-
                                                            
-
-
-
-
-
-
-[16:41:04] - INFO - huggingface_hub.repository - git version 2.25.1
-git-lfs/2.9.2 (GitHub; linux amd64; go 1.13.5)
-[16:41:04] - DEBUG - huggingface_hub.repository - [Repository] is a valid git repo
+

run.sh

@@ -13,6 +13,8 @@ python run_mlm_flax.py \
     --warmup_steps="1000" \
     --overwrite_output_dir \
     --num_train_epochs="8" \
+    --report_to wandb \
+    --run_name hf-flax-robert-base-mr \
     --push_to_hub_model_id="flax-community/roberta-base-mr" \
     --push_to_hub_token=$HUB_TOKEN \
     --push_to_hub 2>&1 | tee run.log

run_mlm_flax.py

@@ -457,7 +457,6 @@ if __name__ == "__main__":
     if has_tensorboard and jax.process_index() == 0:
         try:
             from flax.metrics.tensorboard import SummaryWriter
-
             summary_writer = SummaryWriter(log_dir=Path(training_args.output_dir))
         except ImportError as ie:
             has_tensorboard = False

wandb/debug-internal.log

@@ -0,0 +1 @@
+run-20210709_103433-d7n9baaw/logs/debug-internal.log

wandb/debug.log

@@ -0,0 +1 @@
+run-20210709_103433-d7n9baaw/logs/debug.log

wandb/latest-run

@@ -0,0 +1 @@
+run-20210709_103433-d7n9baaw

wandb/run-20210709_103433-d7n9baaw/files/config.yaml

@@ -0,0 +1,19 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.10.33
+    framework: huggingface
+    huggingface_version: 4.9.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.8.10
+    t:
+      1:
+      - 11
+      4: 3.8.10
+      5: 0.10.33
+      6: 4.9.0.dev0
+      8:
+      - 5

wandb/run-20210709_103433-d7n9baaw/logs/debug-internal.log

@@ -0,0 +1,72 @@
+2021-07-09 10:34:34,450 INFO    MainThread:25100 [internal.py:wandb_internal():88] W&B internal server running at pid: 25100, started at: 2021-07-09 10:34:34.449847
+2021-07-09 10:34:34,452 DEBUG   HandlerThread:25100 [handler.py:handle_request():124] handle_request: check_version
+2021-07-09 10:34:34,452 INFO    WriterThread:25100 [datastore.py:open_for_write():80] open: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_103433-d7n9baaw/run-d7n9baaw.wandb
+2021-07-09 10:34:34,453 DEBUG   SenderThread:25100 [sender.py:send():179] send: header
+2021-07-09 10:34:34,453 DEBUG   SenderThread:25100 [sender.py:send_request():193] send_request: check_version
+2021-07-09 10:34:34,515 DEBUG   SenderThread:25100 [sender.py:send():179] send: run
+2021-07-09 10:34:34,556 ERROR   SenderThread:25100 [internal_api.py:execute():130] 403 response executing GraphQL.
+2021-07-09 10:34:34,556 ERROR   SenderThread:25100 [internal_api.py:execute():131] {"errors":[{"message":"permission denied","path":["upsertBucket"],"extensions":{"code":"PERMISSION_ERROR"}}],"data":{"upsertBucket":null}}
+2021-07-09 10:34:35,516 INFO    HandlerThread:25100 [handler.py:finish():638] shutting down handler
+2021-07-09 10:34:35,517 INFO    WriterThread:25100 [datastore.py:close():288] close: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_103433-d7n9baaw/run-d7n9baaw.wandb
+2021-07-09 10:34:35,517 ERROR   MainThread:25100 [internal.py:wandb_internal():159] Thread SenderThread:
+Traceback (most recent call last):
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/lib/retry.py", line 102, in __call__
+    result = self._call_fn(*args, **kwargs)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/internal/internal_api.py", line 133, in execute
+    six.reraise(*sys.exc_info())
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/six.py", line 719, in reraise
+    raise value
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/internal/internal_api.py", line 127, in execute
+    return self.client.execute(*args, **kwargs)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/vendor/gql-0.2.0/gql/client.py", line 52, in execute
+    result = self._get_result(document, *args, **kwargs)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/vendor/gql-0.2.0/gql/client.py", line 60, in _get_result
+    return self.transport.execute(document, *args, **kwargs)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/vendor/gql-0.2.0/gql/transport/requests.py", line 39, in execute
+    request.raise_for_status()
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/requests/models.py", line 943, in raise_for_status
+    raise HTTPError(http_error_msg, response=self)
+requests.exceptions.HTTPError: 403 Client Error: Forbidden for url: https://api.wandb.ai/graphql
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/apis/normalize.py", line 24, in wrapper
+    return func(*args, **kwargs)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/internal/internal_api.py", line 922, in upsert_run
+    response = self.gql(mutation, variable_values=variable_values, **kwargs)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/lib/retry.py", line 118, in __call__
+    if not check_retry_fn(e):
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/util.py", line 727, in no_retry_auth
+    raise CommError("Permission denied, ask the project owner to grant you access")
+wandb.errors.CommError: Permission denied, ask the project owner to grant you access
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/internal/internal_util.py", line 55, in run
+    self._run()
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/internal/internal_util.py", line 105, in _run
+    self._process(record)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/internal/internal.py", line 292, in _process
+    self._sm.send(record)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/internal/sender.py", line 181, in send
+    send_handler(record)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/internal/sender.py", line 604, in send_run
+    self._init_run(run, config_value_dict)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/internal/sender.py", line 626, in _init_run
+    server_run, inserted = self._api.upsert_run(
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/apis/normalize.py", line 62, in wrapper
+    six.reraise(CommError, CommError(message, err), sys.exc_info()[2])
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/six.py", line 718, in reraise
+    raise value.with_traceback(tb)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/apis/normalize.py", line 24, in wrapper
+    return func(*args, **kwargs)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/internal/internal_api.py", line 922, in upsert_run
+    response = self.gql(mutation, variable_values=variable_values, **kwargs)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/lib/retry.py", line 118, in __call__
+    if not check_retry_fn(e):
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/util.py", line 727, in no_retry_auth
+    raise CommError("Permission denied, ask the project owner to grant you access")
+wandb.errors.CommError: Permission denied, ask the project owner to grant you access
+2021-07-09 10:34:37,565 INFO    MainThread:25100 [internal.py:handle_exit():78] Internal process exited

wandb/run-20210709_103433-d7n9baaw/logs/debug.log

@@ -0,0 +1,35 @@
+2021-07-09 10:34:33,839 INFO    MainThread:24042 [wandb_setup.py:_flush():69] setting env: {}
+2021-07-09 10:34:33,839 INFO    MainThread:24042 [wandb_setup.py:_flush():69] setting login settings: {}
+2021-07-09 10:34:33,839 INFO    MainThread:24042 [wandb_init.py:_log_setup():337] Logging user logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_103433-d7n9baaw/logs/debug.log
+2021-07-09 10:34:33,839 INFO    MainThread:24042 [wandb_init.py:_log_setup():338] Logging internal logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_103433-d7n9baaw/logs/debug-internal.log
+2021-07-09 10:34:33,839 INFO    MainThread:24042 [wandb_init.py:init():370] calling init triggers
+2021-07-09 10:34:33,839 INFO    MainThread:24042 [wandb_init.py:init():375] wandb.init called with sweep_config: {}
+config: {}
+2021-07-09 10:34:33,839 INFO    MainThread:24042 [wandb_init.py:init():419] starting backend
+2021-07-09 10:34:33,839 INFO    MainThread:24042 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2021-07-09 10:34:33,940 INFO    MainThread:24042 [backend.py:ensure_launched():135] starting backend process...
+2021-07-09 10:34:33,967 INFO    MainThread:24042 [backend.py:ensure_launched():139] started backend process with pid: 25100
+2021-07-09 10:34:33,969 INFO    MainThread:24042 [wandb_init.py:init():424] backend started and connected
+2021-07-09 10:34:33,971 INFO    MainThread:24042 [wandb_init.py:init():472] updated telemetry
+2021-07-09 10:34:33,972 INFO    MainThread:24042 [wandb_init.py:init():491] communicating current version
+2021-07-09 10:34:34,514 INFO    MainThread:24042 [wandb_init.py:init():496] got version response 
+2021-07-09 10:34:34,514 INFO    MainThread:24042 [wandb_init.py:init():504] communicating run to backend with 30 second timeout
+2021-07-09 10:34:46,387 WARNING MainThread:24042 [wandb_init.py:init():781] interrupted
+Traceback (most recent call last):
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/wandb_init.py", line 761, in init
+    run = wi.init()
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/wandb_init.py", line 505, in init
+    ret = backend.interface.communicate_run(run, timeout=30)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 667, in communicate_run
+    return self._communicate_run(run, timeout=timeout)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 655, in _communicate_run
+    resp = self._communicate(req, timeout=timeout)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 545, in _communicate
+    return self._communicate_async(rec, local=local).get(timeout=timeout)
+  File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 82, in get
+    is_set = self._object_ready.wait(timeout)
+  File "/usr/lib/python3.8/threading.py", line 558, in wait
+    signaled = self._cond.wait(timeout)
+  File "/usr/lib/python3.8/threading.py", line 306, in wait
+    gotit = waiter.acquire(True, timeout)
+KeyboardInterrupt