step 50000

.gitattributes

@@ -30,3 +30,5 @@ wandb/run-20211111_214446-3xgbgfj2/logs/debug-internal.log filter=lfs diff=lfs m
 wandb/run-20211111_214446-3xgbgfj2/run-3xgbgfj2.wandb filter=lfs diff=lfs merge=lfs -text
 log/debug_0.log filter=lfs diff=lfs merge=lfs -text
 wandb/run-20211111_214446-3xgbgfj2/files/output.log filter=lfs diff=lfs merge=lfs -text
+wandb/run-20220114_155039-1uxxfypf/logs/debug-internal.log filter=lfs diff=lfs merge=lfs -text
+wandb/run-20220114_155039-1uxxfypf/run-1uxxfypf.wandb filter=lfs diff=lfs merge=lfs -text

codeparrot_training.py

@@ -66,8 +66,8 @@ def setup_logging(project_name):
         tb_writer = SummaryWriter()
         tb_writer.add_hparams(vars(args), {'0': 0})
         logger.setLevel(logging.INFO)
-        datasets.utils.logging.set_verbosity_info()
-        transformers.utils.logging.set_verbosity_info()
+        datasets.utils.logging.set_verbosity_warning()
+        transformers.utils.logging.set_verbosity_warning()
     else:
         tb_writer = None
         run_name = ''
@@ -129,7 +129,7 @@ config = {"train_batch_size": 2,
           "valid_batch_size": 2,
           "weight_decay": 0.1,
           "shuffle_buffer": 1_000,
-          "learning_rate": 2e-4,
+          "learning_rate": 5e-5,
           "lr_scheduler_type": "cosine",
           "num_warmup_steps": 750,
           "gradient_accumulation_steps": 16,
@@ -205,4 +205,4 @@ accelerator.wait_for_everyone()
 unwrapped_model = accelerator.unwrap_model(model)
 unwrapped_model.save_pretrained("./", save_function=accelerator.save)
 if accelerator.is_main_process:
-    hf_repo.push_to_hub(commit_message=f'final model')
+    hf_repo.push_to_hub(commit_message=f'final model')

log/debug_0.log

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5cdd17b1924099ddd49556e821405a08bec25355cf02f3e25c527eade05a3084
-size 122816913
+oid sha256:0475c8ce3fa5b0111ead4e575b0d6b41ddf933c6bccf1374c46bf43f376e83f8
+size 130284403

log/debug_1.log

@@ -1 +1,2 @@
 11/11/2021 21:45:50 - INFO - root - Reducer buckets have been rebuilt in this iteration.
+01/14/2022 15:51:31 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_10.log

@@ -1 +1,2 @@
 11/11/2021 21:45:50 - INFO - root - Reducer buckets have been rebuilt in this iteration.
+01/14/2022 15:51:31 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_11.log

@@ -1 +1,2 @@
 11/11/2021 21:45:50 - INFO - root - Reducer buckets have been rebuilt in this iteration.
+01/14/2022 15:51:31 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_12.log

@@ -1 +1,2 @@
 11/11/2021 21:45:50 - INFO - root - Reducer buckets have been rebuilt in this iteration.
+01/14/2022 15:51:31 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_13.log

@@ -1 +1,2 @@
 11/11/2021 21:45:50 - INFO - root - Reducer buckets have been rebuilt in this iteration.
+01/14/2022 15:51:31 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_14.log

@@ -1 +1,2 @@
 11/11/2021 21:45:50 - INFO - root - Reducer buckets have been rebuilt in this iteration.
+01/14/2022 15:51:31 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_15.log

@@ -1 +1,2 @@
 11/11/2021 21:45:50 - INFO - root - Reducer buckets have been rebuilt in this iteration.
+01/14/2022 15:51:31 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_2.log

@@ -1 +1,2 @@
 11/11/2021 21:45:50 - INFO - root - Reducer buckets have been rebuilt in this iteration.
+01/14/2022 15:51:31 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_3.log

@@ -1 +1,2 @@
 11/11/2021 21:45:50 - INFO - root - Reducer buckets have been rebuilt in this iteration.
+01/14/2022 15:51:31 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_4.log

@@ -1 +1,2 @@
 11/11/2021 21:45:50 - INFO - root - Reducer buckets have been rebuilt in this iteration.
+01/14/2022 15:51:31 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_5.log

@@ -1 +1,2 @@
 11/11/2021 21:45:50 - INFO - root - Reducer buckets have been rebuilt in this iteration.
+01/14/2022 15:51:31 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_6.log

@@ -1 +1,2 @@
 11/11/2021 21:45:50 - INFO - root - Reducer buckets have been rebuilt in this iteration.
+01/14/2022 15:51:31 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_7.log

@@ -1 +1,2 @@
 11/11/2021 21:45:50 - INFO - root - Reducer buckets have been rebuilt in this iteration.
+01/14/2022 15:51:31 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_8.log

@@ -1 +1,2 @@
 11/11/2021 21:45:50 - INFO - root - Reducer buckets have been rebuilt in this iteration.
+01/14/2022 15:51:31 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_9.log

@@ -1 +1,2 @@
 11/11/2021 21:45:50 - INFO - root - Reducer buckets have been rebuilt in this iteration.
+01/14/2022 15:51:31 - INFO - root - Reducer buckets have been rebuilt in this iteration.

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6302738ec9cd3c4042a27328e32d46f8d918861c6228127bb680f5ab214465e0
+oid sha256:08a8edda980a53bb2df006ea4c632204bac679c41a49159443ef634983dcf03a
 size 6169094681

runs/Jan14_15-50-40_leandro-16x-a100-b/1642175440.8731604/events.out.tfevents.1642175440.leandro-16x-a100-b.28548.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3500e6cd241c92592124e68388c45a088a9f122c0097e6f123e51f4d74173780
+size 1438

runs/Jan14_15-50-40_leandro-16x-a100-b/events.out.tfevents.1642175440.leandro-16x-a100-b.28548.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18a3d7dfc1bd05e1de4b9b83f86f99705014175418c2c1d563ded9f0144b23bc
+size 9134099

wandb/debug-internal.log

@@ -1 +1 @@
-run-20211111_214446-3xgbgfj2/logs/debug-internal.log
+run-20220114_155039-1uxxfypf/logs/debug-internal.log

wandb/debug.log

@@ -1 +1 @@
-run-20211111_214446-3xgbgfj2/logs/debug.log
+run-20220114_155039-1uxxfypf/logs/debug.log

wandb/latest-run

@@ -1 +1 @@
-run-20211111_214446-3xgbgfj2
+run-20220114_155039-1uxxfypf

wandb/run-20220114_155039-1uxxfypf/files/config.yaml

@@ -0,0 +1,92 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.12.2
+    framework: huggingface
+    huggingface_version: 4.12.2
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.8.11
+    start_time: 1642175439
+    t:
+      1:
+      - 1
+      - 11
+      3:
+      - 16
+      4: 3.8.11
+      5: 0.12.2
+      6: 4.12.2
+      8:
+      - 5
+backend:
+  desc: null
+  value: nccl
+deepspeed_plugin:
+  desc: null
+  value: None
+device:
+  desc: null
+  value: cuda:0
+distributed_type:
+  desc: null
+  value: DistributedType.MULTI_GPU
+gradient_accumulation_steps:
+  desc: null
+  value: 16
+gradient_checkpointing:
+  desc: null
+  value: true
+initialized:
+  desc: null
+  value: 'True'
+learning_rate:
+  desc: null
+  value: 5.0e-05
+local_process_index:
+  desc: null
+  value: '0'
+lr_scheduler_type:
+  desc: null
+  value: cosine
+max_eval_steps:
+  desc: null
+  value: -1
+max_train_steps:
+  desc: null
+  value: 50000
+num_processes:
+  desc: null
+  value: '16'
+num_warmup_steps:
+  desc: null
+  value: 750
+process_index:
+  desc: null
+  value: '0'
+save_checkpoint_steps:
+  desc: null
+  value: 50000
+seed:
+  desc: null
+  value: 1
+seq_length:
+  desc: null
+  value: 1024
+shuffle_buffer:
+  desc: null
+  value: 1000
+train_batch_size:
+  desc: null
+  value: 2
+use_fp16:
+  desc: null
+  value: 'True'
+valid_batch_size:
+  desc: null
+  value: 2
+weight_decay:
+  desc: null
+  value: 0.1

wandb/run-20220114_155039-1uxxfypf/files/requirements.txt

@@ -0,0 +1,81 @@
+absl-py==0.13.0
+accelerate==0.5.1
+aiohttp==3.7.4.post0
+async-timeout==3.0.1
+attrs==21.2.0
+cachetools==4.2.2
+certifi==2021.5.30
+chardet==4.0.0
+charset-normalizer==2.0.5
+click==8.0.1
+configparser==5.0.2
+datasets==1.13.0
+deepspeed==0.5.2
+dill==0.3.4
+docker-pycreds==0.4.0
+filelock==3.0.12
+fsspec==2021.8.1
+gitdb==4.0.7
+gitpython==3.1.18
+google-auth-oauthlib==0.4.6
+google-auth==1.35.0
+grpcio==1.40.0
+huggingface-hub==0.0.19
+idna==3.2
+joblib==1.0.1
+markdown==3.3.4
+mkl-fft==1.3.0
+mkl-random==1.2.2
+mkl-service==2.4.0
+multidict==5.1.0
+multiprocess==0.70.12.2
+ninja==1.10.2
+numpy==1.20.3
+oauthlib==3.1.1
+olefile==0.46
+packaging==21.0
+pandas==1.3.3
+pathtools==0.1.2
+pillow==8.3.1
+pip==21.0.1
+promise==2.3
+protobuf==3.18.0
+psutil==5.8.0
+pyarrow==5.0.0
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pyparsing==2.4.7
+python-dateutil==2.8.2
+pytz==2021.1
+pyyaml==5.4.1
+regex==2021.8.28
+requests-oauthlib==1.3.0
+requests==2.26.0
+rsa==4.7.2
+sacremoses==0.0.45
+sentry-sdk==1.3.1
+setuptools==52.0.0.post20210125
+shortuuid==1.0.1
+six==1.16.0
+smmap==4.0.0
+subprocess32==3.5.4
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.0
+tensorboard==2.6.0
+tensorboardx==1.8
+termcolor==1.1.0
+tokenizers==0.10.3
+torch==1.9.0
+torchaudio==0.9.0a0+33b2469
+torchvision==0.10.0
+tqdm==4.62.2
+transformers==4.12.2
+triton==1.0.0
+typing-extensions==3.10.0.0
+urllib3==1.26.6
+wandb==0.12.2
+werkzeug==2.0.1
+wheel==0.37.0
+xxhash==2.0.2
+yarl==1.6.3
+yaspin==2.1.0

wandb/run-20220114_155039-1uxxfypf/files/wandb-metadata.json

@@ -0,0 +1,24 @@
+{
+    "os": "Linux-4.19.0-18-cloud-amd64-x86_64-with-glibc2.17",
+    "python": "3.8.11",
+    "heartbeatAt": "2022-01-14T15:50:40.774787",
+    "startedAt": "2022-01-14T15:50:39.945346",
+    "docker": null,
+    "gpu": "A100-SXM4-40GB",
+    "gpu_count": 16,
+    "cpu_count": 96,
+    "cuda": null,
+    "args": [],
+    "state": "running",
+    "program": "codeparrot_training.py",
+    "codePath": "codeparrot_training.py",
+    "git": {
+        "remote": "https://huggingface.co/lvwerra/codeparrot",
+        "commit": "23d07ce28c7ec45049bfe92d9e1b74c0acba62f8"
+    },
+    "email": "leandro.vonwerra@gmail.com",
+    "root": "/home/leandro/codeparrot",
+    "host": "leandro-16x-a100-b",
+    "username": "leandro",
+    "executable": "/home/leandro/miniconda3/envs/codeparrot2/bin/python"
+}

wandb/run-20220114_155039-1uxxfypf/files/wandb-summary.json

@@ -0,0 +1 @@
+{"lr": 4.9713892916952546e-05, "samples": 1600000, "steps": 3124, "loss/train": 1.10879647731781, "_runtime": 40015, "_timestamp": 1642215454, "_step": 50000, "loss/eval": 1.0396919250488281, "perplexity": 2.828345537185669}

wandb/run-20220114_155039-1uxxfypf/logs/debug-internal.log

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7c80e994541bf475850a8692ca0f54fc2e8d575f234abc757fb6353bbe84732
+size 26892666

wandb/run-20220114_155039-1uxxfypf/logs/debug.log

@@ -0,0 +1,23 @@
+2022-01-14 15:50:39,947 INFO    MainThread:28548 [wandb_setup.py:_flush():69] setting env: {}
+2022-01-14 15:50:39,947 INFO    MainThread:28548 [wandb_setup.py:_flush():69] setting login settings: {}
+2022-01-14 15:50:39,947 INFO    MainThread:28548 [wandb_init.py:_log_setup():348] Logging user logs to /home/leandro/codeparrot/wandb/run-20220114_155039-1uxxfypf/logs/debug.log
+2022-01-14 15:50:39,947 INFO    MainThread:28548 [wandb_init.py:_log_setup():349] Logging internal logs to /home/leandro/codeparrot/wandb/run-20220114_155039-1uxxfypf/logs/debug-internal.log
+2022-01-14 15:50:39,947 INFO    MainThread:28548 [wandb_init.py:init():381] calling init triggers
+2022-01-14 15:50:39,948 INFO    MainThread:28548 [wandb_init.py:init():386] wandb.init called with sweep_config: {}
+config: {'train_batch_size': 2, 'valid_batch_size': 2, 'weight_decay': 0.1, 'shuffle_buffer': 1000, 'learning_rate': 5e-05, 'lr_scheduler_type': 'cosine', 'num_warmup_steps': 750, 'gradient_accumulation_steps': 16, 'gradient_checkpointing': True, 'max_train_steps': 50000, 'max_eval_steps': -1, 'seq_length': 1024, 'seed': 1, 'save_checkpoint_steps': 50000, 'backend': 'nccl', 'deepspeed_plugin': 'None', 'distributed_type': 'DistributedType.MULTI_GPU', 'num_processes': '16', 'process_index': '0', 'local_process_index': '0', 'device': 'cuda:0', 'use_fp16': 'True', 'initialized': 'True'}
+2022-01-14 15:50:39,948 INFO    MainThread:28548 [wandb_init.py:init():430] starting backend
+2022-01-14 15:50:39,948 INFO    MainThread:28548 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2022-01-14 15:50:39,961 INFO    MainThread:28548 [backend.py:ensure_launched():135] starting backend process...
+2022-01-14 15:50:39,970 INFO    MainThread:28548 [backend.py:ensure_launched():139] started backend process with pid: 28880
+2022-01-14 15:50:39,972 INFO    MainThread:28548 [wandb_init.py:init():435] backend started and connected
+2022-01-14 15:50:39,977 INFO    MainThread:28548 [wandb_init.py:init():494] updated telemetry
+2022-01-14 15:50:39,978 INFO    MainThread:28548 [wandb_init.py:init():517] communicating current version
+2022-01-14 15:50:40,637 INFO    MainThread:28548 [wandb_init.py:init():522] got version response upgrade_message: "wandb version 0.12.9 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2022-01-14 15:50:40,637 INFO    MainThread:28548 [wandb_init.py:init():530] communicating run to backend with 30 second timeout
+2022-01-14 15:50:40,721 INFO    MainThread:28548 [wandb_init.py:init():557] starting run threads in backend
+2022-01-14 15:50:40,860 INFO    MainThread:28548 [wandb_run.py:_console_start():1605] atexit reg
+2022-01-14 15:50:40,861 INFO    MainThread:28548 [wandb_run.py:_redirect():1479] redirect: SettingsConsole.REDIRECT
+2022-01-14 15:50:40,861 INFO    MainThread:28548 [wandb_run.py:_redirect():1484] Redirecting console.
+2022-01-14 15:50:40,865 INFO    MainThread:28548 [wandb_run.py:_redirect():1540] Redirects installed.
+2022-01-14 15:50:40,865 INFO    MainThread:28548 [wandb_init.py:init():582] run started, returning control to user process

wandb/run-20220114_155039-1uxxfypf/run-1uxxfypf.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48ff5908577eae0523c5d4692a24039eb2f6ee01c42bdc2f1d881bf030f84bf0
+size 19776992