step 15000

config.json

@@ -1,4 +1,5 @@
 {
+  "_name_or_path": "./",
   "activation_function": "gelu_new",
   "architectures": [
     "GPT2LMHeadModel"

log/debug_1.log

@@ -0,0 +1 @@
+11/06/2021 21:16:43 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_10.log

@@ -0,0 +1 @@
+11/06/2021 21:16:43 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_11.log

@@ -0,0 +1 @@
+11/06/2021 21:16:43 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_12.log

@@ -0,0 +1 @@
+11/06/2021 21:16:43 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_13.log

@@ -0,0 +1 @@
+11/06/2021 21:16:43 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_14.log

@@ -0,0 +1 @@
+11/06/2021 21:16:43 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_15.log

@@ -0,0 +1 @@
+11/06/2021 21:16:43 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_2.log

@@ -0,0 +1 @@
+11/06/2021 21:16:43 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_3.log

@@ -0,0 +1 @@
+11/06/2021 21:16:43 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_4.log

@@ -0,0 +1 @@
+11/06/2021 21:16:43 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_5.log

@@ -0,0 +1 @@
+11/06/2021 21:16:43 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_6.log

@@ -0,0 +1 @@
+11/06/2021 21:16:43 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_7.log

@@ -0,0 +1 @@
+11/06/2021 21:16:43 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_8.log

@@ -0,0 +1 @@
+11/06/2021 21:16:43 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_9.log

@@ -0,0 +1 @@
+11/06/2021 21:16:43 - INFO - root - Reducer buckets have been rebuilt in this iteration.

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a6cf691aab6812b45fb731c3a079344b6d81c2c7abe896e62785b36da74ce16c
-size 456675177
+oid sha256:c4c47486334eae28a88885b4615c541f9cff91bca771fb28047fb8b8aaa1b009
+size 456677609

runs/Nov06_21-16-12_leandro-16x-v100/1636233372.3289735/events.out.tfevents.1636233372.leandro-16x-v100.4368.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0e4c545c8c00c8dd32e5dadf4db351c0ee2811281dd482a24268755c1c39c00
+size 1438

runs/Nov06_21-16-12_leandro-16x-v100/events.out.tfevents.1636233372.leandro-16x-v100.4368.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd3c4ff9e0ad0eff539befc6cfb2e13a36987484ad8ac891c37bb080ceb05d29
+size 2699629

wandb/debug-internal.log

@@ -0,0 +1 @@
+run-20211106_211610-dtkf2u0m/logs/debug-internal.log

wandb/debug.log

@@ -0,0 +1 @@
+run-20211106_211610-dtkf2u0m/logs/debug.log

wandb/latest-run

@@ -0,0 +1 @@
+run-20211106_211610-dtkf2u0m

wandb/run-20211106_211610-dtkf2u0m/files/conda-environment.yaml

@@ -0,0 +1,131 @@
+name: codeparrot
+channels:
+  - pytorch
+  - nvidia
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=4.5=1_gnu
+  - blas=1.0=mkl
+  - bzip2=1.0.8=h7b6447c_0
+  - ca-certificates=2021.7.5=h06a4308_1
+  - certifi=2021.5.30=py38h06a4308_0
+  - cudatoolkit=11.1.74=h6bb024c_0
+  - ffmpeg=4.3=hf484d3e_0
+  - freetype=2.10.4=h5ab3b9f_0
+  - gmp=6.2.1=h2531618_2
+  - gnutls=3.6.15=he1e5248_0
+  - intel-openmp=2021.3.0=h06a4308_3350
+  - jpeg=9b=h024ee3a_2
+  - lame=3.100=h7b6447c_0
+  - lcms2=2.12=h3be6417_0
+  - ld_impl_linux-64=2.35.1=h7274673_9
+  - libffi=3.3=he6710b0_2
+  - libgcc-ng=9.3.0=h5101ec6_17
+  - libgomp=9.3.0=h5101ec6_17
+  - libiconv=1.15=h63c8f33_5
+  - libidn2=2.3.2=h7f8727e_0
+  - libpng=1.6.37=hbc83047_0
+  - libstdcxx-ng=9.3.0=hd4cf53a_17
+  - libtasn1=4.16.0=h27cfd23_0
+  - libtiff=4.2.0=h85742a9_0
+  - libunistring=0.9.10=h27cfd23_0
+  - libuv=1.40.0=h7b6447c_0
+  - libwebp-base=1.2.0=h27cfd23_0
+  - lz4-c=1.9.3=h295c915_1
+  - mkl=2021.3.0=h06a4308_520
+  - mkl-service=2.4.0=py38h7f8727e_0
+  - mkl_fft=1.3.0=py38h42c9631_2
+  - mkl_random=1.2.2=py38h51133e4_0
+  - ncurses=6.2=he6710b0_1
+  - nettle=3.7.3=hbbd107a_1
+  - numpy=1.20.3=py38hf144106_0
+  - numpy-base=1.20.3=py38h74d4b33_0
+  - olefile=0.46=pyhd3eb1b0_0
+  - openh264=2.1.0=hd408876_0
+  - openjpeg=2.4.0=h3ad879b_0
+  - openssl=1.1.1l=h7f8727e_0
+  - pillow=8.3.1=py38h2c7a002_0
+  - pip=21.0.1=py38h06a4308_0
+  - python=3.8.11=h12debd9_0_cpython
+  - pytorch=1.9.0=py3.8_cuda11.1_cudnn8.0.5_0
+  - readline=8.1=h27cfd23_0
+  - setuptools=52.0.0=py38h06a4308_0
+  - six=1.16.0=pyhd3eb1b0_0
+  - sqlite=3.36.0=hc218d9a_0
+  - tk=8.6.10=hbc83047_0
+  - torchaudio=0.9.0=py38
+  - torchvision=0.10.0=py38_cu111
+  - typing_extensions=3.10.0.0=pyhca03da5_0
+  - wheel=0.37.0=pyhd3eb1b0_1
+  - xz=5.2.5=h7b6447c_0
+  - zlib=1.2.11=h7b6447c_3
+  - zstd=1.4.9=haebb681_0
+  - pip:
+    - absl-py==0.13.0
+    - accelerate==0.5.1
+    - aiohttp==3.7.4.post0
+    - async-timeout==3.0.1
+    - attrs==21.2.0
+    - cachetools==4.2.2
+    - chardet==4.0.0
+    - charset-normalizer==2.0.5
+    - click==8.0.1
+    - configparser==5.0.2
+    - datasets==1.13.0
+    - deepspeed==0.5.2
+    - dill==0.3.4
+    - docker-pycreds==0.4.0
+    - filelock==3.0.12
+    - fsspec==2021.8.1
+    - gitdb==4.0.7
+    - gitpython==3.1.18
+    - google-auth==1.35.0
+    - google-auth-oauthlib==0.4.6
+    - grpcio==1.40.0
+    - huggingface-hub==0.0.19
+    - idna==3.2
+    - joblib==1.0.1
+    - markdown==3.3.4
+    - multidict==5.1.0
+    - multiprocess==0.70.12.2
+    - ninja==1.10.2
+    - oauthlib==3.1.1
+    - packaging==21.0
+    - pandas==1.3.3
+    - pathtools==0.1.2
+    - promise==2.3
+    - protobuf==3.18.0
+    - psutil==5.8.0
+    - pyarrow==5.0.0
+    - pyasn1==0.4.8
+    - pyasn1-modules==0.2.8
+    - pyparsing==2.4.7
+    - python-dateutil==2.8.2
+    - pytz==2021.1
+    - pyyaml==5.4.1
+    - regex==2021.8.28
+    - requests==2.26.0
+    - requests-oauthlib==1.3.0
+    - rsa==4.7.2
+    - sacremoses==0.0.45
+    - sentry-sdk==1.3.1
+    - shortuuid==1.0.1
+    - smmap==4.0.0
+    - subprocess32==3.5.4
+    - tensorboard==2.6.0
+    - tensorboard-data-server==0.6.1
+    - tensorboard-plugin-wit==1.8.0
+    - tensorboardx==1.8
+    - termcolor==1.1.0
+    - tokenizers==0.10.3
+    - tqdm==4.62.2
+    - transformers==4.12.2
+    - triton==1.0.0
+    - urllib3==1.26.6
+    - wandb==0.12.2
+    - werkzeug==2.0.1
+    - xxhash==2.0.2
+    - yarl==1.6.3
+    - yaspin==2.1.0
+prefix: /home/leandro/miniconda3/envs/codeparrot

wandb/run-20211106_211610-dtkf2u0m/files/config.yaml

@@ -0,0 +1,92 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.12.2
+    framework: huggingface
+    huggingface_version: 4.12.2
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.8.11
+    start_time: 1636233370
+    t:
+      1:
+      - 1
+      - 11
+      3:
+      - 16
+      4: 3.8.11
+      5: 0.12.2
+      6: 4.12.2
+      8:
+      - 5
+backend:
+  desc: null
+  value: nccl
+deepspeed_plugin:
+  desc: null
+  value: None
+device:
+  desc: null
+  value: cuda:0
+distributed_type:
+  desc: null
+  value: DistributedType.MULTI_GPU
+gradient_accumulation_steps:
+  desc: null
+  value: 1
+gradient_checkpointing:
+  desc: null
+  value: false
+initialized:
+  desc: null
+  value: 'True'
+learning_rate:
+  desc: null
+  value: 0.0005
+local_process_index:
+  desc: null
+  value: '0'
+lr_scheduler_type:
+  desc: null
+  value: cosine
+max_eval_steps:
+  desc: null
+  value: -1
+max_train_steps:
+  desc: null
+  value: 150000
+num_processes:
+  desc: null
+  value: '16'
+num_warmup_steps:
+  desc: null
+  value: 2000
+process_index:
+  desc: null
+  value: '0'
+save_checkpoint_steps:
+  desc: null
+  value: 15000
+seed:
+  desc: null
+  value: 1
+seq_length:
+  desc: null
+  value: 1024
+shuffle_buffer:
+  desc: null
+  value: 1000
+train_batch_size:
+  desc: null
+  value: 12
+use_fp16:
+  desc: null
+  value: 'True'
+valid_batch_size:
+  desc: null
+  value: 12
+weight_decay:
+  desc: null
+  value: 0.1

wandb/run-20211106_211610-dtkf2u0m/files/requirements.txt

@@ -0,0 +1,81 @@
+absl-py==0.13.0
+accelerate==0.5.1
+aiohttp==3.7.4.post0
+async-timeout==3.0.1
+attrs==21.2.0
+cachetools==4.2.2
+certifi==2021.5.30
+chardet==4.0.0
+charset-normalizer==2.0.5
+click==8.0.1
+configparser==5.0.2
+datasets==1.13.0
+deepspeed==0.5.2
+dill==0.3.4
+docker-pycreds==0.4.0
+filelock==3.0.12
+fsspec==2021.8.1
+gitdb==4.0.7
+gitpython==3.1.18
+google-auth-oauthlib==0.4.6
+google-auth==1.35.0
+grpcio==1.40.0
+huggingface-hub==0.0.19
+idna==3.2
+joblib==1.0.1
+markdown==3.3.4
+mkl-fft==1.3.0
+mkl-random==1.2.2
+mkl-service==2.4.0
+multidict==5.1.0
+multiprocess==0.70.12.2
+ninja==1.10.2
+numpy==1.20.3
+oauthlib==3.1.1
+olefile==0.46
+packaging==21.0
+pandas==1.3.3
+pathtools==0.1.2
+pillow==8.3.1
+pip==21.0.1
+promise==2.3
+protobuf==3.18.0
+psutil==5.8.0
+pyarrow==5.0.0
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pyparsing==2.4.7
+python-dateutil==2.8.2
+pytz==2021.1
+pyyaml==5.4.1
+regex==2021.8.28
+requests-oauthlib==1.3.0
+requests==2.26.0
+rsa==4.7.2
+sacremoses==0.0.45
+sentry-sdk==1.3.1
+setuptools==52.0.0.post20210125
+shortuuid==1.0.1
+six==1.16.0
+smmap==4.0.0
+subprocess32==3.5.4
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.0
+tensorboard==2.6.0
+tensorboardx==1.8
+termcolor==1.1.0
+tokenizers==0.10.3
+torch==1.9.0
+torchaudio==0.9.0a0+33b2469
+torchvision==0.10.0
+tqdm==4.62.2
+transformers==4.12.2
+triton==1.0.0
+typing-extensions==3.10.0.0
+urllib3==1.26.6
+wandb==0.12.2
+werkzeug==2.0.1
+wheel==0.37.0
+xxhash==2.0.2
+yarl==1.6.3
+yaspin==2.1.0

wandb/run-20211106_211610-dtkf2u0m/files/wandb-metadata.json

@@ -0,0 +1,24 @@
+{
+    "os": "Linux-5.4.0-1056-gcp-x86_64-with-glibc2.17",
+    "python": "3.8.11",
+    "heartbeatAt": "2021-11-06T21:16:11.096309",
+    "startedAt": "2021-11-06T21:16:10.355683",
+    "docker": null,
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 16,
+    "cpu_count": 96,
+    "cuda": "10.1.243",
+    "args": [],
+    "state": "running",
+    "program": "codeparrot_training.py",
+    "codePath": "codeparrot_training.py",
+    "git": {
+        "remote": "https://huggingface.co/lvwerra/codeparrot-small",
+        "commit": "61c58c14c5a962d6f8a01bb8ce31737bb4092922"
+    },
+    "email": "leandro.vonwerra@gmail.com",
+    "root": "/home/leandro/codeparrot-small",
+    "host": "leandro-16x-v100",
+    "username": "leandro",
+    "executable": "/home/leandro/miniconda3/envs/codeparrot/bin/python"
+}

wandb/run-20211106_211610-dtkf2u0m/files/wandb-summary.json

@@ -0,0 +1 @@
+{"lr": 0.0004905430961831242, "samples": 2880000, "steps": 14999, "loss/train": 0.5816349983215332, "_runtime": 7561, "_timestamp": 1636240931, "_step": 15000, "loss/eval": 1.610079050064087, "perplexity": 5.003206729888916}

wandb/run-20211106_211610-dtkf2u0m/logs/debug.log

@@ -0,0 +1,23 @@
+2021-11-06 21:16:10,357 INFO    MainThread:4368 [wandb_setup.py:_flush():69] setting env: {}
+2021-11-06 21:16:10,357 INFO    MainThread:4368 [wandb_setup.py:_flush():69] setting login settings: {}
+2021-11-06 21:16:10,357 INFO    MainThread:4368 [wandb_init.py:_log_setup():348] Logging user logs to /home/leandro/codeparrot-small/wandb/run-20211106_211610-dtkf2u0m/logs/debug.log
+2021-11-06 21:16:10,358 INFO    MainThread:4368 [wandb_init.py:_log_setup():349] Logging internal logs to /home/leandro/codeparrot-small/wandb/run-20211106_211610-dtkf2u0m/logs/debug-internal.log
+2021-11-06 21:16:10,358 INFO    MainThread:4368 [wandb_init.py:init():381] calling init triggers
+2021-11-06 21:16:10,358 INFO    MainThread:4368 [wandb_init.py:init():386] wandb.init called with sweep_config: {}
+config: {'train_batch_size': 12, 'valid_batch_size': 12, 'weight_decay': 0.1, 'shuffle_buffer': 1000, 'learning_rate': 0.0005, 'lr_scheduler_type': 'cosine', 'num_warmup_steps': 2000, 'gradient_accumulation_steps': 1, 'gradient_checkpointing': False, 'max_train_steps': 150000, 'max_eval_steps': -1, 'seq_length': 1024, 'seed': 1, 'save_checkpoint_steps': 15000, 'backend': 'nccl', 'deepspeed_plugin': 'None', 'distributed_type': 'DistributedType.MULTI_GPU', 'num_processes': '16', 'process_index': '0', 'local_process_index': '0', 'device': 'cuda:0', 'use_fp16': 'True', 'initialized': 'True'}
+2021-11-06 21:16:10,358 INFO    MainThread:4368 [wandb_init.py:init():430] starting backend
+2021-11-06 21:16:10,358 INFO    MainThread:4368 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2021-11-06 21:16:10,378 INFO    MainThread:4368 [backend.py:ensure_launched():135] starting backend process...
+2021-11-06 21:16:10,389 INFO    MainThread:4368 [backend.py:ensure_launched():139] started backend process with pid: 4634
+2021-11-06 21:16:10,391 INFO    MainThread:4368 [wandb_init.py:init():435] backend started and connected
+2021-11-06 21:16:10,396 INFO    MainThread:4368 [wandb_init.py:init():494] updated telemetry
+2021-11-06 21:16:10,397 INFO    MainThread:4368 [wandb_init.py:init():517] communicating current version
+2021-11-06 21:16:10,957 INFO    MainThread:4368 [wandb_init.py:init():522] got version response upgrade_message: "wandb version 0.12.6 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2021-11-06 21:16:10,957 INFO    MainThread:4368 [wandb_init.py:init():530] communicating run to backend with 30 second timeout
+2021-11-06 21:16:11,044 INFO    MainThread:4368 [wandb_init.py:init():557] starting run threads in backend
+2021-11-06 21:16:12,320 INFO    MainThread:4368 [wandb_run.py:_console_start():1605] atexit reg
+2021-11-06 21:16:12,320 INFO    MainThread:4368 [wandb_run.py:_redirect():1479] redirect: SettingsConsole.REDIRECT
+2021-11-06 21:16:12,320 INFO    MainThread:4368 [wandb_run.py:_redirect():1484] Redirecting console.
+2021-11-06 21:16:12,322 INFO    MainThread:4368 [wandb_run.py:_redirect():1540] Redirects installed.
+2021-11-06 21:16:12,323 INFO    MainThread:4368 [wandb_init.py:init():582] run started, returning control to user process