step 50000

.gitattributes

@@ -15,3 +15,6 @@
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+wandb/debug-internal.log filter=lfs diff=lfs merge=lfs -text
+wandb/run-20210920_142810-36cw69uv/logs/debug-internal.log filter=lfs diff=lfs merge=lfs -text
+wandb/run-20210920_142810-36cw69uv/run-36cw69uv.wandb filter=lfs diff=lfs merge=lfs -text

codeparrot_training.py

@@ -12,24 +12,22 @@ from argparse import Namespace
 import torch
 import logging
 import wandb
-import time
-
 
 class ConstantLengthDataset(IterableDataset):
+    
     def __init__(self, tokenizer, dataset, seq_length=1024,
                  num_of_sequences=1024, chars_per_token=3.6):
         self.tokenizer = tokenizer
-        self.concatenation_token_id = tokenizer.bos_token_id
+        self.concat_token_id = tokenizer.bos_token_id
         self.dataset = dataset
         self.seq_length = seq_length
         self.input_characters = seq_length * chars_per_token * num_of_sequences
-        self.produced_samples = 0
+    
     def __iter__(self):
         iterator = iter(self.dataset)
         more_examples = True
         while more_examples:
-            buffer = []
-            buffer_len = 0
+            buffer, buffer_len = [], 0
             while True:
                 if buffer_len >= self.input_characters:
                     break
@@ -42,7 +40,7 @@ class ConstantLengthDataset(IterableDataset):
             tokenized_inputs = tokenizer(buffer, truncation=False)['input_ids']
             all_token_ids = []
             for tokenized_input in tokenized_inputs:
-                all_token_ids.extend(tokenized_input + [self.concatenation_token_id])
+                all_token_ids.extend(tokenized_input + [self.concat_token_id])
             for i in range(0, len(all_token_ids), self.seq_length):
                 input_ids = all_token_ids[i : i + self.seq_length]
                 if len(input_ids) == self.seq_length:
@@ -52,14 +50,16 @@ def setup_logging(project_name):
     logger = logging.getLogger(__name__)
     logging.basicConfig(
         format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
-        datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO,)
+        datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO, handlers=[
+        logging.FileHandler(f"log/debug_{accelerator.process_index}.log"),
+        logging.StreamHandler()])
     if accelerator.is_main_process: # we only want to setup logging once
         wandb.init(project=project_name, config=args)
         run_name = wandb.run.name
         tb_writer = SummaryWriter()
         tb_writer.add_hparams(vars(args), {'0': 0})
         logger.setLevel(logging.INFO)
-        datasets.utils.logging.set_verbosity_warning()
+        datasets.utils.logging.set_verbosity_info()
         transformers.utils.logging.set_verbosity_info()
     else:
         tb_writer = None
@@ -69,13 +69,12 @@ def setup_logging(project_name):
         transformers.utils.logging.set_verbosity_error()
     return logger, tb_writer, run_name
 
-def create_dataloaders(dataset_name):
-    train_data = load_dataset(dataset_name+'-train', split="train",
-                              streaming=True)
+def create_dataloaders(dataset_name, args):
+    ds_kwargs = {"streaming":True, "chunksize":40<<20, "error_bad_chunk":False}
+    train_data = load_dataset(dataset_name+'-train', split='train', **ds_kwargs)
     train_data = train_data.shuffle(buffer_size=args.shuffle_buffer,
                                     seed=args.seed)
-    valid_data = load_dataset(dataset_name+'-valid', split="train",
-                              streaming=True)
+    valid_data = load_dataset(dataset_name+'-valid', split="train", **ds_kwargs)
     train_dataset = ConstantLengthDataset(tokenizer, train_data,
                                           seq_length=args.seq_length)
     valid_dataset = ConstantLengthDataset(tokenizer, valid_data,
@@ -84,7 +83,7 @@ def create_dataloaders(dataset_name):
     eval_dataloader=DataLoader(valid_dataset, batch_size=args.valid_batch_size)
     return train_dataloader, eval_dataloader
 
-def get_grouped_params(model, no_decay=["bias", "LayerNorm.weight"]):
+def get_grouped_params(model, args, no_decay=["bias", "LayerNorm.weight"]):
     params_with_wd, params_without_wd = [], []
     for n, p in model.named_parameters():
         if any(nd in n for nd in no_decay): params_without_wd.append(p)
@@ -98,7 +97,7 @@ def log_metrics(step, metrics):
         wandb.log(metrics)
         [tb_writer.add_scalar(k, v, step) for k, v in metrics.items()]
 
-def evaluate():
+def evaluate(args):
     model.eval()
     losses = []
     for step, batch in enumerate(eval_dataloader):
@@ -112,44 +111,44 @@ def evaluate():
     except OverflowError: perplexity = float("inf")
     return loss.item(), perplexity.item()
 
+# Accelerator
+accelerator = Accelerator(dispatch_batches=True)
+acc_state = {str(k): str(v) for k, v in accelerator.state.__dict__.items()}
 # Hyperparameters
 project_name = 'transformersbook/codeparrot'
-dataset_name = 'transformersbook/codeparrot'
-config = {"train_batch_size": 4,
-          "valid_batch_size": 4,
+dataset_name = '../codeparrot'
+config = {"train_batch_size": 2,
+          "valid_batch_size": 2,
           "weight_decay": 0.1,
-          "shuffle_buffer": 1000,
-          "learning_rate": 5e-4,
+          "shuffle_buffer": 1_000,
+          "learning_rate": 2e-4,
           "lr_scheduler_type": "cosine",
-          "num_warmup_steps": 1000,
-          "gradient_accumulation_steps": 2,
-          "max_train_steps": 24_000,
-          "max_eval_steps": 500,
+          "num_warmup_steps": 750,
+          "gradient_accumulation_steps": 16,
+          "max_train_steps": 50_000,
+          "max_eval_steps": -1,
           "seq_length": 1024,
           "seed": 1,
-          "save_checkpoint_steps":6_000,}
-args = Namespace(**config)
-set_seed(args.seed)
-
-# Accelerator
-accelerator = Accelerator()
+          "save_checkpoint_steps": 50_000}
+args = Namespace(**config, **acc_state)
 samples_per_step = accelerator.state.num_processes * args.train_batch_size
+set_seed(args.seed)
 
 # Logging
 logger, tb_writer, run_name = setup_logging(project_name.split("/")[1])
 logger.info(accelerator.state)
 
 # Load model and tokenizer
-if accelerator.is_main_process: # we only want to setup logging once
+if accelerator.is_main_process:
     hf_repo = Repository("./", clone_from=project_name, revision=run_name)
-model = GPT2LMHeadModel.from_pretrained("./")
+model = GPT2LMHeadModel.from_pretrained("./", gradient_checkpointing=True)
 tokenizer = AutoTokenizer.from_pretrained("./")
 
 # Load dataset and dataloader
-train_dataloader, eval_dataloader = create_dataloaders(dataset_name)
+train_dataloader, eval_dataloader = create_dataloaders(dataset_name, args)
 
 # Prepare the optimizer and learning rate scheduler
-optimizer = AdamW(get_grouped_params(model), lr=args.learning_rate)
+optimizer = AdamW(get_grouped_params(model, args), lr=args.learning_rate)
 lr_scheduler = get_scheduler(name=args.lr_scheduler_type, optimizer=optimizer,
                              num_warmup_steps=args.num_warmup_steps,
                              num_training_steps=args.max_train_steps,)
@@ -162,24 +161,21 @@ model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare(
 # Train model
 model.train()
 completed_steps = 0
-t0 = time.time()
 for step, batch in enumerate(train_dataloader, start=1):
-    t1 = time.time()
-    loss = model(batch, labels=batch).loss
-    t2 = time.time()
+    loss = model(batch, labels=batch, use_cache=False).loss
     log_metrics(step, {'lr': get_lr(), 'samples': step*samples_per_step,
                        'steps': completed_steps, 'loss/train': loss.item()})
     loss = loss / args.gradient_accumulation_steps
     accelerator.backward(loss)
-    t3 = time.time()
     if step % args.gradient_accumulation_steps == 0:
+        accelerator.clip_grad_norm_(model.parameters(), 1.0)
         optimizer.step()
         lr_scheduler.step()
         optimizer.zero_grad()
         completed_steps += 1
     if step % args.save_checkpoint_steps == 0:
         logger.info('Evaluating and saving model checkpoint')
-        eval_loss, perplexity = evaluate()
+        eval_loss, perplexity = evaluate(args)
         log_metrics(step, {'loss/eval': eval_loss, 'perplexity': perplexity})
         accelerator.wait_for_everyone()
         unwrapped_model = accelerator.unwrap_model(model)
@@ -189,17 +185,13 @@ for step, batch in enumerate(train_dataloader, start=1):
         model.train()
     if completed_steps >= args.max_train_steps:
         break
-    t4 = time.time()
-    #logger.info(f'ITER: {t1-t0:.3f}, FRWD: {t2-t1:.3f}, BKWD: {t3-t2:.3f}, OPT: {t4-t3:.3f}, ALL: {t4-t0}')
-    t0 = time.time()
 
 # Evaluate and save the last checkpoint
 logger.info('Evaluating and saving model after training')
-eval_loss, perplexity = evaluate()
+eval_loss, perplexity = evaluate(args)
 log_metrics(step, {'loss/eval': eval_loss, 'perplexity': perplexity})
 accelerator.wait_for_everyone()
 unwrapped_model = accelerator.unwrap_model(model)
 if accelerator.is_main_process:
     unwrapped_model.save_pretrained("./")
-    try: hf_repo.push_to_hub(commit_message=f'final model')
-    except: logger.info('No changes to previously saved model.')
+    hf_repo.push_to_hub(commit_message=f'final model')

config.json

@@ -1,4 +1,5 @@
 {
+  "_name_or_path": "./",
   "activation_function": "gelu_new",
   "architectures": [
     "GPT2LMHeadModel"
@@ -7,7 +8,7 @@
   "bos_token_id": 50256,
   "embd_pdrop": 0.1,
   "eos_token_id": 50256,
-  "gradient_checkpointing": false,
+  "gradient_checkpointing": true,
   "initializer_range": 0.02,
   "layer_norm_epsilon": 1e-05,
   "model_type": "gpt2",

log/debug_1.log

@@ -0,0 +1 @@
+09/20/2021 14:29:09 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_10.log

@@ -0,0 +1 @@
+09/20/2021 14:29:09 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_11.log

@@ -0,0 +1 @@
+09/20/2021 14:29:09 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_12.log

@@ -0,0 +1 @@
+09/20/2021 14:29:09 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_13.log

@@ -0,0 +1 @@
+09/20/2021 14:29:09 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_14.log

@@ -0,0 +1 @@
+09/20/2021 14:29:09 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_15.log

@@ -0,0 +1 @@
+09/20/2021 14:29:09 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_2.log

@@ -0,0 +1 @@
+09/20/2021 14:29:09 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_3.log

@@ -0,0 +1 @@
+09/20/2021 14:29:09 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_4.log

@@ -0,0 +1 @@
+09/20/2021 14:29:09 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_5.log

@@ -0,0 +1 @@
+09/20/2021 14:29:09 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_6.log

@@ -0,0 +1 @@
+09/20/2021 14:29:09 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_7.log

@@ -0,0 +1 @@
+09/20/2021 14:29:09 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_8.log

@@ -0,0 +1 @@
+09/20/2021 14:29:09 - INFO - root - Reducer buckets have been rebuilt in this iteration.

log/debug_9.log

@@ -0,0 +1 @@
+09/20/2021 14:29:09 - INFO - root - Reducer buckets have been rebuilt in this iteration.

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f700cb9a50ef29578ed0b7d885e8b49208fbca9bdb9247ba9852ea6866d92e8b
+oid sha256:00ec35b14b049e5188c1ba8fd432ffa094b481d96393f02052b1c9a9fa4fdc2a
 size 6169094681

requirements.txt

@@ -3,5 +3,5 @@ wandb
 tensorboard
 git+https://github.com/huggingface/huggingface_hub.git
 git+https://github.com/huggingface/transformers.git
-git+https://github.com/huggingface/datasets.git@load_dataset-no-dataset-script
+git+https://github.com/huggingface/datasets.git@json-dont-raise
 git+https://github.com/huggingface/accelerate.git

runs/Sep20_14-28-12_leandro-16x-v100/1632148092.8874874/events.out.tfevents.1632148092.leandro-16x-v100.8660.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:222b05fb22ccb39b7d43f507f7c672d8c741e4281e65c71c12d98b19c1d3ff1f
+size 1373

runs/Sep20_14-28-12_leandro-16x-v100/events.out.tfevents.1632148092.leandro-16x-v100.8660.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:349e549f0e23501888f84c37ff54aff187c6c97313a732fe502a7cf7c77c3a64
+size 9134099

wandb/debug-internal.log

@@ -0,0 +1 @@
+run-20210920_142810-36cw69uv/logs/debug-internal.log

wandb/debug.log

@@ -0,0 +1 @@
+run-20210920_142810-36cw69uv/logs/debug.log

wandb/latest-run

@@ -0,0 +1 @@
+run-20210920_142810-36cw69uv

wandb/run-20210920_142810-36cw69uv/files/conda-environment.yaml

@@ -0,0 +1,131 @@
+name: codeparrot
+channels:
+  - pytorch
+  - nvidia
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=4.5=1_gnu
+  - blas=1.0=mkl
+  - bzip2=1.0.8=h7b6447c_0
+  - ca-certificates=2021.7.5=h06a4308_1
+  - certifi=2021.5.30=py38h06a4308_0
+  - cudatoolkit=11.1.74=h6bb024c_0
+  - ffmpeg=4.3=hf484d3e_0
+  - freetype=2.10.4=h5ab3b9f_0
+  - gmp=6.2.1=h2531618_2
+  - gnutls=3.6.15=he1e5248_0
+  - intel-openmp=2021.3.0=h06a4308_3350
+  - jpeg=9b=h024ee3a_2
+  - lame=3.100=h7b6447c_0
+  - lcms2=2.12=h3be6417_0
+  - ld_impl_linux-64=2.35.1=h7274673_9
+  - libffi=3.3=he6710b0_2
+  - libgcc-ng=9.3.0=h5101ec6_17
+  - libgomp=9.3.0=h5101ec6_17
+  - libiconv=1.15=h63c8f33_5
+  - libidn2=2.3.2=h7f8727e_0
+  - libpng=1.6.37=hbc83047_0
+  - libstdcxx-ng=9.3.0=hd4cf53a_17
+  - libtasn1=4.16.0=h27cfd23_0
+  - libtiff=4.2.0=h85742a9_0
+  - libunistring=0.9.10=h27cfd23_0
+  - libuv=1.40.0=h7b6447c_0
+  - libwebp-base=1.2.0=h27cfd23_0
+  - lz4-c=1.9.3=h295c915_1
+  - mkl=2021.3.0=h06a4308_520
+  - mkl-service=2.4.0=py38h7f8727e_0
+  - mkl_fft=1.3.0=py38h42c9631_2
+  - mkl_random=1.2.2=py38h51133e4_0
+  - ncurses=6.2=he6710b0_1
+  - nettle=3.7.3=hbbd107a_1
+  - numpy=1.20.3=py38hf144106_0
+  - numpy-base=1.20.3=py38h74d4b33_0
+  - olefile=0.46=pyhd3eb1b0_0
+  - openh264=2.1.0=hd408876_0
+  - openjpeg=2.4.0=h3ad879b_0
+  - openssl=1.1.1l=h7f8727e_0
+  - pillow=8.3.1=py38h2c7a002_0
+  - pip=21.0.1=py38h06a4308_0
+  - python=3.8.11=h12debd9_0_cpython
+  - pytorch=1.9.0=py3.8_cuda11.1_cudnn8.0.5_0
+  - readline=8.1=h27cfd23_0
+  - setuptools=52.0.0=py38h06a4308_0
+  - six=1.16.0=pyhd3eb1b0_0
+  - sqlite=3.36.0=hc218d9a_0
+  - tk=8.6.10=hbc83047_0
+  - torchaudio=0.9.0=py38
+  - torchvision=0.10.0=py38_cu111
+  - typing_extensions=3.10.0.0=pyhca03da5_0
+  - wheel=0.37.0=pyhd3eb1b0_1
+  - xz=5.2.5=h7b6447c_0
+  - zlib=1.2.11=h7b6447c_3
+  - zstd=1.4.9=haebb681_0
+  - pip:
+    - absl-py==0.13.0
+    - accelerate==0.5.0.dev0
+    - aiohttp==3.7.4.post0
+    - async-timeout==3.0.1
+    - attrs==21.2.0
+    - cachetools==4.2.2
+    - chardet==4.0.0
+    - charset-normalizer==2.0.5
+    - click==8.0.1
+    - configparser==5.0.2
+    - datasets==1.10.3.dev0
+    - deepspeed==0.5.2
+    - dill==0.3.4
+    - docker-pycreds==0.4.0
+    - filelock==3.0.12
+    - fsspec==2021.8.1
+    - gitdb==4.0.7
+    - gitpython==3.1.18
+    - google-auth==1.35.0
+    - google-auth-oauthlib==0.4.6
+    - grpcio==1.40.0
+    - huggingface-hub==0.0.17
+    - idna==3.2
+    - joblib==1.0.1
+    - markdown==3.3.4
+    - multidict==5.1.0
+    - multiprocess==0.70.12.2
+    - ninja==1.10.2
+    - oauthlib==3.1.1
+    - packaging==21.0
+    - pandas==1.3.3
+    - pathtools==0.1.2
+    - promise==2.3
+    - protobuf==3.18.0
+    - psutil==5.8.0
+    - pyarrow==5.0.0
+    - pyasn1==0.4.8
+    - pyasn1-modules==0.2.8
+    - pyparsing==2.4.7
+    - python-dateutil==2.8.2
+    - pytz==2021.1
+    - pyyaml==5.4.1
+    - regex==2021.8.28
+    - requests==2.26.0
+    - requests-oauthlib==1.3.0
+    - rsa==4.7.2
+    - sacremoses==0.0.45
+    - sentry-sdk==1.3.1
+    - shortuuid==1.0.1
+    - smmap==4.0.0
+    - subprocess32==3.5.4
+    - tensorboard==2.6.0
+    - tensorboard-data-server==0.6.1
+    - tensorboard-plugin-wit==1.8.0
+    - tensorboardx==1.8
+    - termcolor==1.1.0
+    - tokenizers==0.10.3
+    - tqdm==4.62.2
+    - transformers==4.11.0.dev0
+    - triton==1.0.0
+    - urllib3==1.26.6
+    - wandb==0.12.2
+    - werkzeug==2.0.1
+    - xxhash==2.0.2
+    - yarl==1.6.3
+    - yaspin==2.1.0
+prefix: /home/leandro/miniconda3/envs/codeparrot

wandb/run-20210920_142810-36cw69uv/files/config.yaml

@@ -0,0 +1,89 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.12.2
+    framework: huggingface
+    huggingface_version: 4.11.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.8.11
+    start_time: 1632148090
+    t:
+      1:
+      - 1
+      - 11
+      3:
+      - 16
+      4: 3.8.11
+      5: 0.12.2
+      6: 4.11.0.dev0
+      8:
+      - 5
+backend:
+  desc: null
+  value: nccl
+deepspeed_plugin:
+  desc: null
+  value: None
+device:
+  desc: null
+  value: cuda:0
+distributed_type:
+  desc: null
+  value: DistributedType.MULTI_GPU
+gradient_accumulation_steps:
+  desc: null
+  value: 16
+initialized:
+  desc: null
+  value: 'True'
+learning_rate:
+  desc: null
+  value: 0.0002
+local_process_index:
+  desc: null
+  value: '0'
+lr_scheduler_type:
+  desc: null
+  value: cosine
+max_eval_steps:
+  desc: null
+  value: -1
+max_train_steps:
+  desc: null
+  value: 50000
+num_processes:
+  desc: null
+  value: '16'
+num_warmup_steps:
+  desc: null
+  value: 750
+process_index:
+  desc: null
+  value: '0'
+save_checkpoint_steps:
+  desc: null
+  value: 50000
+seed:
+  desc: null
+  value: 1
+seq_length:
+  desc: null
+  value: 1024
+shuffle_buffer:
+  desc: null
+  value: 1000
+train_batch_size:
+  desc: null
+  value: 2
+use_fp16:
+  desc: null
+  value: 'True'
+valid_batch_size:
+  desc: null
+  value: 2
+weight_decay:
+  desc: null
+  value: 0.1

wandb/run-20210920_142810-36cw69uv/files/requirements.txt

@@ -0,0 +1,81 @@
+absl-py==0.13.0
+accelerate==0.5.0.dev0
+aiohttp==3.7.4.post0
+async-timeout==3.0.1
+attrs==21.2.0
+cachetools==4.2.2
+certifi==2021.5.30
+chardet==4.0.0
+charset-normalizer==2.0.5
+click==8.0.1
+configparser==5.0.2
+datasets==1.10.3.dev0
+deepspeed==0.5.2
+dill==0.3.4
+docker-pycreds==0.4.0
+filelock==3.0.12
+fsspec==2021.8.1
+gitdb==4.0.7
+gitpython==3.1.18
+google-auth-oauthlib==0.4.6
+google-auth==1.35.0
+grpcio==1.40.0
+huggingface-hub==0.0.17
+idna==3.2
+joblib==1.0.1
+markdown==3.3.4
+mkl-fft==1.3.0
+mkl-random==1.2.2
+mkl-service==2.4.0
+multidict==5.1.0
+multiprocess==0.70.12.2
+ninja==1.10.2
+numpy==1.20.3
+oauthlib==3.1.1
+olefile==0.46
+packaging==21.0
+pandas==1.3.3
+pathtools==0.1.2
+pillow==8.3.1
+pip==21.0.1
+promise==2.3
+protobuf==3.18.0
+psutil==5.8.0
+pyarrow==5.0.0
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pyparsing==2.4.7
+python-dateutil==2.8.2
+pytz==2021.1
+pyyaml==5.4.1
+regex==2021.8.28
+requests-oauthlib==1.3.0
+requests==2.26.0
+rsa==4.7.2
+sacremoses==0.0.45
+sentry-sdk==1.3.1
+setuptools==52.0.0.post20210125
+shortuuid==1.0.1
+six==1.16.0
+smmap==4.0.0
+subprocess32==3.5.4
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.0
+tensorboard==2.6.0
+tensorboardx==1.8
+termcolor==1.1.0
+tokenizers==0.10.3
+torch==1.9.0
+torchaudio==0.9.0a0+33b2469
+torchvision==0.10.0
+tqdm==4.62.2
+transformers==4.11.0.dev0
+triton==1.0.0
+typing-extensions==3.10.0.0
+urllib3==1.26.6
+wandb==0.12.2
+werkzeug==2.0.1
+wheel==0.37.0
+xxhash==2.0.2
+yarl==1.6.3
+yaspin==2.1.0

wandb/run-20210920_142810-36cw69uv/files/wandb-metadata.json

@@ -0,0 +1,24 @@
+{
+    "os": "Linux-5.4.0-1052-gcp-x86_64-with-glibc2.17",
+    "python": "3.8.11",
+    "heartbeatAt": "2021-09-20T14:28:11.537999",
+    "startedAt": "2021-09-20T14:28:10.785470",
+    "docker": null,
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 16,
+    "cpu_count": 96,
+    "cuda": "10.1.243",
+    "args": [],
+    "state": "running",
+    "program": "codeparrot_training.py",
+    "codePath": "codeparrot_training.py",
+    "git": {
+        "remote": "https://huggingface.co/transformersbook/codeparrot",
+        "commit": "ea70f93cfbf64eb723d41b350d14827e68b0a6c3"
+    },
+    "email": "leandro.vonwerra@gmail.com",
+    "root": "/home/leandro/codeparrot",
+    "host": "leandro-16x-v100",
+    "username": "leandro",
+    "executable": "/home/leandro/miniconda3/envs/codeparrot/bin/python"
+}

wandb/run-20210920_142810-36cw69uv/files/wandb-summary.json

@@ -0,0 +1 @@
+{"lr": 0.00019885557166781018, "samples": 1600000, "steps": 3124, "loss/train": 1.4012274742126465, "_runtime": 40644, "_timestamp": 1632188734, "_step": 50000, "loss/eval": 1.7745720148086548, "perplexity": 5.897756576538086}

wandb/run-20210920_142810-36cw69uv/logs/debug-internal.log

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4b2174d10551473549ba5ce66d8348228b83bacf57bbc30dca02a46c5e0319c
+size 26678411

wandb/run-20210920_142810-36cw69uv/logs/debug.log

@@ -0,0 +1,22 @@
+2021-09-20 14:28:10,787 INFO    MainThread:8660 [wandb_setup.py:_flush():69] setting env: {}
+2021-09-20 14:28:10,787 INFO    MainThread:8660 [wandb_setup.py:_flush():69] setting login settings: {}
+2021-09-20 14:28:10,787 INFO    MainThread:8660 [wandb_init.py:_log_setup():348] Logging user logs to /home/leandro/codeparrot/wandb/run-20210920_142810-36cw69uv/logs/debug.log
+2021-09-20 14:28:10,787 INFO    MainThread:8660 [wandb_init.py:_log_setup():349] Logging internal logs to /home/leandro/codeparrot/wandb/run-20210920_142810-36cw69uv/logs/debug-internal.log
+2021-09-20 14:28:10,788 INFO    MainThread:8660 [wandb_init.py:init():381] calling init triggers
+2021-09-20 14:28:10,788 INFO    MainThread:8660 [wandb_init.py:init():386] wandb.init called with sweep_config: {}
+config: {'train_batch_size': 2, 'valid_batch_size': 2, 'weight_decay': 0.1, 'shuffle_buffer': 1000, 'learning_rate': 0.0002, 'lr_scheduler_type': 'cosine', 'num_warmup_steps': 750, 'gradient_accumulation_steps': 16, 'max_train_steps': 50000, 'max_eval_steps': -1, 'seq_length': 1024, 'seed': 1, 'save_checkpoint_steps': 50000, 'backend': 'nccl', 'deepspeed_plugin': 'None', 'distributed_type': 'DistributedType.MULTI_GPU', 'num_processes': '16', 'process_index': '0', 'local_process_index': '0', 'device': 'cuda:0', 'use_fp16': 'True', 'initialized': 'True'}
+2021-09-20 14:28:10,788 INFO    MainThread:8660 [wandb_init.py:init():430] starting backend
+2021-09-20 14:28:10,788 INFO    MainThread:8660 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2021-09-20 14:28:10,805 INFO    MainThread:8660 [backend.py:ensure_launched():135] starting backend process...
+2021-09-20 14:28:10,816 INFO    MainThread:8660 [backend.py:ensure_launched():139] started backend process with pid: 9038
+2021-09-20 14:28:10,818 INFO    MainThread:8660 [wandb_init.py:init():435] backend started and connected
+2021-09-20 14:28:10,825 INFO    MainThread:8660 [wandb_init.py:init():494] updated telemetry
+2021-09-20 14:28:10,826 INFO    MainThread:8660 [wandb_init.py:init():517] communicating current version
+2021-09-20 14:28:11,406 INFO    MainThread:8660 [wandb_init.py:init():522] got version response 
+2021-09-20 14:28:11,406 INFO    MainThread:8660 [wandb_init.py:init():530] communicating run to backend with 30 second timeout
+2021-09-20 14:28:11,486 INFO    MainThread:8660 [wandb_init.py:init():557] starting run threads in backend
+2021-09-20 14:28:12,872 INFO    MainThread:8660 [wandb_run.py:_console_start():1605] atexit reg
+2021-09-20 14:28:12,873 INFO    MainThread:8660 [wandb_run.py:_redirect():1479] redirect: SettingsConsole.REDIRECT
+2021-09-20 14:28:12,873 INFO    MainThread:8660 [wandb_run.py:_redirect():1484] Redirecting console.
+2021-09-20 14:28:12,876 INFO    MainThread:8660 [wandb_run.py:_redirect():1540] Redirects installed.
+2021-09-20 14:28:12,876 INFO    MainThread:8660 [wandb_init.py:init():582] run started, returning control to user process

wandb/run-20210920_142810-36cw69uv/run-36cw69uv.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3dea2a070cd3c7d6079d138e9461283968f789819a375be8fd99762250f9064
+size 20083529