Training in progress, epoch 1

config_full.yaml

@@ -22,25 +22,24 @@ gradient_checkpointing: true
 gradient_checkpointing_kwargs:
   use_reentrant: False
 hub_strategy: every_save
-learning_rate: 2.0e-05
+learning_rate: 3.0e-04
 log_level: info
-logging_steps: 5  
+logging_steps: 10
 logging_strategy: steps
 lr_scheduler_type: cosine
 max_seq_length: 2048
 max_steps: -1
-num_train_epochs: 1
+num_train_epochs: 5
 output_dir: ./
 overwrite_output_dir: true
-per_device_eval_batch_size: 8
-per_device_train_batch_size: 16
+per_device_eval_batch_size: 32
+per_device_train_batch_size: 64
 push_to_hub: true
 remove_unused_columns: true
 report_to:
 - tensorboard
 - wandb
-save_strategy: "steps"
-save_steps: 100
+save_strategy: "epoch"
 save_total_limit: 1
 seed: 42
-warmup_ratio: 0.1
+warmup_ratio: 0.1

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b92d043a09b94d7a22c478b47119a6bad9b5e914d6e9d6e6a566e2c9cc0a2be
+oid sha256:b36e69ffbe5ae42f8478ebfbc7c07012a026dc6fbd9407f024781cecd7b9d168
 size 3141646744

runs/Feb01_17-38-02_ip-26-0-165-24/events.out.tfevents.1706809106.ip-26-0-165-24.237059.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:150e10d36280c0b789cf11077e52e9dfc0006692f1a394589a62f320ce274b43
-size 32714
+oid sha256:d9a0969790c64d9b70fd83ebdd3bc06a1dee3d99e192778e00b366d5ec468d17
+size 36796

runs/Feb01_17-58-13_ip-26-0-165-24/events.out.tfevents.1706810328.ip-26-0-165-24.239318.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6335f8b18c2d252b1614b2ce15bdaae4fa5dfa041da34f64c37eeab8379c14b0
+size 9006

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:85527db3b58440cee04521e4eb775e7259e847cd1a669eaf7502bcc6b6feb0ca
+oid sha256:3862821a4246455e69d6e9d62177958c9fe5579bd61be2e28572b81437bb7922
 size 5816

wandb/debug.log

@@ -1,28 +1,28 @@
-2024-02-01 17:38:28,434 INFO    MainThread:237059 [wandb_setup.py:_flush():76] Current SDK version is 0.16.1
-2024-02-01 17:38:28,434 INFO    MainThread:237059 [wandb_setup.py:_flush():76] Configure stats pid to 237059
-2024-02-01 17:38:28,434 INFO    MainThread:237059 [wandb_setup.py:_flush():76] Loading settings from /admin/home/sanchit/.config/wandb/settings
-2024-02-01 17:38:28,434 INFO    MainThread:237059 [wandb_setup.py:_flush():76] Loading settings from /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/settings
-2024-02-01 17:38:28,434 INFO    MainThread:237059 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
-2024-02-01 17:38:28,434 INFO    MainThread:237059 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
-2024-02-01 17:38:28,434 INFO    MainThread:237059 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'run_sft.py', 'program_abspath': '/fsx/sanchit/distil-zephyr-1.5b-ssft/run_sft.py', 'program': '/fsx/sanchit/distil-zephyr-1.5b-ssft/run_sft.py'}
-2024-02-01 17:38:28,435 INFO    MainThread:237059 [wandb_init.py:_log_setup():524] Logging user logs to /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/logs/debug.log
-2024-02-01 17:38:28,435 INFO    MainThread:237059 [wandb_init.py:_log_setup():525] Logging internal logs to /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/logs/debug-internal.log
-2024-02-01 17:38:28,435 INFO    MainThread:237059 [wandb_init.py:init():564] calling init triggers
-2024-02-01 17:38:28,435 INFO    MainThread:237059 [wandb_init.py:init():571] wandb.init called with sweep_config: {}
+2024-02-01 17:58:50,075 INFO    MainThread:239318 [wandb_setup.py:_flush():76] Current SDK version is 0.16.1
+2024-02-01 17:58:50,075 INFO    MainThread:239318 [wandb_setup.py:_flush():76] Configure stats pid to 239318
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_setup.py:_flush():76] Loading settings from /admin/home/sanchit/.config/wandb/settings
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_setup.py:_flush():76] Loading settings from /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/settings
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'run_sft.py', 'program_abspath': '/fsx/sanchit/distil-zephyr-1.5b-ssft/run_sft.py', 'program': '/fsx/sanchit/distil-zephyr-1.5b-ssft/run_sft.py'}
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_init.py:_log_setup():524] Logging user logs to /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/logs/debug.log
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_init.py:_log_setup():525] Logging internal logs to /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/logs/debug-internal.log
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_init.py:init():564] calling init triggers
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_init.py:init():571] wandb.init called with sweep_config: {}
 config: {}
-2024-02-01 17:38:28,435 INFO    MainThread:237059 [wandb_init.py:init():614] starting backend
-2024-02-01 17:38:28,435 INFO    MainThread:237059 [wandb_init.py:init():618] setting up manager
-2024-02-01 17:38:28,441 INFO    MainThread:237059 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
-2024-02-01 17:38:28,451 INFO    MainThread:237059 [wandb_init.py:init():624] backend started and connected
-2024-02-01 17:38:28,453 INFO    MainThread:237059 [wandb_init.py:init():716] updated telemetry
-2024-02-01 17:38:28,475 INFO    MainThread:237059 [wandb_init.py:init():749] communicating run to backend with 90.0 second timeout
-2024-02-01 17:38:28,720 INFO    MainThread:237059 [wandb_run.py:_on_init():2254] communicating current version
-2024-02-01 17:38:28,767 INFO    MainThread:237059 [wandb_run.py:_on_init():2263] got version response upgrade_message: "wandb version 0.16.2 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_init.py:init():614] starting backend
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_init.py:init():618] setting up manager
+2024-02-01 17:58:50,079 INFO    MainThread:239318 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-02-01 17:58:50,083 INFO    MainThread:239318 [wandb_init.py:init():624] backend started and connected
+2024-02-01 17:58:50,087 INFO    MainThread:239318 [wandb_init.py:init():716] updated telemetry
+2024-02-01 17:58:50,107 INFO    MainThread:239318 [wandb_init.py:init():749] communicating run to backend with 90.0 second timeout
+2024-02-01 17:58:50,409 INFO    MainThread:239318 [wandb_run.py:_on_init():2254] communicating current version
+2024-02-01 17:58:50,456 INFO    MainThread:239318 [wandb_run.py:_on_init():2263] got version response upgrade_message: "wandb version 0.16.2 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
 
-2024-02-01 17:38:28,767 INFO    MainThread:237059 [wandb_init.py:init():800] starting run threads in backend
-2024-02-01 17:38:34,465 INFO    MainThread:237059 [wandb_run.py:_console_start():2233] atexit reg
-2024-02-01 17:38:34,465 INFO    MainThread:237059 [wandb_run.py:_redirect():2088] redirect: wrap_raw
-2024-02-01 17:38:34,466 INFO    MainThread:237059 [wandb_run.py:_redirect():2153] Wrapping output streams.
-2024-02-01 17:38:34,466 INFO    MainThread:237059 [wandb_run.py:_redirect():2178] Redirects installed.
-2024-02-01 17:38:34,467 INFO    MainThread:237059 [wandb_init.py:init():841] run started, returning control to user process
-2024-02-01 17:38:34,468 INFO    MainThread:237059 [wandb_run.py:_config_callback():1342] config_cb None None {'vocab_size': 32000, 'max_position_embeddings': 32768, 'hidden_size': 4096, 'intermediate_size': 14336, 'num_hidden_layers': 6, 'num_attention_heads': 32, 'sliding_window': 4096, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'use_cache': False, 'rope_theta': 10000.0, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['MistralForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 1, 'pad_token_id': None, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'sanchit-gandhi/Mistral-7B-v0.1-6-layer', 'transformers_version': '4.36.2', 'model_type': 'mistral', 'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'epoch', 'prediction_loss_only': False, 'per_device_train_batch_size': 16, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 2e-05, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.1, 'warmup_steps': 0, 'log_level': 'info', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './runs/Feb01_17-38-02_ip-26-0-165-24', 'logging_strategy': 'steps', 'logging_first_step': True, 'logging_steps': 5, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 100, 'save_total_limit': 1, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': {'use_reentrant': False}, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': False, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'max_seq_length': 2048}
+2024-02-01 17:58:50,456 INFO    MainThread:239318 [wandb_init.py:init():800] starting run threads in backend
+2024-02-01 17:58:55,347 INFO    MainThread:239318 [wandb_run.py:_console_start():2233] atexit reg
+2024-02-01 17:58:55,347 INFO    MainThread:239318 [wandb_run.py:_redirect():2088] redirect: wrap_raw
+2024-02-01 17:58:55,348 INFO    MainThread:239318 [wandb_run.py:_redirect():2153] Wrapping output streams.
+2024-02-01 17:58:55,348 INFO    MainThread:239318 [wandb_run.py:_redirect():2178] Redirects installed.
+2024-02-01 17:58:55,348 INFO    MainThread:239318 [wandb_init.py:init():841] run started, returning control to user process
+2024-02-01 17:58:55,349 INFO    MainThread:239318 [wandb_run.py:_config_callback():1342] config_cb None None {'vocab_size': 32000, 'max_position_embeddings': 32768, 'hidden_size': 4096, 'intermediate_size': 14336, 'num_hidden_layers': 6, 'num_attention_heads': 32, 'sliding_window': 4096, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'use_cache': False, 'rope_theta': 10000.0, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['MistralForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 1, 'pad_token_id': None, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'sanchit-gandhi/Mistral-7B-v0.1-6-layer', 'transformers_version': '4.36.2', 'model_type': 'mistral', 'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'epoch', 'prediction_loss_only': False, 'per_device_train_batch_size': 64, 'per_device_eval_batch_size': 32, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 0.0003, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.1, 'warmup_steps': 0, 'log_level': 'info', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './runs/Feb01_17-58-13_ip-26-0-165-24', 'logging_strategy': 'steps', 'logging_first_step': True, 'logging_steps': 10, 'logging_nan_inf_filter': True, 'save_strategy': 'epoch', 'save_steps': 500, 'save_total_limit': 1, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': {'use_reentrant': False}, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': False, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'max_seq_length': 2048}

wandb/run-20240201_173828-py26nu6m/files/output.log

@@ -800,3 +800,66 @@
 
 
  88%|██████████████████████████████████████████████████████████████████████▋         | 963/1090 [17:05<01:57,  1.08it/s]
+
+
+
+ 89%|███████████████████████████████████████████████████████████████████████▏        | 970/1090 [17:11<01:51,  1.08it/s]
+
+
+ 89%|███████████████████████████████████████████████████████████████████████▍        | 974/1090 [17:15<01:47,  1.08it/s]
+
+
+ 90%|███████████████████████████████████████████████████████████████████████▊        | 979/1090 [17:20<01:42,  1.08it/s]
+
+
+
+ 90%|████████████████████████████████████████████████████████████████████████▎       | 985/1090 [17:25<01:37,  1.08it/s]
+
+
+ 91%|████████████████████████████████████████████████████████████████████████▌       | 989/1090 [17:29<01:33,  1.08it/s]
+
+
+ 91%|████████████████████████████████████████████████████████████████████████▉       | 994/1090 [17:33<01:28,  1.08it/s]
+
+
+
+ 92%|████████████████████████████████████████████████████████████████████████▍      | 1000/1090 [17:39<01:23,  1.08it/s]
+ 92%|████████████████████████████████████████████████████████████████████████▍      | 1000/1090 [17:39<01:23,  1.08it/s][INFO|trainer.py:2889] 2024-02-01 17:56:15,101 >> Saving model checkpoint to ./tmp-checkpoint-1000
+[INFO|configuration_utils.py:483] 2024-02-01 17:56:15,105 >> Configuration saved in ./tmp-checkpoint-1000/config.json
+[INFO|configuration_utils.py:594] 2024-02-01 17:56:15,107 >> Configuration saved in ./tmp-checkpoint-1000/generation_config.json
+[INFO|modeling_utils.py:2382] 2024-02-01 17:56:18,392 >> Model weights saved in ./tmp-checkpoint-1000/pytorch_model.bin
+[INFO|tokenization_utils_base.py:2432] 2024-02-01 17:56:18,395 >> tokenizer config file saved in ./tmp-checkpoint-1000/tokenizer_config.json
+[INFO|tokenization_utils_base.py:2441] 2024-02-01 17:56:18,397 >> Special tokens file saved in ./tmp-checkpoint-1000/special_tokens_map.json
+/fsx/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1879: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.
+  warnings.warn(
+[2024-02-01 17:56:18,423] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step1000 is about to be saved!
+[2024-02-01 17:56:18,427] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./tmp-checkpoint-1000/global_step1000/zero_pp_rank_0_mp_rank_00_model_states.pt
+[2024-02-01 17:56:18,427] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./tmp-checkpoint-1000/global_step1000/zero_pp_rank_0_mp_rank_00_model_states.pt...
+[2024-02-01 17:56:18,432] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./tmp-checkpoint-1000/global_step1000/zero_pp_rank_0_mp_rank_00_model_states.pt.
+[2024-02-01 17:56:18,439] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./tmp-checkpoint-1000/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2024-02-01 17:56:22,348] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./tmp-checkpoint-1000/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2024-02-01 17:56:22,353] [INFO] [engine.py:3393:_save_zero_checkpoint] zero checkpoint saved ./tmp-checkpoint-1000/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2024-02-01 17:56:22,553] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step1000 is ready now!
+[INFO|tokenization_utils_base.py:2432] 2024-02-01 17:56:25,394 >> tokenizer config file saved in ./tokenizer_config.json
+[INFO|tokenization_utils_base.py:2441] 2024-02-01 17:56:25,396 >> Special tokens file saved in ./special_tokens_map.json
+[INFO|trainer.py:2979] 2024-02-01 17:56:25,422 >> Deleting older checkpoint [checkpoint-900] due to args.save_total_limit
+
+ 92%|████████████████████████████████████████████████████████████████████████▋      | 1003/1090 [17:57<04:31,  3.12s/it]
+
+
+
+ 93%|█████████████████████████████████████████████████████████████████████████▏     | 1010/1090 [18:03<01:28,  1.10s/it]
+
+
+ 93%|█████████████████████████████████████████████████████████████████████████▍     | 1014/1090 [18:07<01:13,  1.03it/s]
+
+
+ 93%|█████████████████████████████████████████████████████████████████████████▊     | 1019/1090 [18:12<01:06,  1.07it/s]
+
+
+
+ 94%|██████████████████████████████████████████████████████████████████████████▎    | 1025/1090 [18:17<01:00,  1.08it/s]
+
+
+ 94%|██████████████████████████████████████████████████████████████████████████▌    | 1029/1090 [18:21<00:56,  1.08it/s]
+

wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 2.475, "train/learning_rate": 7.905777244947954e-07, "train/epoch": 0.89, "train/global_step": 965, "_timestamp": 1706810141.6438708, "_runtime": 1033.1919968128204, "_step": 193}
+{"train/loss": 2.4478, "train/learning_rate": 1.840338184455881e-07, "train/epoch": 0.94, "train/global_step": 1030, "_timestamp": 1706810216.6790407, "_runtime": 1108.2271666526794, "_step": 206}

wandb/run-20240201_173828-py26nu6m/logs/debug-internal.log

@@ -1933,3 +1933,144 @@
 2024-02-01 17:55:41,645 DEBUG   SenderThread:237521 [sender.py:send():382] send: history
 2024-02-01 17:55:41,645 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: summary_record
 2024-02-01 17:55:41,647 INFO    SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-01 17:55:42,330 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json
+2024-02-01 17:55:42,330 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:55:43,503 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report
+2024-02-01 17:55:45,335 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:55:46,281 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history
+2024-02-01 17:55:46,282 DEBUG   SenderThread:237521 [sender.py:send():382] send: history
+2024-02-01 17:55:46,282 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: summary_record
+2024-02-01 17:55:46,284 INFO    SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-01 17:55:46,337 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json
+2024-02-01 17:55:47,338 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:55:48,339 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:55:49,056 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report
+2024-02-01 17:55:50,906 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history
+2024-02-01 17:55:50,907 DEBUG   SenderThread:237521 [sender.py:send():382] send: history
+2024-02-01 17:55:50,907 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: summary_record
+2024-02-01 17:55:50,909 INFO    SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-01 17:55:50,992 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages
+2024-02-01 17:55:51,001 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status
+2024-02-01 17:55:51,001 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: stop_status
+2024-02-01 17:55:51,344 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json
+2024-02-01 17:55:51,344 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:55:52,345 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:55:54,600 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report
+2024-02-01 17:55:55,349 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:55:55,524 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history
+2024-02-01 17:55:55,525 DEBUG   SenderThread:237521 [sender.py:send():382] send: history
+2024-02-01 17:55:55,526 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: summary_record
+2024-02-01 17:55:55,527 INFO    SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-01 17:55:56,351 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json
+2024-02-01 17:55:56,352 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:55:58,964 DEBUG   SenderThread:237521 [sender.py:send():382] send: stats
+2024-02-01 17:55:59,355 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:00,151 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report
+2024-02-01 17:56:00,153 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history
+2024-02-01 17:56:00,154 DEBUG   SenderThread:237521 [sender.py:send():382] send: history
+2024-02-01 17:56:00,154 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: summary_record
+2024-02-01 17:56:00,156 INFO    SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-01 17:56:00,358 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json
+2024-02-01 17:56:01,359 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:02,360 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:04,771 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history
+2024-02-01 17:56:04,772 DEBUG   SenderThread:237521 [sender.py:send():382] send: history
+2024-02-01 17:56:04,772 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: summary_record
+2024-02-01 17:56:04,774 INFO    SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-01 17:56:05,365 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json
+2024-02-01 17:56:05,365 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:05,693 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report
+2024-02-01 17:56:05,992 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages
+2024-02-01 17:56:06,001 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status
+2024-02-01 17:56:06,001 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: stop_status
+2024-02-01 17:56:06,366 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:09,370 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:09,385 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history
+2024-02-01 17:56:09,387 DEBUG   SenderThread:237521 [sender.py:send():382] send: history
+2024-02-01 17:56:09,387 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: summary_record
+2024-02-01 17:56:09,388 INFO    SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-01 17:56:10,373 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json
+2024-02-01 17:56:10,373 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:11,235 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report
+2024-02-01 17:56:13,376 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:14,003 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history
+2024-02-01 17:56:14,004 DEBUG   SenderThread:237521 [sender.py:send():382] send: history
+2024-02-01 17:56:14,005 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: summary_record
+2024-02-01 17:56:14,007 INFO    SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-01 17:56:14,379 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json
+2024-02-01 17:56:15,380 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:16,381 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:17,108 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report
+2024-02-01 17:56:19,385 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:20,388 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:22,348 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages
+2024-02-01 17:56:22,349 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report
+2024-02-01 17:56:22,349 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status
+2024-02-01 17:56:22,350 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: stop_status
+2024-02-01 17:56:24,393 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:27,397 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:27,423 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report
+2024-02-01 17:56:28,965 DEBUG   SenderThread:237521 [sender.py:send():382] send: stats
+2024-02-01 17:56:31,402 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:32,636 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report
+2024-02-01 17:56:33,405 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:33,565 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history
+2024-02-01 17:56:33,567 DEBUG   SenderThread:237521 [sender.py:send():382] send: history
+2024-02-01 17:56:33,567 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: summary_record
+2024-02-01 17:56:33,569 INFO    SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-01 17:56:34,407 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json
+2024-02-01 17:56:34,407 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:36,403 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages
+2024-02-01 17:56:36,403 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status
+2024-02-01 17:56:36,403 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: stop_status
+2024-02-01 17:56:37,412 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:38,179 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report
+2024-02-01 17:56:38,180 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history
+2024-02-01 17:56:38,181 DEBUG   SenderThread:237521 [sender.py:send():382] send: history
+2024-02-01 17:56:38,181 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: summary_record
+2024-02-01 17:56:38,183 INFO    SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-01 17:56:38,414 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json
+2024-02-01 17:56:39,415 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:40,416 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:42,804 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history
+2024-02-01 17:56:42,805 DEBUG   SenderThread:237521 [sender.py:send():382] send: history
+2024-02-01 17:56:42,806 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: summary_record
+2024-02-01 17:56:42,808 INFO    SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-01 17:56:43,421 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json
+2024-02-01 17:56:43,421 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:43,727 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report
+2024-02-01 17:56:44,423 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:47,427 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:47,430 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history
+2024-02-01 17:56:47,432 DEBUG   SenderThread:237521 [sender.py:send():382] send: history
+2024-02-01 17:56:47,432 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: summary_record
+2024-02-01 17:56:47,434 INFO    SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-01 17:56:48,429 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json
+2024-02-01 17:56:48,429 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:49,277 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report
+2024-02-01 17:56:51,403 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: internal_messages
+2024-02-01 17:56:51,403 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: stop_status
+2024-02-01 17:56:51,404 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: stop_status
+2024-02-01 17:56:51,432 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:52,098 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history
+2024-02-01 17:56:52,100 DEBUG   SenderThread:237521 [sender.py:send():382] send: history
+2024-02-01 17:56:52,100 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: summary_record
+2024-02-01 17:56:52,102 INFO    SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-01 17:56:52,435 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json
+2024-02-01 17:56:53,436 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:54,437 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:54,828 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report
+2024-02-01 17:56:56,679 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: partial_history
+2024-02-01 17:56:56,680 DEBUG   SenderThread:237521 [sender.py:send():382] send: history
+2024-02-01 17:56:56,681 DEBUG   SenderThread:237521 [sender.py:send_request():409] send_request: summary_record
+2024-02-01 17:56:56,683 INFO    SenderThread:237521 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end
+2024-02-01 17:56:57,442 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/wandb-summary.json
+2024-02-01 17:56:57,443 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:58,444 INFO    Thread-12 :237521 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_173828-py26nu6m/files/output.log
+2024-02-01 17:56:58,968 DEBUG   SenderThread:237521 [sender.py:send():382] send: stats
+2024-02-01 17:56:59,014 INFO    memory    :237521 [interfaces.py:monitor():140] Process proc.memory.rssMB has exited.
+2024-02-01 17:56:59,015 DEBUG   SystemMonitor:237521 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-02-01 17:56:59,015 DEBUG   SystemMonitor:237521 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-02-01 17:56:59,026 DEBUG   SenderThread:237521 [sender.py:send():382] send: stats
+2024-02-01 17:56:59,959 INFO    MainThread:237521 [internal.py:handle_exit():76] Internal process exited
+2024-02-01 17:57:00,027 DEBUG   HandlerThread:237521 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240201_175850-i93q0p12/files/conda-environment.yaml

@@ -0,0 +1,218 @@
+name: venv
+channels:
+  - pytorch
+  - nvidia
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - blas=1.0=mkl
+  - brotli-python=1.0.9=py311h6a678d5_7
+  - bzip2=1.0.8=h7b6447c_0
+  - ca-certificates=2023.12.12=h06a4308_0
+  - certifi=2023.11.17=py311h06a4308_0
+  - cffi=1.16.0=py311h5eee18b_0
+  - cryptography=41.0.7=py311hdda0065_0
+  - cuda-cudart=12.1.105=0
+  - cuda-cupti=12.1.105=0
+  - cuda-libraries=12.1.0=0
+  - cuda-nvrtc=12.1.105=0
+  - cuda-nvtx=12.1.105=0
+  - cuda-opencl=12.3.101=0
+  - cuda-runtime=12.1.0=0
+  - ffmpeg=4.3=hf484d3e_0
+  - filelock=3.13.1=py311h06a4308_0
+  - freetype=2.12.1=h4a9f257_0
+  - giflib=5.2.1=h5eee18b_3
+  - gmp=6.2.1=h295c915_3
+  - gmpy2=2.1.2=py311hc9b5ff0_0
+  - gnutls=3.6.15=he1e5248_0
+  - intel-openmp=2023.1.0=hdb19cb5_46306
+  - jinja2=3.1.2=py311h06a4308_0
+  - jpeg=9e=h5eee18b_1
+  - lame=3.100=h7b6447c_0
+  - lcms2=2.12=h3be6417_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - lerc=3.0=h295c915_0
+  - libcublas=12.1.0.26=0
+  - libcufft=11.0.2.4=0
+  - libcufile=1.8.1.2=0
+  - libcurand=10.3.4.101=0
+  - libcusolver=11.4.4.55=0
+  - libcusparse=12.0.2.55=0
+  - libdeflate=1.17=h5eee18b_1
+  - libffi=3.4.4=h6a678d5_0
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libiconv=1.16=h7f8727e_2
+  - libidn2=2.3.4=h5eee18b_0
+  - libjpeg-turbo=2.0.0=h9bf148f_0
+  - libnpp=12.0.2.50=0
+  - libnvjitlink=12.1.105=0
+  - libnvjpeg=12.1.1.14=0
+  - libpng=1.6.39=h5eee18b_0
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - libtasn1=4.19.0=h5eee18b_0
+  - libtiff=4.5.1=h6a678d5_0
+  - libunistring=0.9.10=h27cfd23_0
+  - libuuid=1.41.5=h5eee18b_0
+  - libwebp=1.3.2=h11a3e52_0
+  - libwebp-base=1.3.2=h5eee18b_0
+  - llvm-openmp=14.0.6=h9e868ea_0
+  - lz4-c=1.9.4=h6a678d5_0
+  - markupsafe=2.1.1=py311h5eee18b_0
+  - mkl=2023.1.0=h213fc3f_46344
+  - mkl-service=2.4.0=py311h5eee18b_1
+  - mkl_fft=1.3.8=py311h5eee18b_0
+  - mkl_random=1.2.4=py311hdb19cb5_0
+  - mpc=1.1.0=h10f8cd9_1
+  - mpfr=4.0.2=hb69a4c5_1
+  - mpmath=1.3.0=py311h06a4308_0
+  - ncurses=6.4=h6a678d5_0
+  - nettle=3.7.3=hbbd107a_1
+  - networkx=3.1=py311h06a4308_0
+  - numpy=1.26.2=py311h08b1b3b_0
+  - numpy-base=1.26.2=py311hf175353_0
+  - openh264=2.1.1=h4ff587b_0
+  - openjpeg=2.4.0=h3ad879b_0
+  - openssl=3.0.12=h7f8727e_0
+  - pycparser=2.21=pyhd3eb1b0_0
+  - pyopenssl=23.2.0=py311h06a4308_0
+  - pysocks=1.7.1=py311h06a4308_0
+  - python=3.11.5=h955ad1f_0
+  - pytorch=2.1.2=py3.11_cuda12.1_cudnn8.9.2_0
+  - pytorch-cuda=12.1=ha16c6d3_5
+  - pytorch-mutex=1.0=cuda
+  - pyyaml=6.0.1=py311h5eee18b_0
+  - readline=8.2=h5eee18b_0
+  - requests=2.31.0=py311h06a4308_0
+  - setuptools=68.2.2=py311h06a4308_0
+  - sqlite=3.41.2=h5eee18b_0
+  - sympy=1.12=py311h06a4308_0
+  - tbb=2021.8.0=hdb19cb5_0
+  - tk=8.6.12=h1ccaba5_0
+  - torchaudio=2.1.2=py311_cu121
+  - torchtriton=2.1.0=py311
+  - torchvision=0.16.2=py311_cu121
+  - typing_extensions=4.7.1=py311h06a4308_0
+  - wheel=0.41.2=py311h06a4308_0
+  - xz=5.4.5=h5eee18b_0
+  - yaml=0.2.5=h7b6447c_0
+  - zlib=1.2.13=h5eee18b_0
+  - zstd=1.5.5=hc292b87_0
+  - pip:
+      - absl-py==2.0.0
+      - accelerate==0.23.0
+      - aiohttp==3.9.1
+      - aiosignal==1.3.1
+      - annotated-types==0.6.0
+      - appdirs==1.4.4
+      - astunparse==1.6.3
+      - attrs==23.1.0
+      - audioread==3.0.1
+      - bitsandbytes==0.41.2.post2
+      - cachetools==5.3.2
+      - chardet==5.2.0
+      - charset-normalizer==3.3.2
+      - click==8.1.7
+      - datasets==2.14.6
+      - decorator==5.1.1
+      - deepspeed==0.12.2
+      - dill==0.3.7
+      - docker-pycreds==0.4.0
+      - docstring-parser==0.15
+      - einops==0.7.0
+      - evaluate==0.4.0
+      - flash-attn==2.5.2
+      - flatbuffers==23.5.26
+      - frozenlist==1.4.1
+      - fsspec==2023.10.0
+      - gast==0.5.4
+      - gitdb==4.0.11
+      - gitpython==3.1.40
+      - google-auth==2.26.1
+      - google-auth-oauthlib==1.2.0
+      - google-pasta==0.2.0
+      - grpcio==1.60.0
+      - h5py==3.10.0
+      - hf-transfer==0.1.5
+      - hjson==3.1.0
+      - huggingface-hub==0.20.1
+      - idna==3.6
+      - jiwer==3.0.3
+      - joblib==1.3.2
+      - keras==2.15.0
+      - lazy-loader==0.3
+      - libclang==16.0.6
+      - librosa==0.10.1
+      - llvmlite==0.41.1
+      - markdown==3.5.1
+      - markdown-it-py==3.0.0
+      - mdurl==0.1.2
+      - ml-dtypes==0.2.0
+      - msgpack==1.0.7
+      - multidict==6.0.4
+      - multiprocess==0.70.15
+      - ninja==1.11.1.1
+      - nltk==3.8.1
+      - numba==0.58.1
+      - oauthlib==3.2.2
+      - opt-einsum==3.3.0
+      - packaging==23.2
+      - pandas==2.1.4
+      - peft==0.7.1
+      - pillow==10.2.0
+      - pip==23.3.2
+      - platformdirs==4.1.0
+      - pooch==1.8.0
+      - protobuf==3.20.2
+      - psutil==5.9.7
+      - py-cpuinfo==9.0.0
+      - pyarrow==14.0.2
+      - pyarrow-hotfix==0.6
+      - pyasn1==0.5.1
+      - pyasn1-modules==0.3.0
+      - pydantic==2.6.0
+      - pydantic-core==2.16.1
+      - pygments==2.17.2
+      - pynvml==11.5.0
+      - python-dateutil==2.8.2
+      - pytz==2023.3.post1
+      - rapidfuzz==3.6.1
+      - regex==2023.12.25
+      - requests-oauthlib==1.3.1
+      - responses==0.18.0
+      - rich==13.7.0
+      - rsa==4.9
+      - safetensors==0.4.1
+      - scikit-learn==1.3.2
+      - scipy==1.11.4
+      - sentencepiece==0.1.99
+      - sentry-sdk==1.39.1
+      - setproctitle==1.3.3
+      - shtab==1.6.5
+      - six==1.16.0
+      - smmap==5.0.1
+      - soundfile==0.12.1
+      - soxr==0.3.7
+      - tensorboard==2.15.1
+      - tensorboard-data-server==0.7.2
+      - tensorflow-cpu==2.15.0.post1
+      - tensorflow-estimator==2.15.0
+      - tensorflow-io-gcs-filesystem==0.35.0
+      - termcolor==2.4.0
+      - threadpoolctl==3.2.0
+      - tokenizers==0.15.0
+      - tqdm==4.66.1
+      - transformers==4.36.2
+      - trl==0.7.7
+      - typing-extensions==4.9.0
+      - tyro==0.7.0
+      - tzdata==2023.3
+      - urllib3==2.1.0
+      - wandb==0.16.1
+      - werkzeug==3.0.1
+      - wrapt==1.14.1
+      - xxhash==3.4.1
+      - yarl==1.9.4
+prefix: /fsx/sanchit/miniconda3/envs/venv

wandb/run-20240201_175850-i93q0p12/files/config.yaml

@@ -0,0 +1,660 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.11.5
+    cli_version: 0.16.1
+    framework: huggingface
+    huggingface_version: 4.36.2
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1706810330.083967
+    t:
+      1:
+      - 1
+      - 2
+      - 3
+      - 5
+      - 11
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 84
+      - 98
+      2:
+      - 1
+      - 2
+      - 3
+      - 5
+      - 11
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 84
+      - 98
+      3:
+      - 7
+      - 23
+      4: 3.11.5
+      5: 0.16.1
+      6: 4.36.2
+      8:
+      - 5
+      9:
+        1: transformers_trainer
+      13: linux-x86_64
+    m:
+    - 1: train/global_step
+      6:
+      - 3
+    - 1: train/loss
+      5: 1
+      6:
+      - 1
+    - 1: train/learning_rate
+      5: 1
+      6:
+      - 1
+    - 1: train/epoch
+      5: 1
+      6:
+      - 1
+    - 1: eval/loss
+      5: 1
+      6:
+      - 1
+    - 1: eval/runtime
+      5: 1
+      6:
+      - 1
+    - 1: eval/samples_per_second
+      5: 1
+      6:
+      - 1
+    - 1: eval/steps_per_second
+      5: 1
+      6:
+      - 1
+vocab_size:
+  desc: null
+  value: 32000
+max_position_embeddings:
+  desc: null
+  value: 32768
+hidden_size:
+  desc: null
+  value: 4096
+intermediate_size:
+  desc: null
+  value: 14336
+num_hidden_layers:
+  desc: null
+  value: 6
+num_attention_heads:
+  desc: null
+  value: 32
+sliding_window:
+  desc: null
+  value: 4096
+num_key_value_heads:
+  desc: null
+  value: 8
+hidden_act:
+  desc: null
+  value: silu
+initializer_range:
+  desc: null
+  value: 0.02
+rms_norm_eps:
+  desc: null
+  value: 1.0e-05
+use_cache:
+  desc: null
+  value: false
+rope_theta:
+  desc: null
+  value: 10000.0
+attention_dropout:
+  desc: null
+  value: 0.0
+return_dict:
+  desc: null
+  value: true
+output_hidden_states:
+  desc: null
+  value: false
+output_attentions:
+  desc: null
+  value: false
+torchscript:
+  desc: null
+  value: false
+torch_dtype:
+  desc: null
+  value: bfloat16
+use_bfloat16:
+  desc: null
+  value: false
+tf_legacy_loss:
+  desc: null
+  value: false
+pruned_heads:
+  desc: null
+  value: {}
+tie_word_embeddings:
+  desc: null
+  value: false
+is_encoder_decoder:
+  desc: null
+  value: false
+is_decoder:
+  desc: null
+  value: false
+cross_attention_hidden_size:
+  desc: null
+  value: null
+add_cross_attention:
+  desc: null
+  value: false
+tie_encoder_decoder:
+  desc: null
+  value: false
+max_length:
+  desc: null
+  value: 20
+min_length:
+  desc: null
+  value: 0
+do_sample:
+  desc: null
+  value: false
+early_stopping:
+  desc: null
+  value: false
+num_beams:
+  desc: null
+  value: 1
+num_beam_groups:
+  desc: null
+  value: 1
+diversity_penalty:
+  desc: null
+  value: 0.0
+temperature:
+  desc: null
+  value: 1.0
+top_k:
+  desc: null
+  value: 50
+top_p:
+  desc: null
+  value: 1.0
+typical_p:
+  desc: null
+  value: 1.0
+repetition_penalty:
+  desc: null
+  value: 1.0
+length_penalty:
+  desc: null
+  value: 1.0
+no_repeat_ngram_size:
+  desc: null
+  value: 0
+encoder_no_repeat_ngram_size:
+  desc: null
+  value: 0
+bad_words_ids:
+  desc: null
+  value: null
+num_return_sequences:
+  desc: null
+  value: 1
+chunk_size_feed_forward:
+  desc: null
+  value: 0
+output_scores:
+  desc: null
+  value: false
+return_dict_in_generate:
+  desc: null
+  value: false
+forced_bos_token_id:
+  desc: null
+  value: null
+forced_eos_token_id:
+  desc: null
+  value: null
+remove_invalid_values:
+  desc: null
+  value: false
+exponential_decay_length_penalty:
+  desc: null
+  value: null
+suppress_tokens:
+  desc: null
+  value: null
+begin_suppress_tokens:
+  desc: null
+  value: null
+architectures:
+  desc: null
+  value:
+  - MistralForCausalLM
+finetuning_task:
+  desc: null
+  value: null
+id2label:
+  desc: null
+  value:
+    '0': LABEL_0
+    '1': LABEL_1
+label2id:
+  desc: null
+  value:
+    LABEL_0: 0
+    LABEL_1: 1
+tokenizer_class:
+  desc: null
+  value: null
+prefix:
+  desc: null
+  value: null
+bos_token_id:
+  desc: null
+  value: 1
+pad_token_id:
+  desc: null
+  value: null
+eos_token_id:
+  desc: null
+  value: 2
+sep_token_id:
+  desc: null
+  value: null
+decoder_start_token_id:
+  desc: null
+  value: null
+task_specific_params:
+  desc: null
+  value: null
+problem_type:
+  desc: null
+  value: null
+_name_or_path:
+  desc: null
+  value: sanchit-gandhi/Mistral-7B-v0.1-6-layer
+transformers_version:
+  desc: null
+  value: 4.36.2
+model_type:
+  desc: null
+  value: mistral
+output_dir:
+  desc: null
+  value: ./
+overwrite_output_dir:
+  desc: null
+  value: true
+do_train:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: true
+do_predict:
+  desc: null
+  value: false
+evaluation_strategy:
+  desc: null
+  value: epoch
+prediction_loss_only:
+  desc: null
+  value: false
+per_device_train_batch_size:
+  desc: null
+  value: 64
+per_device_eval_batch_size:
+  desc: null
+  value: 32
+per_gpu_train_batch_size:
+  desc: null
+  value: null
+per_gpu_eval_batch_size:
+  desc: null
+  value: null
+gradient_accumulation_steps:
+  desc: null
+  value: 1
+eval_accumulation_steps:
+  desc: null
+  value: null
+eval_delay:
+  desc: null
+  value: 0
+learning_rate:
+  desc: null
+  value: 0.0003
+weight_decay:
+  desc: null
+  value: 0.0
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+max_grad_norm:
+  desc: null
+  value: 1.0
+num_train_epochs:
+  desc: null
+  value: 5
+max_steps:
+  desc: null
+  value: -1
+lr_scheduler_type:
+  desc: null
+  value: cosine
+lr_scheduler_kwargs:
+  desc: null
+  value: {}
+warmup_ratio:
+  desc: null
+  value: 0.1
+warmup_steps:
+  desc: null
+  value: 0
+log_level:
+  desc: null
+  value: info
+log_level_replica:
+  desc: null
+  value: warning
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: ./runs/Feb01_17-58-13_ip-26-0-165-24
+logging_strategy:
+  desc: null
+  value: steps
+logging_first_step:
+  desc: null
+  value: true
+logging_steps:
+  desc: null
+  value: 10
+logging_nan_inf_filter:
+  desc: null
+  value: true
+save_strategy:
+  desc: null
+  value: epoch
+save_steps:
+  desc: null
+  value: 500
+save_total_limit:
+  desc: null
+  value: 1
+save_safetensors:
+  desc: null
+  value: true
+save_on_each_node:
+  desc: null
+  value: false
+save_only_model:
+  desc: null
+  value: false
+no_cuda:
+  desc: null
+  value: false
+use_cpu:
+  desc: null
+  value: false
+use_mps_device:
+  desc: null
+  value: false
+seed:
+  desc: null
+  value: 42
+data_seed:
+  desc: null
+  value: null
+jit_mode_eval:
+  desc: null
+  value: false
+use_ipex:
+  desc: null
+  value: false
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+half_precision_backend:
+  desc: null
+  value: auto
+bf16_full_eval:
+  desc: null
+  value: false
+fp16_full_eval:
+  desc: null
+  value: false
+tf32:
+  desc: null
+  value: null
+local_rank:
+  desc: null
+  value: 0
+ddp_backend:
+  desc: null
+  value: null
+tpu_num_cores:
+  desc: null
+  value: null
+tpu_metrics_debug:
+  desc: null
+  value: false
+debug:
+  desc: null
+  value: []
+dataloader_drop_last:
+  desc: null
+  value: false
+eval_steps:
+  desc: null
+  value: null
+dataloader_num_workers:
+  desc: null
+  value: 0
+past_index:
+  desc: null
+  value: -1
+run_name:
+  desc: null
+  value: ./
+disable_tqdm:
+  desc: null
+  value: false
+remove_unused_columns:
+  desc: null
+  value: true
+label_names:
+  desc: null
+  value: null
+load_best_model_at_end:
+  desc: null
+  value: false
+metric_for_best_model:
+  desc: null
+  value: null
+greater_is_better:
+  desc: null
+  value: null
+ignore_data_skip:
+  desc: null
+  value: false
+fsdp:
+  desc: null
+  value: []
+fsdp_min_num_params:
+  desc: null
+  value: 0
+fsdp_config:
+  desc: null
+  value:
+    min_num_params: 0
+    xla: false
+    xla_fsdp_grad_ckpt: false
+fsdp_transformer_layer_cls_to_wrap:
+  desc: null
+  value: null
+deepspeed:
+  desc: null
+  value: null
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+optim:
+  desc: null
+  value: adamw_torch
+optim_args:
+  desc: null
+  value: null
+adafactor:
+  desc: null
+  value: false
+group_by_length:
+  desc: null
+  value: false
+length_column_name:
+  desc: null
+  value: length
+report_to:
+  desc: null
+  value:
+  - tensorboard
+  - wandb
+ddp_find_unused_parameters:
+  desc: null
+  value: null
+ddp_bucket_cap_mb:
+  desc: null
+  value: null
+ddp_broadcast_buffers:
+  desc: null
+  value: null
+dataloader_pin_memory:
+  desc: null
+  value: true
+dataloader_persistent_workers:
+  desc: null
+  value: false
+skip_memory_metrics:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+push_to_hub:
+  desc: null
+  value: true
+resume_from_checkpoint:
+  desc: null
+  value: null
+hub_model_id:
+  desc: null
+  value: null
+hub_strategy:
+  desc: null
+  value: every_save
+hub_token:
+  desc: null
+  value: <HUB_TOKEN>
+hub_private_repo:
+  desc: null
+  value: false
+hub_always_push:
+  desc: null
+  value: false
+gradient_checkpointing:
+  desc: null
+  value: true
+gradient_checkpointing_kwargs:
+  desc: null
+  value:
+    use_reentrant: false
+include_inputs_for_metrics:
+  desc: null
+  value: false
+fp16_backend:
+  desc: null
+  value: auto
+push_to_hub_model_id:
+  desc: null
+  value: null
+push_to_hub_organization:
+  desc: null
+  value: null
+push_to_hub_token:
+  desc: null
+  value: <PUSH_TO_HUB_TOKEN>
+mp_parameters:
+  desc: null
+  value: ''
+auto_find_batch_size:
+  desc: null
+  value: false
+full_determinism:
+  desc: null
+  value: false
+torchdynamo:
+  desc: null
+  value: null
+ray_scope:
+  desc: null
+  value: last
+ddp_timeout:
+  desc: null
+  value: 1800
+torch_compile:
+  desc: null
+  value: false
+torch_compile_backend:
+  desc: null
+  value: null
+torch_compile_mode:
+  desc: null
+  value: null
+dispatch_batches:
+  desc: null
+  value: null
+split_batches:
+  desc: null
+  value: false
+include_tokens_per_second:
+  desc: null
+  value: false
+include_num_input_tokens_seen:
+  desc: null
+  value: false
+neftune_noise_alpha:
+  desc: null
+  value: null
+max_seq_length:
+  desc: null
+  value: 2048

wandb/run-20240201_175850-i93q0p12/files/output.log

@@ -0,0 +1,358 @@
+
+  0%|                                                                                          | 0/1365 [00:00<?, ?it/s][WARNING|logging.py:314] 2024-02-01 17:58:55,470 >> You're using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.
+  0%|                                                                                | 1/1365 [00:11<4:10:37, 11.02s/it]
+[2024-02-01 17:59:06,373] [WARNING] [stage3.py:1949:step] 1 pytorch allocator cache flushes since last step. this happens when there is high memory pressure and is detrimental to performance. if this is happening frequently consider adjusting settings to reduce memory consumption. If you are unable to make the cache flushes go away consider adding get_accelerator().empty_cache() calls in your training loop to ensure that all ranks flush their caches at the same time
+
+
+
+
+
+
+
+
+
+  1%|▌                                                                              | 10/1365 [00:43<1:23:26,  3.70s/it]
+
+
+
+
+
+
+
+
+
+
+  1%|█▏                                                                             | 20/1365 [01:19<1:20:49,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+  2%|█▋                                                                             | 30/1365 [01:55<1:20:11,  3.60s/it]
+
+
+
+
+
+
+
+
+
+
+  3%|██▎                                                                            | 40/1365 [02:31<1:19:28,  3.60s/it]
+
+
+
+
+
+
+
+
+
+
+  4%|██▉                                                                            | 50/1365 [03:07<1:19:00,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+  4%|███▍                                                                           | 60/1365 [03:43<1:18:34,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+  5%|████                                                                           | 70/1365 [04:19<1:17:55,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+  6%|████▋                                                                          | 80/1365 [04:55<1:17:12,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+  7%|█████▏                                                                         | 90/1365 [05:31<1:16:32,  3.60s/it]
+
+
+
+
+
+
+
+
+
+
+  7%|█████▋                                                                        | 100/1365 [06:07<1:16:06,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+  8%|██████▎                                                                       | 110/1365 [06:44<1:15:24,  3.60s/it]
+
+
+
+
+
+
+
+
+
+
+  9%|██████▊                                                                       | 120/1365 [07:20<1:14:48,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 10%|███████▍                                                                      | 130/1365 [07:56<1:14:12,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 10%|████████                                                                      | 140/1365 [08:32<1:13:29,  3.60s/it]
+
+
+
+
+
+
+
+
+
+
+ 11%|████████▌                                                                     | 150/1365 [09:07<1:12:25,  3.58s/it]
+
+
+
+
+
+
+
+
+
+
+ 12%|█████████▏                                                                    | 160/1365 [09:43<1:12:00,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 12%|█████████▋                                                                    | 170/1365 [10:20<1:12:29,  3.64s/it]
+
+
+
+
+
+
+
+
+
+ 13%|██████████▏                                                                   | 179/1365 [10:52<1:11:26,  3.61s/it]
+
+
+
+
+
+
+
+
+
+
+ 14%|██████████▊                                                                   | 189/1365 [11:28<1:10:32,  3.60s/it]
+
+
+
+
+
+
+
+
+
+
+ 15%|███████████▎                                                                  | 199/1365 [12:04<1:09:55,  3.60s/it]
+
+
+
+
+
+
+
+
+
+
+
+ 15%|████████████                                                                  | 210/1365 [12:44<1:09:11,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 16%|████████████▌                                                                 | 220/1365 [13:20<1:08:33,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 17%|█████████████▏                                                                | 230/1365 [13:56<1:08:01,  3.60s/it]
+
+
+
+
+
+
+
+
+
+
+ 18%|█████████████▋                                                                | 240/1365 [14:32<1:07:15,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 18%|██████████████▎                                                               | 250/1365 [15:07<1:06:48,  3.59s/it]
+
+
+
+
+
+
+
+
+
+
+ 19%|██████████████▊                                                               | 260/1365 [15:44<1:06:51,  3.63s/it]
+
+
+
+
+
+
+
+
+
+
+ 20%|███████████████▍                                                              | 270/1365 [16:20<1:06:13,  3.63s/it]
+
+
+ 20%|███████████████▌                                                              | 273/1365 [16:31<1:05:09,  3.58s/it][INFO|trainer.py:3166] 2024-02-01 18:15:26,429 >> ***** Running Evaluation *****
+[INFO|trainer.py:3168] 2024-02-01 18:15:26,429 >>   Num examples = 15431
+[INFO|trainer.py:3171] 2024-02-01 18:15:26,429 >>   Batch size = 32
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 95%|██████████████████████████████████████████████████████████████████████████████▉    | 58/61 [00:28<00:01,  1.99it/s]
+ 20%|███████████████▌                                                              | 273/1365 [17:01<1:05:09,  3.58s/it][INFO|trainer.py:2889] 2024-02-01 18:15:58,319 >> Saving model checkpoint to ./tmp-checkpoint-273
+[INFO|configuration_utils.py:483] 2024-02-01 18:15:58,323 >> Configuration saved in ./tmp-checkpoint-273/config.json
+[INFO|configuration_utils.py:594] 2024-02-01 18:15:58,326 >> Configuration saved in ./tmp-checkpoint-273/generation_config.json
+[INFO|modeling_utils.py:2382] 2024-02-01 18:16:01,522 >> Model weights saved in ./tmp-checkpoint-273/pytorch_model.bin
+[INFO|tokenization_utils_base.py:2432] 2024-02-01 18:16:01,541 >> tokenizer config file saved in ./tmp-checkpoint-273/tokenizer_config.json
+[INFO|tokenization_utils_base.py:2441] 2024-02-01 18:16:01,543 >> Special tokens file saved in ./tmp-checkpoint-273/special_tokens_map.json
+/fsx/sanchit/miniconda3/envs/venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1879: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.
+  warnings.warn(
+[2024-02-01 18:16:01,626] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step273 is about to be saved!
+[2024-02-01 18:16:01,783] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./tmp-checkpoint-273/global_step273/zero_pp_rank_0_mp_rank_00_model_states.pt
+[2024-02-01 18:16:01,784] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./tmp-checkpoint-273/global_step273/zero_pp_rank_0_mp_rank_00_model_states.pt...
+[2024-02-01 18:16:01,787] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./tmp-checkpoint-273/global_step273/zero_pp_rank_0_mp_rank_00_model_states.pt.
+[2024-02-01 18:16:01,792] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./tmp-checkpoint-273/global_step273/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2024-02-01 18:16:05,650] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./tmp-checkpoint-273/global_step273/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2024-02-01 18:16:05,658] [INFO] [engine.py:3393:_save_zero_checkpoint] zero checkpoint saved ./tmp-checkpoint-273/global_step273/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2024-02-01 18:16:05,936] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step273 is ready now!
+[INFO|tokenization_utils_base.py:2432] 2024-02-01 18:16:08,421 >> tokenizer config file saved in ./tokenizer_config.json
+[INFO|tokenization_utils_base.py:2441] 2024-02-01 18:16:08,424 >> Special tokens file saved in ./special_tokens_map.json
+
+
+
+
+
+
+ 21%|████████████████                                                              | 280/1365 [17:38<1:31:41,  5.07s/it]
+
+
+
+
+
+
+
+
+
+
+ 21%|████████████████▌                                                             | 290/1365 [18:14<1:04:59,  3.63s/it]
+

wandb/run-20240201_175850-i93q0p12/files/requirements.txt

@@ -0,0 +1,141 @@
+absl-py==2.0.0
+accelerate==0.23.0
+aiohttp==3.9.1
+aiosignal==1.3.1
+alignment-handbook==0.4.0.dev0
+annotated-types==0.6.0
+appdirs==1.4.4
+astunparse==1.6.3
+attrs==23.1.0
+audioread==3.0.1
+bitsandbytes==0.41.2.post2
+brotli==1.0.9
+cachetools==5.3.2
+certifi==2023.11.17
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==2.0.4
+click==8.1.7
+cryptography==41.0.7
+datasets==2.14.6
+decorator==5.1.1
+deepspeed==0.12.2
+dill==0.3.7
+docker-pycreds==0.4.0
+docstring-parser==0.15
+einops==0.7.0
+evaluate==0.4.0
+filelock==3.13.1
+flash-attn==2.5.2
+flatbuffers==23.5.26
+frozenlist==1.4.1
+fsspec==2023.10.0
+gast==0.5.4
+gitdb==4.0.11
+gitpython==3.1.40
+gmpy2==2.1.2
+google-auth-oauthlib==1.2.0
+google-auth==2.26.1
+google-pasta==0.2.0
+grpcio==1.60.0
+h5py==3.10.0
+hf-transfer==0.1.5
+hjson==3.1.0
+huggingface-hub==0.20.1
+idna==3.4
+jinja2==3.1.2
+jiwer==3.0.3
+joblib==1.3.2
+keras==2.15.0
+lazy-loader==0.3
+libclang==16.0.6
+librosa==0.10.1
+llvmlite==0.41.1
+markdown-it-py==3.0.0
+markdown==3.5.1
+markupsafe==2.1.1
+mdurl==0.1.2
+mkl-fft==1.3.8
+mkl-random==1.2.4
+mkl-service==2.4.0
+ml-dtypes==0.2.0
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+multiprocess==0.70.15
+networkx==3.1
+ninja==1.11.1.1
+nltk==3.8.1
+numba==0.58.1
+numpy==1.26.2
+oauthlib==3.2.2
+opt-einsum==3.3.0
+packaging==23.2
+pandas==2.1.4
+peft==0.7.1
+pillow==10.2.0
+pip==23.3.2
+platformdirs==4.1.0
+pooch==1.8.0
+protobuf==3.20.2
+psutil==5.9.7
+py-cpuinfo==9.0.0
+pyarrow-hotfix==0.6
+pyarrow==14.0.2
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pycparser==2.21
+pydantic-core==2.16.1
+pydantic==2.6.0
+pygments==2.17.2
+pynvml==11.5.0
+pyopenssl==23.2.0
+pysocks==1.7.1
+python-dateutil==2.8.2
+pytz==2023.3.post1
+pyyaml==6.0.1
+rapidfuzz==3.6.1
+regex==2023.12.25
+requests-oauthlib==1.3.1
+requests==2.31.0
+responses==0.18.0
+rich==13.7.0
+rsa==4.9
+safetensors==0.4.1
+scikit-learn==1.3.2
+scipy==1.11.4
+sentencepiece==0.1.99
+sentry-sdk==1.39.1
+setproctitle==1.3.3
+setuptools==68.2.2
+shtab==1.6.5
+six==1.16.0
+smmap==5.0.1
+soundfile==0.12.1
+soxr==0.3.7
+sympy==1.12
+tensorboard-data-server==0.7.2
+tensorboard==2.15.1
+tensorflow-cpu==2.15.0.post1
+tensorflow-estimator==2.15.0
+tensorflow-io-gcs-filesystem==0.35.0
+termcolor==2.4.0
+threadpoolctl==3.2.0
+tokenizers==0.15.0
+torch==2.1.2
+torchaudio==2.1.2
+torchvision==0.16.2
+tqdm==4.66.1
+transformers==4.36.2
+triton==2.1.0
+trl==0.7.7
+typing-extensions==4.7.1
+tyro==0.7.0
+tzdata==2023.3
+urllib3==1.26.18
+wandb==0.16.1
+werkzeug==3.0.1
+wheel==0.41.2
+wrapt==1.14.1
+xxhash==3.4.1
+yarl==1.9.4

wandb/run-20240201_175850-i93q0p12/files/wandb-metadata.json

@@ -0,0 +1,558 @@
+{
+    "os": "Linux-5.15.0-1048-aws-x86_64-with-glibc2.31",
+    "python": "3.11.5",
+    "heartbeatAt": "2024-02-01T17:58:50.615903",
+    "startedAt": "2024-02-01T17:58:50.064651",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "config_full.yaml"
+    ],
+    "state": "running",
+    "program": "/fsx/sanchit/distil-zephyr-1.5b-ssft/run_sft.py",
+    "codePathLocal": "run_sft.py",
+    "codePath": "run_sft.py",
+    "git": {
+        "remote": "https://huggingface.co/sanchit-gandhi/distil-zephyr-1.5b-ssft",
+        "commit": "79a4ae874a71e67016ded927e7d23351e5c7dab8"
+    },
+    "email": null,
+    "root": "/fsx/sanchit/distil-zephyr-1.5b-ssft",
+    "host": "ip-26-0-165-24",
+    "username": "sanchit",
+    "executable": "/fsx/sanchit/miniconda3/envs/venv/bin/python",
+    "cpu_count": 96,
+    "cpu_count_logical": 96,
+    "cpu_freq": {
+        "current": 2728.6971041666666,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3589.018,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3587.597,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3596.958,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3597.83,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3595.048,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3591.991,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3598.381,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 3598.099,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2650.0,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 290.7472343444824,
+            "used": 57.44965744018555
+        }
+    },
+    "gpu": "NVIDIA H100 80GB HBM3",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA H100 80GB HBM3",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 80GB HBM3",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 80GB HBM3",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 80GB HBM3",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 80GB HBM3",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 80GB HBM3",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 80GB HBM3",
+            "memory_total": 85520809984
+        },
+        {
+            "name": "NVIDIA H100 80GB HBM3",
+            "memory_total": 85520809984
+        }
+    ],
+    "memory": {
+        "total": 1999.9855346679688
+    }
+}

wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/loss": 4.6222, "train/learning_rate": 0.00028865524291388006, "train/epoch": 1.06, "train/global_step": 290, "_timestamp": 1706811429.4937887, "_runtime": 1099.4098217487335, "_step": 30, "eval/loss": 4.737916946411133, "eval/runtime": 30.7851, "eval/samples_per_second": 501.248, "eval/steps_per_second": 1.981}

wandb/run-20240201_175850-i93q0p12/logs/debug.log

@@ -0,0 +1,28 @@
+2024-02-01 17:58:50,075 INFO    MainThread:239318 [wandb_setup.py:_flush():76] Current SDK version is 0.16.1
+2024-02-01 17:58:50,075 INFO    MainThread:239318 [wandb_setup.py:_flush():76] Configure stats pid to 239318
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_setup.py:_flush():76] Loading settings from /admin/home/sanchit/.config/wandb/settings
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_setup.py:_flush():76] Loading settings from /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/settings
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'run_sft.py', 'program_abspath': '/fsx/sanchit/distil-zephyr-1.5b-ssft/run_sft.py', 'program': '/fsx/sanchit/distil-zephyr-1.5b-ssft/run_sft.py'}
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_init.py:_log_setup():524] Logging user logs to /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/logs/debug.log
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_init.py:_log_setup():525] Logging internal logs to /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/logs/debug-internal.log
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_init.py:init():564] calling init triggers
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_init.py:init():571] wandb.init called with sweep_config: {}
+config: {}
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_init.py:init():614] starting backend
+2024-02-01 17:58:50,076 INFO    MainThread:239318 [wandb_init.py:init():618] setting up manager
+2024-02-01 17:58:50,079 INFO    MainThread:239318 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-02-01 17:58:50,083 INFO    MainThread:239318 [wandb_init.py:init():624] backend started and connected
+2024-02-01 17:58:50,087 INFO    MainThread:239318 [wandb_init.py:init():716] updated telemetry
+2024-02-01 17:58:50,107 INFO    MainThread:239318 [wandb_init.py:init():749] communicating run to backend with 90.0 second timeout
+2024-02-01 17:58:50,409 INFO    MainThread:239318 [wandb_run.py:_on_init():2254] communicating current version
+2024-02-01 17:58:50,456 INFO    MainThread:239318 [wandb_run.py:_on_init():2263] got version response upgrade_message: "wandb version 0.16.2 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-02-01 17:58:50,456 INFO    MainThread:239318 [wandb_init.py:init():800] starting run threads in backend
+2024-02-01 17:58:55,347 INFO    MainThread:239318 [wandb_run.py:_console_start():2233] atexit reg
+2024-02-01 17:58:55,347 INFO    MainThread:239318 [wandb_run.py:_redirect():2088] redirect: wrap_raw
+2024-02-01 17:58:55,348 INFO    MainThread:239318 [wandb_run.py:_redirect():2153] Wrapping output streams.
+2024-02-01 17:58:55,348 INFO    MainThread:239318 [wandb_run.py:_redirect():2178] Redirects installed.
+2024-02-01 17:58:55,348 INFO    MainThread:239318 [wandb_init.py:init():841] run started, returning control to user process
+2024-02-01 17:58:55,349 INFO    MainThread:239318 [wandb_run.py:_config_callback():1342] config_cb None None {'vocab_size': 32000, 'max_position_embeddings': 32768, 'hidden_size': 4096, 'intermediate_size': 14336, 'num_hidden_layers': 6, 'num_attention_heads': 32, 'sliding_window': 4096, 'num_key_value_heads': 8, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'use_cache': False, 'rope_theta': 10000.0, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['MistralForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 1, 'pad_token_id': None, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'sanchit-gandhi/Mistral-7B-v0.1-6-layer', 'transformers_version': '4.36.2', 'model_type': 'mistral', 'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'epoch', 'prediction_loss_only': False, 'per_device_train_batch_size': 64, 'per_device_eval_batch_size': 32, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 0.0003, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 5, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.1, 'warmup_steps': 0, 'log_level': 'info', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './runs/Feb01_17-58-13_ip-26-0-165-24', 'logging_strategy': 'steps', 'logging_first_step': True, 'logging_steps': 10, 'logging_nan_inf_filter': True, 'save_strategy': 'epoch', 'save_steps': 500, 'save_total_limit': 1, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'deepspeed': None, 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': {'use_reentrant': False}, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': False, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'max_seq_length': 2048}