Training in progress, step 40

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7cee5316db6e08bda593292c7431684b3dc73870ce8a54a0c3a013496aab2e9e
+oid sha256:c05b93acda6a7b2f81f0afd85f5badfd9cfa3a6e9e1606092480a0fbff648c88
 size 4725640

runs/Apr13_04-53-24_c5a47843c998/events.out.tfevents.1712984741.c5a47843c998.109.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d96cf0882700d1dbb5c8221808737e688d46416d72bbe5177a5bb46c68f44e9b
-size 5487
+oid sha256:6fc8f09b4ae665bdd4a2a08f56faec5206d9c03028fc53aa009026fd8ff8d950
+size 5694

wandb/debug-internal.log

@@ -74,3 +74,15 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-13 05:07:13,338 DEBUG   SenderThread:162 [sender.py:send_request():406] send_request: summary_record
 2024-04-13 05:07:13,340 INFO    SenderThread:162 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-13 05:07:13,509 INFO    Thread-12 :162 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240413_050649-ne3279ey/files/wandb-summary.json
+2024-04-13 05:07:15,995 DEBUG   HandlerThread:162 [handler.py:handle_request():146] handle_request: status_report
+2024-04-13 05:07:16,510 INFO    Thread-12 :162 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240413_050649-ne3279ey/files/output.log
+2024-04-13 05:07:21,001 DEBUG   HandlerThread:162 [handler.py:handle_request():146] handle_request: status_report
+2024-04-13 05:07:21,512 INFO    Thread-12 :162 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240413_050649-ne3279ey/files/config.yaml
+2024-04-13 05:07:21,646 DEBUG   HandlerThread:162 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-13 05:07:21,646 DEBUG   HandlerThread:162 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-13 05:07:21,647 DEBUG   SenderThread:162 [sender.py:send_request():406] send_request: stop_status
+2024-04-13 05:07:22,111 DEBUG   HandlerThread:162 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-13 05:07:22,112 DEBUG   SenderThread:162 [sender.py:send():379] send: history
+2024-04-13 05:07:22,113 DEBUG   SenderThread:162 [sender.py:send_request():406] send_request: summary_record
+2024-04-13 05:07:22,115 INFO    SenderThread:162 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-13 05:07:22,512 INFO    Thread-12 :162 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240413_050649-ne3279ey/files/wandb-summary.json

wandb/run-20240413_050649-ne3279ey/files/config.yaml

@@ -26,7 +26,23 @@ _wandb:
       - 84
       - 98
       - 105
+      2:
+      - 1
+      - 2
+      - 3
+      - 5
+      - 11
+      - 12
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      - 84
+      - 98
+      - 105
       3:
+      - 7
       - 23
       4: 3.10.13
       5: 0.16.5
@@ -35,4 +51,651 @@ _wandb:
       - 1
       - 2
       - 5
+      9:
+        1: transformers_trainer
       13: linux-x86_64
+    m:
+    - 1: train/global_step
+      6:
+      - 3
+    - 1: train/loss
+      5: 1
+      6:
+      - 1
+    - 1: train/grad_norm
+      5: 1
+      6:
+      - 1
+    - 1: train/learning_rate
+      5: 1
+      6:
+      - 1
+    - 1: train/epoch
+      5: 1
+      6:
+      - 1
+vocab_size:
+  desc: null
+  value: 250880
+hidden_size:
+  desc: null
+  value: 1536
+n_layer:
+  desc: null
+  value: 24
+n_head:
+  desc: null
+  value: 16
+layer_norm_epsilon:
+  desc: null
+  value: 1.0e-05
+initializer_range:
+  desc: null
+  value: 0.02
+use_cache:
+  desc: null
+  value: false
+pretraining_tp:
+  desc: null
+  value: 1
+apply_residual_connection_post_layernorm:
+  desc: null
+  value: false
+hidden_dropout:
+  desc: null
+  value: 0.0
+attention_dropout:
+  desc: null
+  value: 0.0
+bos_token_id:
+  desc: null
+  value: 1
+eos_token_id:
+  desc: null
+  value: 2
+slow_but_exact:
+  desc: null
+  value: false
+return_dict:
+  desc: null
+  value: true
+output_hidden_states:
+  desc: null
+  value: false
+output_attentions:
+  desc: null
+  value: false
+torchscript:
+  desc: null
+  value: false
+torch_dtype:
+  desc: null
+  value: float16
+use_bfloat16:
+  desc: null
+  value: false
+tf_legacy_loss:
+  desc: null
+  value: false
+pruned_heads:
+  desc: null
+  value: {}
+tie_word_embeddings:
+  desc: null
+  value: true
+chunk_size_feed_forward:
+  desc: null
+  value: 0
+is_encoder_decoder:
+  desc: null
+  value: false
+is_decoder:
+  desc: null
+  value: false
+cross_attention_hidden_size:
+  desc: null
+  value: null
+add_cross_attention:
+  desc: null
+  value: false
+tie_encoder_decoder:
+  desc: null
+  value: false
+max_length:
+  desc: null
+  value: 20
+min_length:
+  desc: null
+  value: 0
+do_sample:
+  desc: null
+  value: false
+early_stopping:
+  desc: null
+  value: false
+num_beams:
+  desc: null
+  value: 1
+num_beam_groups:
+  desc: null
+  value: 1
+diversity_penalty:
+  desc: null
+  value: 0.0
+temperature:
+  desc: null
+  value: 1.0
+top_k:
+  desc: null
+  value: 50
+top_p:
+  desc: null
+  value: 1.0
+typical_p:
+  desc: null
+  value: 1.0
+repetition_penalty:
+  desc: null
+  value: 1.0
+length_penalty:
+  desc: null
+  value: 1.0
+no_repeat_ngram_size:
+  desc: null
+  value: 0
+encoder_no_repeat_ngram_size:
+  desc: null
+  value: 0
+bad_words_ids:
+  desc: null
+  value: null
+num_return_sequences:
+  desc: null
+  value: 1
+output_scores:
+  desc: null
+  value: false
+return_dict_in_generate:
+  desc: null
+  value: false
+forced_bos_token_id:
+  desc: null
+  value: null
+forced_eos_token_id:
+  desc: null
+  value: null
+remove_invalid_values:
+  desc: null
+  value: false
+exponential_decay_length_penalty:
+  desc: null
+  value: null
+suppress_tokens:
+  desc: null
+  value: null
+begin_suppress_tokens:
+  desc: null
+  value: null
+architectures:
+  desc: null
+  value:
+  - BloomForCausalLM
+finetuning_task:
+  desc: null
+  value: null
+id2label:
+  desc: null
+  value:
+    '0': LABEL_0
+    '1': LABEL_1
+label2id:
+  desc: null
+  value:
+    LABEL_0: 0
+    LABEL_1: 1
+tokenizer_class:
+  desc: null
+  value: null
+prefix:
+  desc: null
+  value: null
+pad_token_id:
+  desc: null
+  value: 3
+sep_token_id:
+  desc: null
+  value: null
+decoder_start_token_id:
+  desc: null
+  value: null
+task_specific_params:
+  desc: null
+  value: null
+problem_type:
+  desc: null
+  value: null
+_name_or_path:
+  desc: null
+  value: bigscience/bloomz-1b1
+transformers_version:
+  desc: null
+  value: 4.39.3
+attention_softmax_in_fp32:
+  desc: null
+  value: true
+bias_dropout_fusion:
+  desc: null
+  value: true
+unk_token_id:
+  desc: null
+  value: 0
+masked_softmax_fusion:
+  desc: null
+  value: true
+model_type:
+  desc: null
+  value: bloom
+n_inner:
+  desc: null
+  value: null
+offset_alibi:
+  desc: null
+  value: 100
+seq_length:
+  desc: null
+  value: 2048
+skip_bias_add:
+  desc: null
+  value: true
+skip_bias_add_qkv:
+  desc: null
+  value: false
+quantization_config:
+  desc: null
+  value:
+    quant_method: QuantizationMethod.BITS_AND_BYTES
+    _load_in_8bit: false
+    _load_in_4bit: true
+    llm_int8_threshold: 6.0
+    llm_int8_skip_modules: null
+    llm_int8_enable_fp32_cpu_offload: false
+    llm_int8_has_fp16_weight: false
+    bnb_4bit_quant_type: nf4
+    bnb_4bit_use_double_quant: true
+    bnb_4bit_compute_dtype: float16
+    bnb_4bit_quant_storage: uint8
+    load_in_4bit: true
+    load_in_8bit: false
+output_dir:
+  desc: null
+  value: /kaggle/working/
+overwrite_output_dir:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: false
+do_predict:
+  desc: null
+  value: false
+evaluation_strategy:
+  desc: null
+  value: 'no'
+prediction_loss_only:
+  desc: null
+  value: false
+per_device_train_batch_size:
+  desc: null
+  value: 1
+per_device_eval_batch_size:
+  desc: null
+  value: 8
+per_gpu_train_batch_size:
+  desc: null
+  value: null
+per_gpu_eval_batch_size:
+  desc: null
+  value: null
+gradient_accumulation_steps:
+  desc: null
+  value: 1
+eval_accumulation_steps:
+  desc: null
+  value: null
+eval_delay:
+  desc: null
+  value: 0
+learning_rate:
+  desc: null
+  value: 5.0e-05
+weight_decay:
+  desc: null
+  value: 0.0001
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+max_grad_norm:
+  desc: null
+  value: 0.3
+num_train_epochs:
+  desc: null
+  value: 5
+max_steps:
+  desc: null
+  value: 20000
+lr_scheduler_type:
+  desc: null
+  value: cosine
+lr_scheduler_kwargs:
+  desc: null
+  value: {}
+warmup_ratio:
+  desc: null
+  value: 0.03
+warmup_steps:
+  desc: null
+  value: 0
+log_level:
+  desc: null
+  value: passive
+log_level_replica:
+  desc: null
+  value: warning
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: /kaggle/working/runs/Apr13_04-53-24_c5a47843c998
+logging_strategy:
+  desc: null
+  value: steps
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 20
+logging_nan_inf_filter:
+  desc: null
+  value: true
+save_strategy:
+  desc: null
+  value: steps
+save_steps:
+  desc: null
+  value: 20
+save_total_limit:
+  desc: null
+  value: 1
+save_safetensors:
+  desc: null
+  value: true
+save_on_each_node:
+  desc: null
+  value: false
+save_only_model:
+  desc: null
+  value: false
+no_cuda:
+  desc: null
+  value: false
+use_cpu:
+  desc: null
+  value: false
+use_mps_device:
+  desc: null
+  value: false
+seed:
+  desc: null
+  value: 42
+data_seed:
+  desc: null
+  value: null
+jit_mode_eval:
+  desc: null
+  value: false
+use_ipex:
+  desc: null
+  value: false
+bf16:
+  desc: null
+  value: false
+fp16:
+  desc: null
+  value: true
+fp16_opt_level:
+  desc: null
+  value: O1
+half_precision_backend:
+  desc: null
+  value: auto
+bf16_full_eval:
+  desc: null
+  value: false
+fp16_full_eval:
+  desc: null
+  value: false
+tf32:
+  desc: null
+  value: null
+local_rank:
+  desc: null
+  value: 0
+ddp_backend:
+  desc: null
+  value: null
+tpu_num_cores:
+  desc: null
+  value: null
+tpu_metrics_debug:
+  desc: null
+  value: false
+debug:
+  desc: null
+  value: []
+dataloader_drop_last:
+  desc: null
+  value: false
+eval_steps:
+  desc: null
+  value: null
+dataloader_num_workers:
+  desc: null
+  value: 2
+dataloader_prefetch_factor:
+  desc: null
+  value: null
+past_index:
+  desc: null
+  value: -1
+run_name:
+  desc: null
+  value: /kaggle/working/
+disable_tqdm:
+  desc: null
+  value: false
+remove_unused_columns:
+  desc: null
+  value: true
+label_names:
+  desc: null
+  value: null
+load_best_model_at_end:
+  desc: null
+  value: false
+metric_for_best_model:
+  desc: null
+  value: null
+greater_is_better:
+  desc: null
+  value: null
+ignore_data_skip:
+  desc: null
+  value: false
+fsdp:
+  desc: null
+  value: []
+fsdp_min_num_params:
+  desc: null
+  value: 0
+fsdp_config:
+  desc: null
+  value:
+    min_num_params: 0
+    xla: false
+    xla_fsdp_v2: false
+    xla_fsdp_grad_ckpt: false
+fsdp_transformer_layer_cls_to_wrap:
+  desc: null
+  value: null
+accelerator_config:
+  desc: null
+  value:
+    split_batches: false
+    dispatch_batches: null
+    even_batches: true
+    use_seedable_sampler: true
+deepspeed:
+  desc: null
+  value: null
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+optim:
+  desc: null
+  value: paged_adamw_8bit
+optim_args:
+  desc: null
+  value: null
+adafactor:
+  desc: null
+  value: false
+group_by_length:
+  desc: null
+  value: false
+length_column_name:
+  desc: null
+  value: length
+report_to:
+  desc: null
+  value:
+  - tensorboard
+  - wandb
+ddp_find_unused_parameters:
+  desc: null
+  value: null
+ddp_bucket_cap_mb:
+  desc: null
+  value: null
+ddp_broadcast_buffers:
+  desc: null
+  value: null
+dataloader_pin_memory:
+  desc: null
+  value: true
+dataloader_persistent_workers:
+  desc: null
+  value: false
+skip_memory_metrics:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+push_to_hub:
+  desc: null
+  value: true
+resume_from_checkpoint:
+  desc: null
+  value: null
+hub_model_id:
+  desc: null
+  value: Femboyuwu2000/bloomz-1b1-vn-chat
+hub_strategy:
+  desc: null
+  value: checkpoint
+hub_token:
+  desc: null
+  value: <HUB_TOKEN>
+hub_private_repo:
+  desc: null
+  value: false
+hub_always_push:
+  desc: null
+  value: false
+gradient_checkpointing:
+  desc: null
+  value: true
+gradient_checkpointing_kwargs:
+  desc: null
+  value: null
+include_inputs_for_metrics:
+  desc: null
+  value: false
+fp16_backend:
+  desc: null
+  value: auto
+push_to_hub_model_id:
+  desc: null
+  value: null
+push_to_hub_organization:
+  desc: null
+  value: null
+push_to_hub_token:
+  desc: null
+  value: <PUSH_TO_HUB_TOKEN>
+mp_parameters:
+  desc: null
+  value: ''
+auto_find_batch_size:
+  desc: null
+  value: false
+full_determinism:
+  desc: null
+  value: false
+torchdynamo:
+  desc: null
+  value: null
+ray_scope:
+  desc: null
+  value: last
+ddp_timeout:
+  desc: null
+  value: 1800
+torch_compile:
+  desc: null
+  value: false
+torch_compile_backend:
+  desc: null
+  value: null
+torch_compile_mode:
+  desc: null
+  value: null
+dispatch_batches:
+  desc: null
+  value: null
+split_batches:
+  desc: null
+  value: null
+include_tokens_per_second:
+  desc: null
+  value: false
+include_num_input_tokens_seen:
+  desc: null
+  value: false
+neftune_noise_alpha:
+  desc: null
+  value: null
+optim_target_modules:
+  desc: null
+  value: null

wandb/run-20240413_050649-ne3279ey/files/output.log

@@ -1 +1,5 @@
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
+/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
+  warnings.warn(
 /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.

wandb/run-20240413_050649-ne3279ey/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 3.7393, "train/grad_norm": 4.044270038604736, "train/learning_rate": 1.6666666666666667e-06, "train/epoch": 0.0, "train/global_step": 20, "_timestamp": 1712984833.3348606, "_runtime": 23.452062606811523, "_step": 0}
+{"train/loss": 3.6056, "train/grad_norm": 1.908144235610962, "train/learning_rate": 3.2500000000000002e-06, "train/epoch": 0.0, "train/global_step": 40, "_timestamp": 1712984842.1107213, "_runtime": 32.22792339324951, "_step": 1}

wandb/run-20240413_050649-ne3279ey/logs/debug-internal.log

@@ -74,3 +74,15 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-13 05:07:13,338 DEBUG   SenderThread:162 [sender.py:send_request():406] send_request: summary_record
 2024-04-13 05:07:13,340 INFO    SenderThread:162 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
 2024-04-13 05:07:13,509 INFO    Thread-12 :162 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240413_050649-ne3279ey/files/wandb-summary.json
+2024-04-13 05:07:15,995 DEBUG   HandlerThread:162 [handler.py:handle_request():146] handle_request: status_report
+2024-04-13 05:07:16,510 INFO    Thread-12 :162 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240413_050649-ne3279ey/files/output.log
+2024-04-13 05:07:21,001 DEBUG   HandlerThread:162 [handler.py:handle_request():146] handle_request: status_report
+2024-04-13 05:07:21,512 INFO    Thread-12 :162 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240413_050649-ne3279ey/files/config.yaml
+2024-04-13 05:07:21,646 DEBUG   HandlerThread:162 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-13 05:07:21,646 DEBUG   HandlerThread:162 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-13 05:07:21,647 DEBUG   SenderThread:162 [sender.py:send_request():406] send_request: stop_status
+2024-04-13 05:07:22,111 DEBUG   HandlerThread:162 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-13 05:07:22,112 DEBUG   SenderThread:162 [sender.py:send():379] send: history
+2024-04-13 05:07:22,113 DEBUG   SenderThread:162 [sender.py:send_request():406] send_request: summary_record
+2024-04-13 05:07:22,115 INFO    SenderThread:162 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-13 05:07:22,512 INFO    Thread-12 :162 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240413_050649-ne3279ey/files/wandb-summary.json