Upload

Predict_20/all_results.json

@@ -0,0 +1,9 @@
+{
+    "predict_bleu-4": 84.02507,
+    "predict_rouge-1": 88.655265,
+    "predict_rouge-2": 80.23742000000001,
+    "predict_rouge-l": 86.469845,
+    "predict_runtime": 37.4743,
+    "predict_samples_per_second": 0.534,
+    "predict_steps_per_second": 0.534
+}

Predict_20/generated_predictions.jsonl

@@ -0,0 +1,20 @@
+{"label": "I'm sorry, but I don't have the capability to book flights. My current function allows me to get the exchange rate between two currencies. If you need help with that, feel free to ask!", "predict": "I'm sorry, but I'm unable to assist with booking flights. My current capabilities are limited to providing exchange rates between two currencies."}
+{"label": "<functioncall> {\"name\": \"get_news_headlines\", \"arguments\": '{\"country\": \"United States\"}'} </functioncall>", "predict": "<functioncall> {\"name\": \"get_news_headlines\", \"arguments\": '{\"country\": \"United States\"}'} </functioncall>"}
+{"label": "Here are the latest news headlines for the United States:\n1. Biden announces new vaccine mandates\n2. Hurricane Ida devastates Louisiana\n3. Apple unveils new iPhone\n4. NASA's Perseverance rover collects first Mars rock sample", "predict": "Here are the latest news headlines for the United States:\n1. Biden announces new vaccine mandates\n2. Hurricane Ida devastates Louisiana\n3. Apple unveils new iPhone\n4. NASA's Perseverance rover collects first Mars rock sample"}
+{"label": "<functioncall> {\"name\": \"get_news_headlines\", \"arguments\": '{\"country\": \"France\"}'} </functioncall>", "predict": "<functioncall> {\"name\": \"get_news_headlines\", \"arguments\": '{\"country\": \"France\"}'} </functioncall>"}
+{"label": "Here are the latest news headlines for France:\n1. France recalls ambassadors to US and Australia\n2. French election: Macron's party braces for tough fight\n3. Louvre Museum to undergo major overhaul\n4. France to offer free birth control to all women under 25", "predict": "Here are the latest news headlines for France:\n1. France recalls ambassadors to US and Australia\n2. French election: Macron's party braces for tough fight\n3. Louvre Museum to undergo major overhaul\n4. France to offer free birth control to all women under 25"}
+{"label": "Of course. How long would you like your password to be? And would you like it to include symbols?", "predict": "Of course! How long would you like your password to be? And do you want it to include symbols?"}
+{"label": "<functioncall> {\"name\": \"generate_password\", \"arguments\": '{\"length\": 12, \"include_symbols\": true}'} </functioncall>", "predict": "<functioncall> {\"name\": \"generate_password\", \"arguments\": '{\"length\": 12, \"include_symbols\": true}'} </functioncall>"}
+{"label": "Here is your new password: 4&7j#9@1Q6*. Please make sure to save it in a secure location.", "predict": "Here is your new password: 4&7j#9@1Q6*. Please make sure to save it in a secure place."}
+{"label": "Of course! How long would you like your password to be? And do you want it to include numbers and special characters?", "predict": "Of course! How long would you like your password to be? And would you like it to include numbers and special characters?"}
+{"label": "<functioncall> {\"name\": \"generate_password\", \"arguments\": '{\"length\": 12, \"include_numbers\": true, \"include_special_characters\": true}'} </functioncall>", "predict": "<functioncall> {\"name\": \"generate_password\", \"arguments\": '{\"length\": 12, \"include_numbers\": true, \"include_special_characters\": true}'} </functioncall>"}
+{"label": "Here is your new password: 4#7gB6&9L1!0. Please make sure to save it in a secure place.", "predict": "Here is your new password: 4#7gB6&9L1!0. Please make sure to store it in a safe place."}
+{"label": "Of course, I can help you with that. Please provide me with the list of numbers.", "predict": "Of course, I can help you with that. Please provide me with the list of numbers."}
+{"label": "<functioncall> {\"name\": \"calculate_median\", \"arguments\": '{\"numbers\": [5, 2, 9, 1, 7, 4, 6, 3, 8]}'} </functioncall>", "predict": "<functioncall> {\"name\": \"calculate_median\", \"arguments\": '{\"numbers\": [5, 2, 9, 1, 7, 4, 6, 3, 8]}'} </functioncall>"}
+{"label": "The median of the provided list of numbers is 5.", "predict": "The median of the given list of numbers is 5."}
+{"label": "Of course, I can help you with that. Let me calculate it for you.\n<functioncall> {\"name\": \"calculate_loan_payment\", \"arguments\": '{\"principal\": 50000, \"interest_rate\": 5, \"loan_term\": 10}'} </functioncall>", "predict": "Of course, I can help you with that. Let me calculate the monthly payment for you.\n<functioncall> {\"name\": \"calculate_loan_payment\", \"arguments\": '{\"principal\": 50000, \"interest_rate\": 5, \"loan_term\": 10}'} </functioncall>"}
+{"label": "Based on the information provided, your monthly loan payment would be $530.33.", "predict": "The monthly payment for your loan would be $530.33."}
+{"label": "I'm sorry, but as an AI, I don't have the capability to book flights or perform external tasks. My current function is to calculate loan payments. I recommend using a travel booking website or app for your flight booking needs.", "predict": "I'm sorry, but as an AI, I don't have the capability to book flights or perform external tasks. My current functions are limited to the ones provided to me."}
+{"label": "<functioncall> {\"name\": \"convert_temperature\", \"arguments\": '{\"temperature\": 30, \"from_unit\": \"Celsius\", \"to_unit\": \"Fahrenheit\"}'} </functioncall>", "predict": "<functioncall> {\"name\": \"convert_temperature\", \"arguments\": '{\"temperature\": 30, \"from_unit\": \"Celsius\", \"to_unit\": \"Fahrenheit\"}'} </functioncall>"}
+{"label": "The converted temperature from 30 degrees Celsius to Fahrenheit is 86 degrees Fahrenheit.", "predict": "The temperature in Fahrenheit is 86 degrees."}
+{"label": "I'm sorry, but I can't assist with that. My current capabilities allow me to provide information about movies based on their titles. I don't have the ability to place orders or perform external tasks.", "predict": "I'm sorry, but I'm unable to perform external tasks such as ordering a pizza. My current capabilities are limited to the functions provided to me, and I can only assist with getting movie details."}

Predict_20/predict_log.txt

@@ -0,0 +1,290 @@
+01/18/2024 19:24:50 - WARNING - llmtuner.model.parser - `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
+[INFO|training_args.py:1838] 2024-01-18 19:24:50,331 >> PyTorch: setting up devices
+/home/hangyu5/anaconda3/envs/llama_factory/lib/python3.11/site-packages/transformers/training_args.py:1751: FutureWarning: `--push_to_hub_token` is deprecated and will be removed in version 5 of 🤗 Transformers. Use `--hub_token` instead.
+  warnings.warn(
+01/18/2024 19:24:50 - INFO - llmtuner.model.parser - Process rank: 0, device: cuda:0, n_gpu: 1
+  distributed training: True, compute dtype: None
+01/18/2024 19:24:50 - INFO - llmtuner.model.parser - Training/evaluation parameters Seq2SeqTrainingArguments(
+_n_gpu=1,
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=False,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=True,
+do_train=False,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_steps=None,
+evaluation_strategy=IntervalStrategy.NO,
+fp16=False,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+generation_config=None,
+generation_max_length=None,
+generation_num_beams=None,
+gradient_accumulation_steps=1,
+gradient_checkpointing=False,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=HubStrategy.EVERY_SAVE,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=5e-05,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=./models/sft/LMCocktail-10.7B-v1-sft-glaive-function-calling-v2-ep1-lora/Predict_20/runs/Jan18_19-24-50_yhyu13fuwuqi,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=500,
+logging_strategy=IntervalStrategy.STEPS,
+lr_scheduler_kwargs={},
+lr_scheduler_type=SchedulerType.LINEAR,
+max_grad_norm=1.0,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=3.0,
+optim=OptimizerNames.ADAMW_TORCH,
+optim_args=None,
+output_dir=./models/sft/LMCocktail-10.7B-v1-sft-glaive-function-calling-v2-ep1-lora/Predict_20,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=1,
+per_device_train_batch_size=8,
+predict_with_generate=True,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=True,
+report_to=['tensorboard'],
+resume_from_checkpoint=None,
+run_name=./models/sft/LMCocktail-10.7B-v1-sft-glaive-function-calling-v2-ep1-lora/Predict_20,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=500,
+save_strategy=IntervalStrategy.STEPS,
+save_total_limit=None,
+seed=42,
+skip_memory_metrics=True,
+sortish_sampler=False,
+split_batches=False,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.0,
+warmup_steps=0,
+weight_decay=0.0,
+)
+01/18/2024 19:24:50 - INFO - llmtuner.data.loader - Loading dataset ./glaive-function-calling-v2-llama-factory-convert/simple-function-calling-v2_converted_2000.json...
+01/18/2024 19:24:50 - WARNING - llmtuner.data.utils - Checksum failed: missing SHA-1 hash value in dataset_info.json.
+Using custom data configuration default-cb85ddec01d455d4
+Loading Dataset Infos from /home/hangyu5/anaconda3/envs/llama_factory/lib/python3.11/site-packages/datasets/packaged_modules/json
+Overwrite dataset info from restored data version if exists.
+Loading Dataset info from /home/hangyu5/.cache/huggingface/datasets/json/default-cb85ddec01d455d4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96
+Found cached dataset json (/home/hangyu5/.cache/huggingface/datasets/json/default-cb85ddec01d455d4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96)
+Loading Dataset info from /home/hangyu5/.cache/huggingface/datasets/json/default-cb85ddec01d455d4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96
+[INFO|tokenization_utils_base.py:2024] 2024-01-18 19:24:51,385 >> loading file tokenizer.model
+[INFO|tokenization_utils_base.py:2024] 2024-01-18 19:24:51,385 >> loading file added_tokens.json
+[INFO|tokenization_utils_base.py:2024] 2024-01-18 19:24:51,385 >> loading file special_tokens_map.json
+[INFO|tokenization_utils_base.py:2024] 2024-01-18 19:24:51,385 >> loading file tokenizer_config.json
+[INFO|tokenization_utils_base.py:2024] 2024-01-18 19:24:51,385 >> loading file tokenizer.json
+[INFO|configuration_utils.py:737] 2024-01-18 19:24:51,427 >> loading configuration file Yhyu13/LMCocktail-10.7B-v1/config.json
+[INFO|configuration_utils.py:802] 2024-01-18 19:24:51,428 >> Model config LlamaConfig {
+  "_name_or_path": "Yhyu13/LMCocktail-10.7B-v1",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 4096,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 48,
+  "num_key_value_heads": 8,
+  "pad_token_id": 2,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.36.2",
+  "use_cache": true,
+  "vocab_size": 32000
+}
+
+[INFO|modeling_utils.py:3341] 2024-01-18 19:24:51,444 >> loading weights file Yhyu13/LMCocktail-10.7B-v1/model.safetensors.index.json
+[INFO|modeling_utils.py:1341] 2024-01-18 19:24:51,444 >> Instantiating LlamaForCausalLM model under default dtype torch.float16.
+[INFO|configuration_utils.py:826] 2024-01-18 19:24:51,445 >> Generate config GenerationConfig {
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 2
+}
+
+
+Loading checkpoint shards:   0%|          | 0/5 [00:00<?, ?it/s]
+Loading checkpoint shards:  20%|██        | 1/5 [00:00<00:00,  6.36it/s]
+Loading checkpoint shards:  40%|████      | 2/5 [00:00<00:00,  6.36it/s]Yhyu13/LMCocktail-10.7B-v1
+Loading checkpoint shards:  60%|██████    | 3/5 [00:00<00:00,  6.36it/s]
+Loading checkpoint shards:  80%|████████  | 4/5 [00:00<00:00,  6.36it/s]Yhyu13/LMCocktail-10.7B-v1
+Loading checkpoint shards: 100%|██████████| 5/5 [00:00<00:00,  6.42it/s]
+Loading checkpoint shards: 100%|██████████| 5/5 [00:00<00:00,  6.39it/s]
+[INFO|modeling_utils.py:4185] 2024-01-18 19:24:52,397 >> All model checkpoint weights were used when initializing LlamaForCausalLM.
+
+[INFO|modeling_utils.py:4193] 2024-01-18 19:24:52,397 >> All the weights of LlamaForCausalLM were initialized from the model checkpoint at ./models/LMCocktail-10.7B-v1.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
+[INFO|configuration_utils.py:779] 2024-01-18 19:24:52,400 >> loading configuration file ./models/LMCocktail-10.7B-v1/generation_config.json
+[INFO|configuration_utils.py:826] 2024-01-18 19:24:52,400 >> Generate config GenerationConfig {
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 2,
+  "use_cache": false
+}
+
+01/18/2024 19:24:52 - INFO - llmtuner.model.adapter - Fine-tuning method: LoRA
+01/18/2024 19:24:54 - INFO - llmtuner.model.adapter - Merged 1 adapter(s).
+01/18/2024 19:24:54 - INFO - llmtuner.model.adapter - Loaded adapter(s): ./models/sft/LMCocktail-10.7B-v1-sft-glaive-function-calling-v2-ep1-lora
+01/18/2024 19:24:54 - INFO - llmtuner.model.loader - trainable params: 0 || all params: 10731524096 || trainable%: 0.0000
+01/18/2024 19:24:54 - INFO - llmtuner.model.loader - This IS expected that the trainable params is 0 if you are using model for inference only.
+
+Running tokenizer on dataset:   0%|          | 0/20 [00:00<?, ? examples/s]Caching processed dataset at /home/hangyu5/.cache/huggingface/datasets/json/default-cb85ddec01d455d4/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96/cache-700bf363697824f9.arrow
+
+Running tokenizer on dataset: 100%|██████████| 20/20 [00:00<00:00, 529.06 examples/s]
+[INFO|training_args.py:1838] 2024-01-18 19:24:54,939 >> PyTorch: setting up devices
+Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.
+[INFO|trainer.py:3166] 2024-01-18 19:24:57,618 >> ***** Running Prediction *****
+[INFO|trainer.py:3168] 2024-01-18 19:24:57,618 >>   Num examples = 20
+[INFO|trainer.py:3171] 2024-01-18 19:24:57,618 >>   Batch size = 1
+[INFO|configuration_utils.py:826] 2024-01-18 19:24:57,631 >> Generate config GenerationConfig {
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 2
+}
+
+/home/hangyu5/anaconda3/envs/llama_factory/lib/python3.11/site-packages/transformers/generation/utils.py:1518: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use and modify the model generation configuration (see https://huggingface.co/docs/transformers/generation_strategies#default-text-generation-configuration )
+  warnings.warn(
+input_ids:
+[1, 774, 1247, 28747, 13, 27842, 28747, 995, 460, 264, 10865, 13892, 395, 2735, 298, 272, 2296, 5572, 28723, 5938, 706, 513, 3030, 387, 13, 28751, 13, 2287, 345, 861, 1264, 345, 527, 28730, 720, 4078, 28730, 6036, 548, 13, 2287, 345, 6518, 1264, 345, 1458, 272, 8877, 4338, 1444, 989, 1191, 951, 20023, 548, 13, 2287, 345, 11438, 1264, 371, 13, 5390, 345, 1123, 1264, 345, 2814, 548, 13, 5390, 345, 10723, 1264, 371, 13, 17422, 345, 2893, 28730, 16714, 1264, 371, 13, 1417, 28705, 345, 1123, 1264, 345, 1427, 548, 13, 1417, 28705, 345, 6518, 1264, 345, 1014, 15547, 298, 6603, 477, 28739, 13, 17422, 1630, 13, 17422, 345, 3731, 28730, 16714, 1264, 371, 13, 1417, 28705, 345, 1123, 1264, 345, 1427, 548, 13, 1417, 28705, 345, 6518, 1264, 345, 1014, 15547, 298, 6603, 298, 28739, 13, 17422, 443, 13, 5390, 1630, 13, 5390, 345, 10893, 1264, 733, 13, 17422, 345, 2893, 28730, 16714, 548, 13, 17422, 345, 3731, 28730, 16714, 28739, 13, 5390, 4709, 13, 2287, 443, 13, 28752, 13, 13, 6325, 368, 1820, 264, 9314, 354, 528, 477, 1450, 2726, 298, 4222, 28804, 13, 13, 27332, 21631, 28747, 13]
+inputs:
+<s>### User:
+SYSTEM: You are a helpful assistant with access to the following functions. Use them if required -
+{
+    "name": "get_exchange_rate",
+    "description": "Get the exchange rate between two currencies",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "base_currency": {
+                "type": "string",
+                "description": "The currency to convert from"
+            },
+            "target_currency": {
+                "type": "string",
+                "description": "The currency to convert to"
+            }
+        },
+        "required": [
+            "base_currency",
+            "target_currency"
+        ]
+    }
+}
+
+Can you book a flight for me from New York to London?
+
+### Assistant:
+
+
+  0%|          | 0/20 [00:00<?, ?it/s]
+ 10%|█         | 2/20 [00:01<00:13,  1.31it/s]
+ 15%|█▌        | 3/20 [00:04<00:26,  1.56s/it]
+ 20%|██        | 4/20 [00:05<00:25,  1.60s/it]
+ 25%|██▌       | 5/20 [00:09<00:32,  2.16s/it]
+ 30%|███       | 6/20 [00:10<00:26,  1.87s/it]
+ 35%|███▌      | 7/20 [00:12<00:24,  1.87s/it]
+ 40%|████      | 8/20 [00:13<00:21,  1.79s/it]
+ 45%|████▌     | 9/20 [00:15<00:17,  1.62s/it]
+ 50%|█████     | 10/20 [00:17<00:18,  1.81s/it]
+ 55%|█████▌    | 11/20 [00:18<00:15,  1.75s/it]
+ 60%|██████    | 12/20 [00:19<00:12,  1.51s/it]
+ 65%|██████▌   | 13/20 [00:22<00:12,  1.79s/it]
+ 70%|███████   | 14/20 [00:23<00:08,  1.50s/it]
+ 75%|███████▌  | 15/20 [00:26<00:10,  2.07s/it]
+ 80%|████████  | 16/20 [00:27<00:06,  1.74s/it]
+ 85%|████████▌ | 17/20 [00:29<00:05,  1.79s/it]
+ 90%|█████████ | 18/20 [00:31<00:03,  1.99s/it]
+ 95%|█████████▌| 19/20 [00:32<00:01,  1.63s/it]
+100%|██████████| 20/20 [00:34<00:00,  1.72s/it]Building prefix dict from the default dictionary ...
+Loading model from cache /tmp/jieba.cache
+Loading model cost 0.675 seconds.
+Prefix dict has been built successfully.
+
+100%|██████████| 20/20 [00:35<00:00,  1.77s/it]
+***** predict metrics *****
+  predict_bleu-4             =    84.0251
+  predict_rouge-1            =    88.6553
+  predict_rouge-2            =    80.2374
+  predict_rouge-l            =    86.4698
+  predict_runtime            = 0:00:37.47
+  predict_samples_per_second =      0.534
+  predict_steps_per_second   =      0.534
+01/18/2024 19:25:35 - INFO - llmtuner.train.sft.trainer - Saving prediction results to ./models/sft/LMCocktail-10.7B-v1-sft-glaive-function-calling-v2-ep1-lora/Predict_20/generated_predictions.jsonl

Predict_20/predict_results.json

@@ -0,0 +1,9 @@
+{
+    "predict_bleu-4": 84.02507,
+    "predict_rouge-1": 88.655265,
+    "predict_rouge-2": 80.23742000000001,
+    "predict_rouge-l": 86.469845,
+    "predict_runtime": 37.4743,
+    "predict_samples_per_second": 0.534,
+    "predict_steps_per_second": 0.534
+}

README.md

@@ -1,3 +1,68 @@
 ---
 license: llama2
+library_name: peft
+tags:
+- llama-factory
+- lora
+- generated_from_trainer
+base_model: Yhyu13/LMCocktail-10.7B-v1
+model-index:
+- name: LMCocktail-10.7B-v1-sft-glaive-function-calling-v2-ep1-lora
+  results: []
 ---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# LMCocktail-10.7B-v1-sft-glaive-function-calling-v2-ep1-lora
+
+This model is a fine-tuned version of [Yhyu13/LMCocktail-10.7B-v1](https://huggingface.co/Yhyu13/LMCocktail-10.7B-v1) on the https://huggingface.co/datasets/Yhyu13/glaive-function-calling-v2-llama-factory-convert dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.2787
+
+Training script is availbale at ./scripts/
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 1
+- eval_batch_size: 1
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 2
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 8
+- total_eval_batch_size: 2
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- num_epochs: 1.0
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 0.2696        | 1.0   | 747  | 0.2787          |
+
+
+### Framework versions
+
+- PEFT 0.7.1
+- Transformers 4.36.2
+- Pytorch 2.1.2+cu121
+- Datasets 2.16.1
+- Tokenizers 0.15.0

adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Yhyu13/LMCocktail-10.7B-v1",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6cac1463452a8fc823b715bbd6df7f3a3fad2a24ea5c3ed495e58545c07ea9b
+size 20472752

all_results.json

@@ -0,0 +1,11 @@
+{
+    "epoch": 1.0,
+    "eval_loss": 0.27866294980049133,
+    "eval_runtime": 101.2511,
+    "eval_samples_per_second": 6.558,
+    "eval_steps_per_second": 3.279,
+    "train_loss": 0.3299662241814446,
+    "train_runtime": 2859.8545,
+    "train_samples_per_second": 2.089,
+    "train_steps_per_second": 0.261
+}

eval_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 1.0,
+    "eval_loss": 0.27866294980049133,
+    "eval_runtime": 101.2511,
+    "eval_samples_per_second": 6.558,
+    "eval_steps_per_second": 3.279
+}

scripts/local_sft_fn_lmcocktail.sh

@@ -0,0 +1,111 @@
+#!/bin/bash
+
+eval "$(conda shell.bash hook)"
+conda activate llama_factory
+
+MODEL_NAME=LMCocktail-10.7B-v1
+STAGE=sft
+EPOCH=1 #3.0
+DATA=glaive-function-calling-v2
+
+FT_TYPE=lora
+LoRA_TARGET=q_proj,v_proj
+TEMPLATE=solar
+PREDICTION_SAMPLES=20
+
+MODEL_PATH=./models/$MODEL_NAME
+if [ ! -d $MODEL_PATH ]; then
+    echo "Model not found: $MODEL_PATH"
+    return 1
+fi
+
+SAVE_PATH=./models/$STAGE/$MODEL_NAME-$STAGE-$DATA-ep$EPOCH-$FT_TYPE
+if [ ! -d $SAVE_PATH ]; then
+    mkdir -p $SAVE_PATH
+fi
+
+DO_TRAIN=false
+DO_PREDICT=false
+DO_EXPORT=false
+
+for arg in "$@"
+do
+ if [[ "$arg" == "--train" ]]; then
+   echo "The '--train' argument is present in an argument: $arg"
+   DO_TRAIN=true
+ fi
+  if [[ "$arg" == "--pred" ]]; then
+   echo "The '--pred' argument is present in an argument: $arg"
+   DO_PREDICT=true
+ fi
+  if [[ "$arg" == "--exp" ]]; then
+   echo "The '--exp' argument is present in an argument: $arg"
+   DO_EXPORT=true
+ fi
+done
+
+if [ $DO_TRAIN == true ]; then
+    accelerate launch src/train_bash.py \
+        --seed 42 \
+        --stage $STAGE \
+        --model_name_or_path $MODEL_PATH \
+        --dataset $DATA \
+        --val_size .1 \
+        --template $TEMPLATE \
+        --finetuning_type $FT_TYPE \
+        --do_train \
+        --lora_target $LoRA_TARGET \
+        --output_dir $SAVE_PATH \
+        --overwrite_output_dir \
+        --overwrite_cache \
+        --per_device_train_batch_size 1 \
+        --gradient_accumulation_steps 4 \
+        --lr_scheduler_type cosine \
+        --logging_steps 500 \
+        --save_steps 500 \
+        --learning_rate 5e-5 \
+        --num_train_epochs $EPOCH \
+        --do_eval \
+        --evaluation_strategy steps \
+        --per_device_eval_batch_size 1 \
+        --prediction_loss_only \
+        --plot_loss \
+        --quantization_bit 4 \
+        --report_to tensorboard \
+        |& tee $SAVE_PATH/train_eval_log.txt
+fi
+
+if [ $DO_PREDICT == true ]; then
+    SAVE_PATH_PREDICT=$SAVE_PATH/Predict_$PREDICTION_SAMPLES
+    if [ ! -d $SAVE_PATH_PREDICT ]; then
+        mkdir -p $SAVE_PATH_PREDICT
+    fi
+    CUDA_VISIBLE_DEVICES=0 python src/train_bash.py \
+        --stage $STAGE \
+        --model_name_or_path $MODEL_PATH \
+        --do_predict \
+        --max_samples $PREDICTION_SAMPLES \
+        --predict_with_generate \
+        --dataset $DATA \
+        --template $TEMPLATE \
+        --finetuning_type $FT_TYPE \
+        --adapter_name_or_path $SAVE_PATH \
+        --output_dir $SAVE_PATH_PREDICT \
+        --per_device_eval_batch_size 1 \
+        |& tee $SAVE_PATH_PREDICT/predict_log.txt
+fi
+
+if [ $DO_EXPORT == true ]; then
+    EXPORT_PATH=./models/export/$MODEL_NAME-$STAGE-$DATA-ep$EPOCH
+    if [ ! -d $EXPORT_PATH ]; then
+        mkdir -p $EXPORT_PATH
+    fi
+    CUDA_VISIBLE_DEVICES=0 python src/export_model.py \
+        --model_name_or_path $MODEL_PATH \
+        --adapter_name_or_path $SAVE_PATH \
+        --template $TEMPLATE \
+        --finetuning_type $FT_TYPE \
+        --export_dir $EXPORT_PATH \
+        --export_size 5 \
+        |& tee $EXPORT_PATH/export_log.txt
+fi

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,45 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{% if message['content']%}{{'### System:\n' + message['content']+'\n\n'}}{% endif %}{% elif message['role'] == 'user' %}{{'### User:\n' + message['content']+'\n\n'}}{% elif message['role'] == 'assistant' %}{{'### Assistant:\n'  + message['content']}}{% endif %}{% if loop.last and add_generation_prompt %}{{ '### Assistant:\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true
+}

train_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.3299662241814446,
+    "train_runtime": 2859.8545,
+    "train_samples_per_second": 2.089,
+    "train_steps_per_second": 0.261
+}

trainer_log.jsonl

@@ -0,0 +1,77 @@
+{"current_steps": 10, "total_steps": 747, "loss": 1.6077, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.997789428625975e-05, "epoch": 0.01, "percentage": 1.34, "elapsed_time": "0:00:36", "remaining_time": "0:44:29"}
+{"current_steps": 20, "total_steps": 747, "loss": 0.9415, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.99116162380454e-05, "epoch": 0.03, "percentage": 2.68, "elapsed_time": "0:01:13", "remaining_time": "0:44:28"}
+{"current_steps": 30, "total_steps": 747, "loss": 0.6271, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.980128306524183e-05, "epoch": 0.04, "percentage": 4.02, "elapsed_time": "0:01:48", "remaining_time": "0:43:13"}
+{"current_steps": 40, "total_steps": 747, "loss": 0.5335, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.964708988733178e-05, "epoch": 0.05, "percentage": 5.35, "elapsed_time": "0:02:27", "remaining_time": "0:43:20"}
+{"current_steps": 50, "total_steps": 747, "loss": 0.3693, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.944930938833535e-05, "epoch": 0.07, "percentage": 6.69, "elapsed_time": "0:03:01", "remaining_time": "0:42:14"}
+{"current_steps": 60, "total_steps": 747, "loss": 0.47, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.9208291334580104e-05, "epoch": 0.08, "percentage": 8.03, "elapsed_time": "0:03:39", "remaining_time": "0:41:57"}
+{"current_steps": 70, "total_steps": 747, "loss": 0.4268, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.892446195615423e-05, "epoch": 0.09, "percentage": 9.37, "elapsed_time": "0:04:16", "remaining_time": "0:41:19"}
+{"current_steps": 80, "total_steps": 747, "loss": 0.3653, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.859832319313697e-05, "epoch": 0.11, "percentage": 10.71, "elapsed_time": "0:04:53", "remaining_time": "0:40:49"}
+{"current_steps": 90, "total_steps": 747, "loss": 0.3466, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.8230451807939135e-05, "epoch": 0.12, "percentage": 12.05, "elapsed_time": "0:05:31", "remaining_time": "0:40:21"}
+{"current_steps": 100, "total_steps": 747, "loss": 0.3241, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.782149836532345e-05, "epoch": 0.13, "percentage": 13.39, "elapsed_time": "0:06:07", "remaining_time": "0:39:39"}
+{"current_steps": 110, "total_steps": 747, "loss": 0.3763, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.737218608190878e-05, "epoch": 0.15, "percentage": 14.73, "elapsed_time": "0:06:47", "remaining_time": "0:39:21"}
+{"current_steps": 120, "total_steps": 747, "loss": 0.3333, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.6883309547192476e-05, "epoch": 0.16, "percentage": 16.06, "elapsed_time": "0:07:25", "remaining_time": "0:38:48"}
+{"current_steps": 130, "total_steps": 747, "loss": 0.3106, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.635573331835302e-05, "epoch": 0.17, "percentage": 17.4, "elapsed_time": "0:08:01", "remaining_time": "0:38:05"}
+{"current_steps": 140, "total_steps": 747, "loss": 0.3241, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.5790390391317675e-05, "epoch": 0.19, "percentage": 18.74, "elapsed_time": "0:08:39", "remaining_time": "0:37:33"}
+{"current_steps": 150, "total_steps": 747, "loss": 0.2781, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.518828055079925e-05, "epoch": 0.2, "percentage": 20.08, "elapsed_time": "0:09:15", "remaining_time": "0:36:50"}
+{"current_steps": 160, "total_steps": 747, "loss": 0.3, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.4550468602219716e-05, "epoch": 0.21, "percentage": 21.42, "elapsed_time": "0:09:54", "remaining_time": "0:36:19"}
+{"current_steps": 170, "total_steps": 747, "loss": 0.3229, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.387808248864751e-05, "epoch": 0.23, "percentage": 22.76, "elapsed_time": "0:10:29", "remaining_time": "0:35:37"}
+{"current_steps": 180, "total_steps": 747, "loss": 0.3347, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.3172311296078595e-05, "epoch": 0.24, "percentage": 24.1, "elapsed_time": "0:11:05", "remaining_time": "0:34:56"}
+{"current_steps": 190, "total_steps": 747, "loss": 0.2897, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.2434403150588895e-05, "epoch": 0.25, "percentage": 25.44, "elapsed_time": "0:11:42", "remaining_time": "0:34:19"}
+{"current_steps": 200, "total_steps": 747, "loss": 0.2951, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.166566301107687e-05, "epoch": 0.27, "percentage": 26.77, "elapsed_time": "0:12:18", "remaining_time": "0:33:40"}
+{"current_steps": 210, "total_steps": 747, "loss": 0.3081, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.08674503614997e-05, "epoch": 0.28, "percentage": 28.11, "elapsed_time": "0:12:56", "remaining_time": "0:33:04"}
+{"current_steps": 220, "total_steps": 747, "loss": 0.2889, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.004117680668422e-05, "epoch": 0.29, "percentage": 29.45, "elapsed_time": "0:13:29", "remaining_time": "0:32:18"}
+{"current_steps": 230, "total_steps": 747, "loss": 0.3207, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.918830357596434e-05, "epoch": 0.31, "percentage": 30.79, "elapsed_time": "0:14:05", "remaining_time": "0:31:40"}
+{"current_steps": 240, "total_steps": 747, "loss": 0.3238, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.8310338939059644e-05, "epoch": 0.32, "percentage": 32.13, "elapsed_time": "0:14:41", "remaining_time": "0:31:02"}
+{"current_steps": 250, "total_steps": 747, "loss": 0.3146, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.740883553876515e-05, "epoch": 0.33, "percentage": 33.47, "elapsed_time": "0:15:19", "remaining_time": "0:30:27"}
+{"current_steps": 260, "total_steps": 747, "loss": 0.2973, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.6485387645169064e-05, "epoch": 0.35, "percentage": 34.81, "elapsed_time": "0:15:53", "remaining_time": "0:29:46"}
+{"current_steps": 270, "total_steps": 747, "loss": 0.2631, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.55416283362546e-05, "epoch": 0.36, "percentage": 36.14, "elapsed_time": "0:16:31", "remaining_time": "0:29:11"}
+{"current_steps": 280, "total_steps": 747, "loss": 0.3323, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.457922660987155e-05, "epoch": 0.37, "percentage": 37.48, "elapsed_time": "0:17:10", "remaining_time": "0:28:38"}
+{"current_steps": 290, "total_steps": 747, "loss": 0.2631, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.3599884432185225e-05, "epoch": 0.39, "percentage": 38.82, "elapsed_time": "0:17:47", "remaining_time": "0:28:02"}
+{"current_steps": 300, "total_steps": 747, "loss": 0.3066, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.260533372782234e-05, "epoch": 0.4, "percentage": 40.16, "elapsed_time": "0:18:22", "remaining_time": "0:27:22"}
+{"current_steps": 310, "total_steps": 747, "loss": 0.3131, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.1597333317036545e-05, "epoch": 0.41, "percentage": 41.5, "elapsed_time": "0:18:58", "remaining_time": "0:26:44"}
+{"current_steps": 320, "total_steps": 747, "loss": 0.3101, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.057766580531031e-05, "epoch": 0.43, "percentage": 42.84, "elapsed_time": "0:19:36", "remaining_time": "0:26:09"}
+{"current_steps": 330, "total_steps": 747, "loss": 0.3022, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.9548134430893604e-05, "epoch": 0.44, "percentage": 44.18, "elapsed_time": "0:20:14", "remaining_time": "0:25:34"}
+{"current_steps": 340, "total_steps": 747, "loss": 0.2496, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.8510559875854377e-05, "epoch": 0.46, "percentage": 45.52, "elapsed_time": "0:20:49", "remaining_time": "0:24:56"}
+{"current_steps": 350, "total_steps": 747, "loss": 0.3218, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.7466777046280457e-05, "epoch": 0.47, "percentage": 46.85, "elapsed_time": "0:21:25", "remaining_time": "0:24:17"}
+{"current_steps": 360, "total_steps": 747, "loss": 0.3064, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.6418631827326857e-05, "epoch": 0.48, "percentage": 48.19, "elapsed_time": "0:21:59", "remaining_time": "0:23:38"}
+{"current_steps": 370, "total_steps": 747, "loss": 0.2937, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.5367977818847034e-05, "epoch": 0.5, "percentage": 49.53, "elapsed_time": "0:22:38", "remaining_time": "0:23:04"}
+{"current_steps": 380, "total_steps": 747, "loss": 0.2708, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.431667305738112e-05, "epoch": 0.51, "percentage": 50.87, "elapsed_time": "0:23:19", "remaining_time": "0:22:32"}
+{"current_steps": 390, "total_steps": 747, "loss": 0.2838, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.3266576730297956e-05, "epoch": 0.52, "percentage": 52.21, "elapsed_time": "0:23:56", "remaining_time": "0:21:54"}
+{"current_steps": 400, "total_steps": 747, "loss": 0.2987, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.221954588790206e-05, "epoch": 0.54, "percentage": 53.55, "elapsed_time": "0:24:34", "remaining_time": "0:21:19"}
+{"current_steps": 410, "total_steps": 747, "loss": 0.3069, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.1177432159319754e-05, "epoch": 0.55, "percentage": 54.89, "elapsed_time": "0:25:11", "remaining_time": "0:20:42"}
+{"current_steps": 420, "total_steps": 747, "loss": 0.2709, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.014207847797256e-05, "epoch": 0.56, "percentage": 56.22, "elapsed_time": "0:25:47", "remaining_time": "0:20:05"}
+{"current_steps": 430, "total_steps": 747, "loss": 0.3159, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.9115315822428437e-05, "epoch": 0.58, "percentage": 57.56, "elapsed_time": "0:26:26", "remaining_time": "0:19:29"}
+{"current_steps": 440, "total_steps": 747, "loss": 0.2997, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.809895997839482e-05, "epoch": 0.59, "percentage": 58.9, "elapsed_time": "0:27:01", "remaining_time": "0:18:51"}
+{"current_steps": 450, "total_steps": 747, "loss": 0.2347, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.70948083275794e-05, "epoch": 0.6, "percentage": 60.24, "elapsed_time": "0:27:37", "remaining_time": "0:18:14"}
+{"current_steps": 460, "total_steps": 747, "loss": 0.2634, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.6104636669097776e-05, "epoch": 0.62, "percentage": 61.58, "elapsed_time": "0:28:12", "remaining_time": "0:17:35"}
+{"current_steps": 470, "total_steps": 747, "loss": 0.2438, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.513019607904882e-05, "epoch": 0.63, "percentage": 62.92, "elapsed_time": "0:28:49", "remaining_time": "0:16:59"}
+{"current_steps": 480, "total_steps": 747, "loss": 0.2289, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.4173209813811788e-05, "epoch": 0.64, "percentage": 64.26, "elapsed_time": "0:29:26", "remaining_time": "0:16:22"}
+{"current_steps": 490, "total_steps": 747, "loss": 0.2609, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.3235370262541272e-05, "epoch": 0.66, "percentage": 65.6, "elapsed_time": "0:30:05", "remaining_time": "0:15:47"}
+{"current_steps": 500, "total_steps": 747, "loss": 0.2724, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.2318335954249669e-05, "epoch": 0.67, "percentage": 66.93, "elapsed_time": "0:30:43", "remaining_time": "0:15:10"}
+{"current_steps": 510, "total_steps": 747, "loss": 0.272, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.1423728624769695e-05, "epoch": 0.68, "percentage": 68.27, "elapsed_time": "0:31:18", "remaining_time": "0:14:33"}
+{"current_steps": 520, "total_steps": 747, "loss": 0.303, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.0553130348784182e-05, "epoch": 0.7, "percentage": 69.61, "elapsed_time": "0:31:57", "remaining_time": "0:13:56"}
+{"current_steps": 530, "total_steps": 747, "loss": 0.2766, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 9.708080741994868e-06, "epoch": 0.71, "percentage": 70.95, "elapsed_time": "0:32:35", "remaining_time": "0:13:20"}
+{"current_steps": 540, "total_steps": 747, "loss": 0.2333, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 8.890074238378074e-06, "epoch": 0.72, "percentage": 72.29, "elapsed_time": "0:33:13", "remaining_time": "0:12:44"}
+{"current_steps": 550, "total_steps": 747, "loss": 0.2326, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 8.100557447342327e-06, "epoch": 0.74, "percentage": 73.63, "elapsed_time": "0:33:51", "remaining_time": "0:12:07"}
+{"current_steps": 560, "total_steps": 747, "loss": 0.2274, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 7.340926595461687e-06, "epoch": 0.75, "percentage": 74.97, "elapsed_time": "0:34:28", "remaining_time": "0:11:30"}
+{"current_steps": 570, "total_steps": 747, "loss": 0.3255, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 6.612525057308949e-06, "epoch": 0.76, "percentage": 76.31, "elapsed_time": "0:35:05", "remaining_time": "0:10:53"}
+{"current_steps": 580, "total_steps": 747, "loss": 0.2631, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 5.9166409797553415e-06, "epoch": 0.78, "percentage": 77.64, "elapsed_time": "0:35:41", "remaining_time": "0:10:16"}
+{"current_steps": 590, "total_steps": 747, "loss": 0.2938, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 5.254505003938043e-06, "epoch": 0.79, "percentage": 78.98, "elapsed_time": "0:36:17", "remaining_time": "0:09:39"}
+{"current_steps": 600, "total_steps": 747, "loss": 0.2807, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.627288088924156e-06, "epoch": 0.8, "percentage": 80.32, "elapsed_time": "0:36:53", "remaining_time": "0:09:02"}
+{"current_steps": 610, "total_steps": 747, "loss": 0.2909, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.036099440919763e-06, "epoch": 0.82, "percentage": 81.66, "elapsed_time": "0:37:34", "remaining_time": "0:08:26"}
+{"current_steps": 620, "total_steps": 747, "loss": 0.2997, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.481984551686429e-06, "epoch": 0.83, "percentage": 83.0, "elapsed_time": "0:38:11", "remaining_time": "0:07:49"}
+{"current_steps": 630, "total_steps": 747, "loss": 0.2424, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.9659233496337786e-06, "epoch": 0.84, "percentage": 84.34, "elapsed_time": "0:38:48", "remaining_time": "0:07:12"}
+{"current_steps": 640, "total_steps": 747, "loss": 0.2862, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.4888284668582285e-06, "epoch": 0.86, "percentage": 85.68, "elapsed_time": "0:39:27", "remaining_time": "0:06:35"}
+{"current_steps": 650, "total_steps": 747, "loss": 0.3124, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.051543625192226e-06, "epoch": 0.87, "percentage": 87.01, "elapsed_time": "0:40:04", "remaining_time": "0:05:58"}
+{"current_steps": 660, "total_steps": 747, "loss": 0.2644, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.6548421441183875e-06, "epoch": 0.88, "percentage": 88.35, "elapsed_time": "0:40:43", "remaining_time": "0:05:22"}
+{"current_steps": 670, "total_steps": 747, "loss": 0.2622, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.2994255731871963e-06, "epoch": 0.9, "percentage": 89.69, "elapsed_time": "0:41:18", "remaining_time": "0:04:44"}
+{"current_steps": 680, "total_steps": 747, "loss": 0.2234, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 9.85922451356694e-07, "epoch": 0.91, "percentage": 91.03, "elapsed_time": "0:41:52", "remaining_time": "0:04:07"}
+{"current_steps": 690, "total_steps": 747, "loss": 0.334, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 7.148871954483105e-07, "epoch": 0.92, "percentage": 92.37, "elapsed_time": "0:42:29", "remaining_time": "0:03:30"}
+{"current_steps": 700, "total_steps": 747, "loss": 0.2915, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.867991196844918e-07, "epoch": 0.94, "percentage": 93.71, "elapsed_time": "0:43:07", "remaining_time": "0:02:53"}
+{"current_steps": 710, "total_steps": 747, "loss": 0.2802, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.020615880420713e-07, "epoch": 0.95, "percentage": 95.05, "elapsed_time": "0:43:42", "remaining_time": "0:02:16"}
+{"current_steps": 720, "total_steps": 747, "loss": 0.2975, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.6100130092037703e-07, "epoch": 0.96, "percentage": 96.39, "elapsed_time": "0:44:17", "remaining_time": "0:01:39"}
+{"current_steps": 730, "total_steps": 747, "loss": 0.2531, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 6.386771738558506e-08, "epoch": 0.98, "percentage": 97.72, "elapsed_time": "0:44:53", "remaining_time": "0:01:02"}
+{"current_steps": 740, "total_steps": 747, "loss": 0.2696, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.0832614013073228e-08, "epoch": 0.99, "percentage": 99.06, "elapsed_time": "0:45:30", "remaining_time": "0:00:25"}
+{"current_steps": 747, "total_steps": 747, "loss": null, "eval_loss": 0.27866294980049133, "predict_loss": null, "reward": null, "learning_rate": null, "epoch": 1.0, "percentage": 100.0, "elapsed_time": "0:45:56", "remaining_time": "0:00:00"}
+{"current_steps": 747, "total_steps": 747, "loss": null, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": null, "epoch": 1.0, "percentage": 100.0, "elapsed_time": "0:45:56", "remaining_time": "0:00:00"}
+{"current_steps": 332, "total_steps": 332, "loss": null, "eval_loss": 0.27866294980049133, "predict_loss": null, "reward": null, "learning_rate": null, "epoch": 1.0, "percentage": 100.0, "elapsed_time": "0:49:22", "remaining_time": "0:00:00"}

trainer_state.json

@@ -0,0 +1,482 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 747,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01,
+      "learning_rate": 4.997789428625975e-05,
+      "loss": 1.6077,
+      "step": 10
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 4.99116162380454e-05,
+      "loss": 0.9415,
+      "step": 20
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 4.980128306524183e-05,
+      "loss": 0.6271,
+      "step": 30
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 4.964708988733178e-05,
+      "loss": 0.5335,
+      "step": 40
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 4.944930938833535e-05,
+      "loss": 0.3693,
+      "step": 50
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.9208291334580104e-05,
+      "loss": 0.47,
+      "step": 60
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.892446195615423e-05,
+      "loss": 0.4268,
+      "step": 70
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.859832319313697e-05,
+      "loss": 0.3653,
+      "step": 80
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.8230451807939135e-05,
+      "loss": 0.3466,
+      "step": 90
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.782149836532345e-05,
+      "loss": 0.3241,
+      "step": 100
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.737218608190878e-05,
+      "loss": 0.3763,
+      "step": 110
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.6883309547192476e-05,
+      "loss": 0.3333,
+      "step": 120
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.635573331835302e-05,
+      "loss": 0.3106,
+      "step": 130
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.5790390391317675e-05,
+      "loss": 0.3241,
+      "step": 140
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.518828055079925e-05,
+      "loss": 0.2781,
+      "step": 150
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.4550468602219716e-05,
+      "loss": 0.3,
+      "step": 160
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.387808248864751e-05,
+      "loss": 0.3229,
+      "step": 170
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.3172311296078595e-05,
+      "loss": 0.3347,
+      "step": 180
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.2434403150588895e-05,
+      "loss": 0.2897,
+      "step": 190
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.166566301107687e-05,
+      "loss": 0.2951,
+      "step": 200
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.08674503614997e-05,
+      "loss": 0.3081,
+      "step": 210
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.004117680668422e-05,
+      "loss": 0.2889,
+      "step": 220
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 3.918830357596434e-05,
+      "loss": 0.3207,
+      "step": 230
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 3.8310338939059644e-05,
+      "loss": 0.3238,
+      "step": 240
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 3.740883553876515e-05,
+      "loss": 0.3146,
+      "step": 250
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 3.6485387645169064e-05,
+      "loss": 0.2973,
+      "step": 260
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 3.55416283362546e-05,
+      "loss": 0.2631,
+      "step": 270
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.457922660987155e-05,
+      "loss": 0.3323,
+      "step": 280
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.3599884432185225e-05,
+      "loss": 0.2631,
+      "step": 290
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.260533372782234e-05,
+      "loss": 0.3066,
+      "step": 300
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.1597333317036545e-05,
+      "loss": 0.3131,
+      "step": 310
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.057766580531031e-05,
+      "loss": 0.3101,
+      "step": 320
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 2.9548134430893604e-05,
+      "loss": 0.3022,
+      "step": 330
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 2.8510559875854377e-05,
+      "loss": 0.2496,
+      "step": 340
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 2.7466777046280457e-05,
+      "loss": 0.3218,
+      "step": 350
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 2.6418631827326857e-05,
+      "loss": 0.3064,
+      "step": 360
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.5367977818847034e-05,
+      "loss": 0.2937,
+      "step": 370
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.431667305738112e-05,
+      "loss": 0.2708,
+      "step": 380
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.3266576730297956e-05,
+      "loss": 0.2838,
+      "step": 390
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.221954588790206e-05,
+      "loss": 0.2987,
+      "step": 400
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.1177432159319754e-05,
+      "loss": 0.3069,
+      "step": 410
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.014207847797256e-05,
+      "loss": 0.2709,
+      "step": 420
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 1.9115315822428437e-05,
+      "loss": 0.3159,
+      "step": 430
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 1.809895997839482e-05,
+      "loss": 0.2997,
+      "step": 440
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 1.70948083275794e-05,
+      "loss": 0.2347,
+      "step": 450
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.6104636669097776e-05,
+      "loss": 0.2634,
+      "step": 460
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.513019607904882e-05,
+      "loss": 0.2438,
+      "step": 470
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.4173209813811788e-05,
+      "loss": 0.2289,
+      "step": 480
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.3235370262541272e-05,
+      "loss": 0.2609,
+      "step": 490
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.2318335954249669e-05,
+      "loss": 0.2724,
+      "step": 500
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.1423728624769695e-05,
+      "loss": 0.272,
+      "step": 510
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.0553130348784182e-05,
+      "loss": 0.303,
+      "step": 520
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 9.708080741994868e-06,
+      "loss": 0.2766,
+      "step": 530
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 8.890074238378074e-06,
+      "loss": 0.2333,
+      "step": 540
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 8.100557447342327e-06,
+      "loss": 0.2326,
+      "step": 550
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 7.340926595461687e-06,
+      "loss": 0.2274,
+      "step": 560
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 6.612525057308949e-06,
+      "loss": 0.3255,
+      "step": 570
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 5.9166409797553415e-06,
+      "loss": 0.2631,
+      "step": 580
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 5.254505003938043e-06,
+      "loss": 0.2938,
+      "step": 590
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 4.627288088924156e-06,
+      "loss": 0.2807,
+      "step": 600
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.036099440919763e-06,
+      "loss": 0.2909,
+      "step": 610
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 3.481984551686429e-06,
+      "loss": 0.2997,
+      "step": 620
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 2.9659233496337786e-06,
+      "loss": 0.2424,
+      "step": 630
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.4888284668582285e-06,
+      "loss": 0.2862,
+      "step": 640
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.051543625192226e-06,
+      "loss": 0.3124,
+      "step": 650
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 1.6548421441183875e-06,
+      "loss": 0.2644,
+      "step": 660
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.2994255731871963e-06,
+      "loss": 0.2622,
+      "step": 670
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 9.85922451356694e-07,
+      "loss": 0.2234,
+      "step": 680
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 7.148871954483105e-07,
+      "loss": 0.334,
+      "step": 690
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.867991196844918e-07,
+      "loss": 0.2915,
+      "step": 700
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.020615880420713e-07,
+      "loss": 0.2802,
+      "step": 710
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.6100130092037703e-07,
+      "loss": 0.2975,
+      "step": 720
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 6.386771738558506e-08,
+      "loss": 0.2531,
+      "step": 730
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.0832614013073228e-08,
+      "loss": 0.2696,
+      "step": 740
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 0.27866294980049133,
+      "eval_runtime": 101.2195,
+      "eval_samples_per_second": 6.56,
+      "eval_steps_per_second": 3.28,
+      "step": 747
+    },
+    {
+      "epoch": 1.0,
+      "step": 747,
+      "total_flos": 1.7883373351469056e+17,
+      "train_loss": 0.3299662241814446,
+      "train_runtime": 2859.8545,
+      "train_samples_per_second": 2.089,
+      "train_steps_per_second": 0.261
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 747,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "total_flos": 1.7883373351469056e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05d9caf8806a8cf459511820a5f5aad699ab76b8ce2d0478c978f6d2e0bdc492
+size 4984