diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f3cb5980881141d8dab1c0d57416c53aaa64470b --- /dev/null +++ b/README.md @@ -0,0 +1,18 @@ +## Training procedure + + +The following `bitsandbytes` quantization config was used during training: +- quant_method: bitsandbytes +- load_in_8bit: True +- load_in_4bit: False +- llm_int8_threshold: 6.0 +- llm_int8_skip_modules: None +- llm_int8_enable_fp32_cpu_offload: False +- llm_int8_has_fp16_weight: False +- bnb_4bit_quant_type: fp4 +- bnb_4bit_use_double_quant: False +- bnb_4bit_compute_dtype: float32 +### Framework versions + + +- PEFT 0.6.0.dev0 diff --git a/adapter_config.json b/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..aeb529678e2f5a4463eb51f5ae04ecbbb9ede5bb --- /dev/null +++ b/adapter_config.json @@ -0,0 +1,18 @@ +{ + "base_model_name_or_path": "decapoda-research/llama-7b-hf", + "bias": "none", + "enable_lora": null, + "fan_in_fan_out": false, + "inference_mode": true, + "lora_alpha": 16, + "lora_dropout": 0.05, + "merge_weights": false, + "modules_to_save": null, + "peft_type": "LORA", + "r": 8, + "target_modules": [ + "q_proj", + "v_proj" + ], + "task_type": "CAUSAL_LM" +} \ No newline at end of file diff --git a/adapter_model.bin b/adapter_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..8710093665dfdb9819e2f8817a1e25a4ccdd9935 --- /dev/null +++ b/adapter_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5e1621f48d9ad8feb1d6d31050275f0aafd080c5c07153301fe2f48411f4406 +size 443 diff --git a/checkpoint-1000/README.md b/checkpoint-1000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5f4f5625039e17bc7326cc10fbbc7896bb06453b --- /dev/null +++ b/checkpoint-1000/README.md @@ -0,0 +1,21 @@ +--- +library_name: peft +--- +## Training procedure + + +The following `bitsandbytes` quantization config was used during training: +- quant_method: bitsandbytes +- load_in_8bit: True +- load_in_4bit: False +- llm_int8_threshold: 6.0 +- llm_int8_skip_modules: None +- llm_int8_enable_fp32_cpu_offload: False +- llm_int8_has_fp16_weight: False +- bnb_4bit_quant_type: fp4 +- bnb_4bit_use_double_quant: False +- bnb_4bit_compute_dtype: float32 +### Framework versions + + +- PEFT 0.6.0.dev0 diff --git a/checkpoint-1000/adapter_config.json b/checkpoint-1000/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6fddf8fb6febbb9b217e5849180ec2fc8aab3c0e --- /dev/null +++ b/checkpoint-1000/adapter_config.json @@ -0,0 +1,21 @@ +{ + "auto_mapping": null, + "base_model_name_or_path": "../chinese-llama-2-13b", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layers_pattern": null, + "layers_to_transform": null, + "lora_alpha": 8, + "lora_dropout": 0.05, + "modules_to_save": null, + "peft_type": "LORA", + "r": 8, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj" + ], + "task_type": "CAUSAL_LM" +} \ No newline at end of file diff --git a/checkpoint-1000/adapter_model.bin b/checkpoint-1000/adapter_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..8710093665dfdb9819e2f8817a1e25a4ccdd9935 --- /dev/null +++ b/checkpoint-1000/adapter_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5e1621f48d9ad8feb1d6d31050275f0aafd080c5c07153301fe2f48411f4406 +size 443 diff --git a/checkpoint-1000/optimizer.pt b/checkpoint-1000/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a41f5f8762e6ba5af97252348be0cf673b71e21 --- /dev/null +++ b/checkpoint-1000/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:519989b5c4203349488ab3a72641e430710473f479d606018f1c5ea510bda3ad +size 52562757 diff --git a/checkpoint-1000/rng_state_0.pth b/checkpoint-1000/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..df5e2930df0216449742769f05d8587fd1c09da4 --- /dev/null +++ b/checkpoint-1000/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d555fa2b7f06822b0ea390d8be6e59b9c2c1df90e9a970218bbcaa9e658dd3f +size 18679 diff --git a/checkpoint-1000/rng_state_1.pth b/checkpoint-1000/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..1cc20e0acf715a87c0c97cb6b6a74a1dcf31f427 --- /dev/null +++ b/checkpoint-1000/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6c65da58622be8f443206bf19b99171fcc17cd5b3062a5d474cfea0d421928e +size 18679 diff --git a/checkpoint-1000/rng_state_2.pth b/checkpoint-1000/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..c227bd7c5e7f4b1e44c525d06457b325fc0a5630 --- /dev/null +++ b/checkpoint-1000/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9e46a0408e00234db670e5678b09d818d17644da30b372c23365cf4553d0eab +size 18679 diff --git a/checkpoint-1000/rng_state_3.pth b/checkpoint-1000/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..e68eac62d2b6f5b955ce89870c83e491a9ee3ee8 --- /dev/null +++ b/checkpoint-1000/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37db40b7a6693356857b0b1ef19519a6e374c4a22ecba286d5be22660bd721c8 +size 18679 diff --git a/checkpoint-1000/rng_state_4.pth b/checkpoint-1000/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..17c97806ff07950e4b70d631095de042b3442a52 --- /dev/null +++ b/checkpoint-1000/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e2da9f2cedddbb29bd7fdcf9b4ccb986ed0aae0e221ad3615744cbbc3e72a38 +size 18679 diff --git a/checkpoint-1000/scheduler.pt b/checkpoint-1000/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..115627f12fc729fa65cd18cceb30e90b7f6994ab --- /dev/null +++ b/checkpoint-1000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85d443feb657783013f5feaa6f3fca061b9db934a594151c8d1707fc020521ed +size 627 diff --git a/checkpoint-1000/trainer_state.json b/checkpoint-1000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..4b7cbd3c4f4a26f4808ce044196f9896aa278daa --- /dev/null +++ b/checkpoint-1000/trainer_state.json @@ -0,0 +1,359 @@ +{ + "best_metric": 1.2406948804855347, + "best_model_checkpoint": "../llama2-9439-21sept/checkpoint-1000", + "epoch": 1.7366136034732271, + "eval_steps": 200, + "global_step": 1000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.03, + "learning_rate": 5.9999999999999995e-05, + "loss": 3.4609, + "step": 20 + }, + { + "epoch": 0.07, + "learning_rate": 0.00011999999999999999, + "loss": 2.8346, + "step": 40 + }, + { + "epoch": 0.1, + "learning_rate": 0.00017999999999999998, + "loss": 1.7597, + "step": 60 + }, + { + "epoch": 0.14, + "learning_rate": 0.00023999999999999998, + "loss": 1.542, + "step": 80 + }, + { + "epoch": 0.17, + "learning_rate": 0.0003, + "loss": 1.4567, + "step": 100 + }, + { + "epoch": 0.21, + "learning_rate": 0.0002963076923076923, + "loss": 1.4081, + "step": 120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0002926153846153846, + "loss": 1.3805, + "step": 140 + }, + { + "epoch": 0.28, + "learning_rate": 0.0002889230769230769, + "loss": 1.3605, + "step": 160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00028523076923076923, + "loss": 1.3329, + "step": 180 + }, + { + "epoch": 0.35, + "learning_rate": 0.0002815384615384615, + "loss": 1.3367, + "step": 200 + }, + { + "epoch": 0.35, + "eval_loss": 1.3450770378112793, + "eval_runtime": 6.306, + "eval_samples_per_second": 47.574, + "eval_steps_per_second": 1.269, + "step": 200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0002778461538461538, + "loss": 1.3152, + "step": 220 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002741538461538461, + "loss": 1.312, + "step": 240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027046153846153843, + "loss": 1.2883, + "step": 260 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026676923076923074, + "loss": 1.2843, + "step": 280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00026307692307692306, + "loss": 1.2609, + "step": 300 + }, + { + "epoch": 0.56, + "learning_rate": 0.00025938461538461537, + "loss": 1.2707, + "step": 320 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002556923076923077, + "loss": 1.2624, + "step": 340 + }, + { + "epoch": 0.63, + "learning_rate": 0.00025199999999999995, + "loss": 1.273, + "step": 360 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002483076923076923, + "loss": 1.251, + "step": 380 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002446153846153846, + "loss": 1.2648, + "step": 400 + }, + { + "epoch": 0.69, + "eval_loss": 1.292348861694336, + "eval_runtime": 6.304, + "eval_samples_per_second": 47.589, + "eval_steps_per_second": 1.269, + "step": 400 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002409230769230769, + "loss": 1.265, + "step": 420 + }, + { + "epoch": 0.76, + "learning_rate": 0.0002372307692307692, + "loss": 1.2516, + "step": 440 + }, + { + "epoch": 0.8, + "learning_rate": 0.00023353846153846151, + "loss": 1.2441, + "step": 460 + }, + { + "epoch": 0.83, + "learning_rate": 0.00022984615384615383, + "loss": 1.2204, + "step": 480 + }, + { + "epoch": 0.87, + "learning_rate": 0.00022615384615384614, + "loss": 1.2221, + "step": 500 + }, + { + "epoch": 0.9, + "learning_rate": 0.00022246153846153846, + "loss": 1.2246, + "step": 520 + }, + { + "epoch": 0.94, + "learning_rate": 0.00021876923076923074, + "loss": 1.2227, + "step": 540 + }, + { + "epoch": 0.97, + "learning_rate": 0.00021507692307692306, + "loss": 1.2124, + "step": 560 + }, + { + "epoch": 1.01, + "learning_rate": 0.00021138461538461537, + "loss": 1.2065, + "step": 580 + }, + { + "epoch": 1.04, + "learning_rate": 0.00020769230769230766, + "loss": 1.2106, + "step": 600 + }, + { + "epoch": 1.04, + "eval_loss": 1.2669302225112915, + "eval_runtime": 6.3128, + "eval_samples_per_second": 47.523, + "eval_steps_per_second": 1.267, + "step": 600 + }, + { + "epoch": 1.08, + "learning_rate": 0.000204, + "loss": 1.2046, + "step": 620 + }, + { + "epoch": 1.11, + "learning_rate": 0.00020030769230769229, + "loss": 1.2114, + "step": 640 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001966153846153846, + "loss": 1.2122, + "step": 660 + }, + { + "epoch": 1.18, + "learning_rate": 0.00019310769230769227, + "loss": 1.1963, + "step": 680 + }, + { + "epoch": 1.22, + "learning_rate": 0.00018941538461538461, + "loss": 1.1965, + "step": 700 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001857230769230769, + "loss": 1.1937, + "step": 720 + }, + { + "epoch": 1.29, + "learning_rate": 0.00018203076923076921, + "loss": 1.186, + "step": 740 + }, + { + "epoch": 1.32, + "learning_rate": 0.00017833846153846153, + "loss": 1.1732, + "step": 760 + }, + { + "epoch": 1.35, + "learning_rate": 0.00017464615384615381, + "loss": 1.1911, + "step": 780 + }, + { + "epoch": 1.39, + "learning_rate": 0.00017095384615384616, + "loss": 1.2153, + "step": 800 + }, + { + "epoch": 1.39, + "eval_loss": 1.2513903379440308, + "eval_runtime": 6.3069, + "eval_samples_per_second": 47.567, + "eval_steps_per_second": 1.268, + "step": 800 + }, + { + "epoch": 1.42, + "learning_rate": 0.00016726153846153844, + "loss": 1.1991, + "step": 820 + }, + { + "epoch": 1.46, + "learning_rate": 0.00016356923076923073, + "loss": 1.1934, + "step": 840 + }, + { + "epoch": 1.49, + "learning_rate": 0.00015987692307692307, + "loss": 1.2047, + "step": 860 + }, + { + "epoch": 1.53, + "learning_rate": 0.00015618461538461536, + "loss": 1.2204, + "step": 880 + }, + { + "epoch": 1.56, + "learning_rate": 0.0001524923076923077, + "loss": 1.1785, + "step": 900 + }, + { + "epoch": 1.6, + "learning_rate": 0.00014879999999999998, + "loss": 1.1756, + "step": 920 + }, + { + "epoch": 1.63, + "learning_rate": 0.0001451076923076923, + "loss": 1.214, + "step": 940 + }, + { + "epoch": 1.67, + "learning_rate": 0.0001414153846153846, + "loss": 1.1794, + "step": 960 + }, + { + "epoch": 1.7, + "learning_rate": 0.00013772307692307693, + "loss": 1.1946, + "step": 980 + }, + { + "epoch": 1.74, + "learning_rate": 0.0001340307692307692, + "loss": 1.175, + "step": 1000 + }, + { + "epoch": 1.74, + "eval_loss": 1.2406948804855347, + "eval_runtime": 6.3315, + "eval_samples_per_second": 47.382, + "eval_steps_per_second": 1.264, + "step": 1000 + } + ], + "logging_steps": 20, + "max_steps": 1725, + "num_train_epochs": 3, + "save_steps": 200, + "total_flos": 2.3920781700366336e+18, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1000/training_args.bin b/checkpoint-1000/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..14a45b06b3e8ec543e02c64e959e660395a27125 --- /dev/null +++ b/checkpoint-1000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e5fa4a6be80dea6c8018bb49732568d9b22103b1e3942cf21f3b067cbdff444 +size 4027 diff --git a/checkpoint-1200/README.md b/checkpoint-1200/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5f4f5625039e17bc7326cc10fbbc7896bb06453b --- /dev/null +++ b/checkpoint-1200/README.md @@ -0,0 +1,21 @@ +--- +library_name: peft +--- +## Training procedure + + +The following `bitsandbytes` quantization config was used during training: +- quant_method: bitsandbytes +- load_in_8bit: True +- load_in_4bit: False +- llm_int8_threshold: 6.0 +- llm_int8_skip_modules: None +- llm_int8_enable_fp32_cpu_offload: False +- llm_int8_has_fp16_weight: False +- bnb_4bit_quant_type: fp4 +- bnb_4bit_use_double_quant: False +- bnb_4bit_compute_dtype: float32 +### Framework versions + + +- PEFT 0.6.0.dev0 diff --git a/checkpoint-1200/adapter_config.json b/checkpoint-1200/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6fddf8fb6febbb9b217e5849180ec2fc8aab3c0e --- /dev/null +++ b/checkpoint-1200/adapter_config.json @@ -0,0 +1,21 @@ +{ + "auto_mapping": null, + "base_model_name_or_path": "../chinese-llama-2-13b", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layers_pattern": null, + "layers_to_transform": null, + "lora_alpha": 8, + "lora_dropout": 0.05, + "modules_to_save": null, + "peft_type": "LORA", + "r": 8, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj" + ], + "task_type": "CAUSAL_LM" +} \ No newline at end of file diff --git a/checkpoint-1200/adapter_model.bin b/checkpoint-1200/adapter_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..8710093665dfdb9819e2f8817a1e25a4ccdd9935 --- /dev/null +++ b/checkpoint-1200/adapter_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5e1621f48d9ad8feb1d6d31050275f0aafd080c5c07153301fe2f48411f4406 +size 443 diff --git a/checkpoint-1200/optimizer.pt b/checkpoint-1200/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b6aabbccae52b4181422720d49e3569a1d52938 --- /dev/null +++ b/checkpoint-1200/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2ee310a6cf6a32e11abc2f6a70bd55159774708ca656382570d6b68a4b853d4 +size 52562757 diff --git a/checkpoint-1200/rng_state_0.pth b/checkpoint-1200/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..92a4b26c790ba16ebadbcd3e60c5ba5c722d6c9a --- /dev/null +++ b/checkpoint-1200/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:688bcd22520549b8a89c6a81ce93e9faccc9eb18550468cab05ae219f18538ff +size 18679 diff --git a/checkpoint-1200/rng_state_1.pth b/checkpoint-1200/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..c5b53d7213d80eef494ed6d4f53ade7f8fe2947e --- /dev/null +++ b/checkpoint-1200/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b994e3ef13cb877d826e3c6e57158a640a118b797654c5dd97ff90090d295e3 +size 18679 diff --git a/checkpoint-1200/rng_state_2.pth b/checkpoint-1200/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..a78e4ebc0fb49c52f3698f48fe39ec658fef3565 --- /dev/null +++ b/checkpoint-1200/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be4b3e81825c582eb5c46ad800e0b5989101507dbf4122ddcdf1cab9a466e935 +size 18679 diff --git a/checkpoint-1200/rng_state_3.pth b/checkpoint-1200/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..4f9c75bb3774f90eb70960b89cf7bfe70bf5f0ec --- /dev/null +++ b/checkpoint-1200/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbdbbb81ee067d0a8931757b45688b5cb869fe14445248e936ab6ba60e3c9781 +size 18679 diff --git a/checkpoint-1200/rng_state_4.pth b/checkpoint-1200/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..1ed9442320b7af9b0b24c38d94296ea3d9727078 --- /dev/null +++ b/checkpoint-1200/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:367ccf72cc4f5ec03b6635cb5a96094b34ab82051aefa6cddd5ff80a2655e832 +size 18679 diff --git a/checkpoint-1200/scheduler.pt b/checkpoint-1200/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..b53cb6ef7113e166e5699c3ed9b335a0538370ce --- /dev/null +++ b/checkpoint-1200/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2691a01f405e053da7ad7bcb0b3a2022cbc94dd82e3b77720da780c7ba897642 +size 627 diff --git a/checkpoint-1200/trainer_state.json b/checkpoint-1200/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a770dd297e7a15a9c7bc26ea3a1a49a4d77e6ebf --- /dev/null +++ b/checkpoint-1200/trainer_state.json @@ -0,0 +1,427 @@ +{ + "best_metric": 1.2316911220550537, + "best_model_checkpoint": "../llama2-9439-21sept/checkpoint-1200", + "epoch": 2.0839363241678726, + "eval_steps": 200, + "global_step": 1200, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.03, + "learning_rate": 5.9999999999999995e-05, + "loss": 3.4609, + "step": 20 + }, + { + "epoch": 0.07, + "learning_rate": 0.00011999999999999999, + "loss": 2.8346, + "step": 40 + }, + { + "epoch": 0.1, + "learning_rate": 0.00017999999999999998, + "loss": 1.7597, + "step": 60 + }, + { + "epoch": 0.14, + "learning_rate": 0.00023999999999999998, + "loss": 1.542, + "step": 80 + }, + { + "epoch": 0.17, + "learning_rate": 0.0003, + "loss": 1.4567, + "step": 100 + }, + { + "epoch": 0.21, + "learning_rate": 0.0002963076923076923, + "loss": 1.4081, + "step": 120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0002926153846153846, + "loss": 1.3805, + "step": 140 + }, + { + "epoch": 0.28, + "learning_rate": 0.0002889230769230769, + "loss": 1.3605, + "step": 160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00028523076923076923, + "loss": 1.3329, + "step": 180 + }, + { + "epoch": 0.35, + "learning_rate": 0.0002815384615384615, + "loss": 1.3367, + "step": 200 + }, + { + "epoch": 0.35, + "eval_loss": 1.3450770378112793, + "eval_runtime": 6.306, + "eval_samples_per_second": 47.574, + "eval_steps_per_second": 1.269, + "step": 200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0002778461538461538, + "loss": 1.3152, + "step": 220 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002741538461538461, + "loss": 1.312, + "step": 240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027046153846153843, + "loss": 1.2883, + "step": 260 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026676923076923074, + "loss": 1.2843, + "step": 280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00026307692307692306, + "loss": 1.2609, + "step": 300 + }, + { + "epoch": 0.56, + "learning_rate": 0.00025938461538461537, + "loss": 1.2707, + "step": 320 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002556923076923077, + "loss": 1.2624, + "step": 340 + }, + { + "epoch": 0.63, + "learning_rate": 0.00025199999999999995, + "loss": 1.273, + "step": 360 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002483076923076923, + "loss": 1.251, + "step": 380 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002446153846153846, + "loss": 1.2648, + "step": 400 + }, + { + "epoch": 0.69, + "eval_loss": 1.292348861694336, + "eval_runtime": 6.304, + "eval_samples_per_second": 47.589, + "eval_steps_per_second": 1.269, + "step": 400 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002409230769230769, + "loss": 1.265, + "step": 420 + }, + { + "epoch": 0.76, + "learning_rate": 0.0002372307692307692, + "loss": 1.2516, + "step": 440 + }, + { + "epoch": 0.8, + "learning_rate": 0.00023353846153846151, + "loss": 1.2441, + "step": 460 + }, + { + "epoch": 0.83, + "learning_rate": 0.00022984615384615383, + "loss": 1.2204, + "step": 480 + }, + { + "epoch": 0.87, + "learning_rate": 0.00022615384615384614, + "loss": 1.2221, + "step": 500 + }, + { + "epoch": 0.9, + "learning_rate": 0.00022246153846153846, + "loss": 1.2246, + "step": 520 + }, + { + "epoch": 0.94, + "learning_rate": 0.00021876923076923074, + "loss": 1.2227, + "step": 540 + }, + { + "epoch": 0.97, + "learning_rate": 0.00021507692307692306, + "loss": 1.2124, + "step": 560 + }, + { + "epoch": 1.01, + "learning_rate": 0.00021138461538461537, + "loss": 1.2065, + "step": 580 + }, + { + "epoch": 1.04, + "learning_rate": 0.00020769230769230766, + "loss": 1.2106, + "step": 600 + }, + { + "epoch": 1.04, + "eval_loss": 1.2669302225112915, + "eval_runtime": 6.3128, + "eval_samples_per_second": 47.523, + "eval_steps_per_second": 1.267, + "step": 600 + }, + { + "epoch": 1.08, + "learning_rate": 0.000204, + "loss": 1.2046, + "step": 620 + }, + { + "epoch": 1.11, + "learning_rate": 0.00020030769230769229, + "loss": 1.2114, + "step": 640 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001966153846153846, + "loss": 1.2122, + "step": 660 + }, + { + "epoch": 1.18, + "learning_rate": 0.00019310769230769227, + "loss": 1.1963, + "step": 680 + }, + { + "epoch": 1.22, + "learning_rate": 0.00018941538461538461, + "loss": 1.1965, + "step": 700 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001857230769230769, + "loss": 1.1937, + "step": 720 + }, + { + "epoch": 1.29, + "learning_rate": 0.00018203076923076921, + "loss": 1.186, + "step": 740 + }, + { + "epoch": 1.32, + "learning_rate": 0.00017833846153846153, + "loss": 1.1732, + "step": 760 + }, + { + "epoch": 1.35, + "learning_rate": 0.00017464615384615381, + "loss": 1.1911, + "step": 780 + }, + { + "epoch": 1.39, + "learning_rate": 0.00017095384615384616, + "loss": 1.2153, + "step": 800 + }, + { + "epoch": 1.39, + "eval_loss": 1.2513903379440308, + "eval_runtime": 6.3069, + "eval_samples_per_second": 47.567, + "eval_steps_per_second": 1.268, + "step": 800 + }, + { + "epoch": 1.42, + "learning_rate": 0.00016726153846153844, + "loss": 1.1991, + "step": 820 + }, + { + "epoch": 1.46, + "learning_rate": 0.00016356923076923073, + "loss": 1.1934, + "step": 840 + }, + { + "epoch": 1.49, + "learning_rate": 0.00015987692307692307, + "loss": 1.2047, + "step": 860 + }, + { + "epoch": 1.53, + "learning_rate": 0.00015618461538461536, + "loss": 1.2204, + "step": 880 + }, + { + "epoch": 1.56, + "learning_rate": 0.0001524923076923077, + "loss": 1.1785, + "step": 900 + }, + { + "epoch": 1.6, + "learning_rate": 0.00014879999999999998, + "loss": 1.1756, + "step": 920 + }, + { + "epoch": 1.63, + "learning_rate": 0.0001451076923076923, + "loss": 1.214, + "step": 940 + }, + { + "epoch": 1.67, + "learning_rate": 0.0001414153846153846, + "loss": 1.1794, + "step": 960 + }, + { + "epoch": 1.7, + "learning_rate": 0.00013772307692307693, + "loss": 1.1946, + "step": 980 + }, + { + "epoch": 1.74, + "learning_rate": 0.0001340307692307692, + "loss": 1.175, + "step": 1000 + }, + { + "epoch": 1.74, + "eval_loss": 1.2406948804855347, + "eval_runtime": 6.3315, + "eval_samples_per_second": 47.382, + "eval_steps_per_second": 1.264, + "step": 1000 + }, + { + "epoch": 1.77, + "learning_rate": 0.00013033846153846153, + "loss": 1.178, + "step": 1020 + }, + { + "epoch": 1.81, + "learning_rate": 0.00012664615384615384, + "loss": 1.1859, + "step": 1040 + }, + { + "epoch": 1.84, + "learning_rate": 0.00012295384615384615, + "loss": 1.1801, + "step": 1060 + }, + { + "epoch": 1.88, + "learning_rate": 0.00011926153846153845, + "loss": 1.1743, + "step": 1080 + }, + { + "epoch": 1.91, + "learning_rate": 0.00011556923076923076, + "loss": 1.1596, + "step": 1100 + }, + { + "epoch": 1.95, + "learning_rate": 0.00011187692307692307, + "loss": 1.1495, + "step": 1120 + }, + { + "epoch": 1.98, + "learning_rate": 0.00010818461538461537, + "loss": 1.1808, + "step": 1140 + }, + { + "epoch": 2.01, + "learning_rate": 0.00010449230769230768, + "loss": 1.1743, + "step": 1160 + }, + { + "epoch": 2.05, + "learning_rate": 0.0001008, + "loss": 1.1354, + "step": 1180 + }, + { + "epoch": 2.08, + "learning_rate": 9.71076923076923e-05, + "loss": 1.1712, + "step": 1200 + }, + { + "epoch": 2.08, + "eval_loss": 1.2316911220550537, + "eval_runtime": 6.3191, + "eval_samples_per_second": 47.475, + "eval_steps_per_second": 1.266, + "step": 1200 + } + ], + "logging_steps": 20, + "max_steps": 1725, + "num_train_epochs": 3, + "save_steps": 200, + "total_flos": 2.8704938040439603e+18, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1200/training_args.bin b/checkpoint-1200/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..14a45b06b3e8ec543e02c64e959e660395a27125 --- /dev/null +++ b/checkpoint-1200/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e5fa4a6be80dea6c8018bb49732568d9b22103b1e3942cf21f3b067cbdff444 +size 4027 diff --git a/checkpoint-1400/README.md b/checkpoint-1400/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5f4f5625039e17bc7326cc10fbbc7896bb06453b --- /dev/null +++ b/checkpoint-1400/README.md @@ -0,0 +1,21 @@ +--- +library_name: peft +--- +## Training procedure + + +The following `bitsandbytes` quantization config was used during training: +- quant_method: bitsandbytes +- load_in_8bit: True +- load_in_4bit: False +- llm_int8_threshold: 6.0 +- llm_int8_skip_modules: None +- llm_int8_enable_fp32_cpu_offload: False +- llm_int8_has_fp16_weight: False +- bnb_4bit_quant_type: fp4 +- bnb_4bit_use_double_quant: False +- bnb_4bit_compute_dtype: float32 +### Framework versions + + +- PEFT 0.6.0.dev0 diff --git a/checkpoint-1400/adapter_config.json b/checkpoint-1400/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6fddf8fb6febbb9b217e5849180ec2fc8aab3c0e --- /dev/null +++ b/checkpoint-1400/adapter_config.json @@ -0,0 +1,21 @@ +{ + "auto_mapping": null, + "base_model_name_or_path": "../chinese-llama-2-13b", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layers_pattern": null, + "layers_to_transform": null, + "lora_alpha": 8, + "lora_dropout": 0.05, + "modules_to_save": null, + "peft_type": "LORA", + "r": 8, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj" + ], + "task_type": "CAUSAL_LM" +} \ No newline at end of file diff --git a/checkpoint-1400/adapter_model.bin b/checkpoint-1400/adapter_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..8710093665dfdb9819e2f8817a1e25a4ccdd9935 --- /dev/null +++ b/checkpoint-1400/adapter_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5e1621f48d9ad8feb1d6d31050275f0aafd080c5c07153301fe2f48411f4406 +size 443 diff --git a/checkpoint-1400/optimizer.pt b/checkpoint-1400/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e1457ae780179b913cc936e91b007ed06884c6b --- /dev/null +++ b/checkpoint-1400/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2a49b0f46ce2aee7f28498fb481e3333d0b4b3e6587d7060cd0db1d38793b26 +size 52562757 diff --git a/checkpoint-1400/rng_state_0.pth b/checkpoint-1400/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..a637f32ed3dc6a07a9ad1c9c761b9431d5619efe --- /dev/null +++ b/checkpoint-1400/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1e3dc83a5fca822891bba2d605222ab74cd99f1918102304ca4b65d3137d789 +size 18679 diff --git a/checkpoint-1400/rng_state_1.pth b/checkpoint-1400/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..989f832f555f5546819793d38e3359fdb81c0085 --- /dev/null +++ b/checkpoint-1400/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e64ecf9f57a459867d36288ab900f39ea916387621609dd3d66048fb0ea5d7c +size 18679 diff --git a/checkpoint-1400/rng_state_2.pth b/checkpoint-1400/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..f7780fcab85535dcfe531c08b5c3c0c78996380b --- /dev/null +++ b/checkpoint-1400/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58d7efa004a69734b841cd2f4519073da72d8de1db4e8465a104023e1003e073 +size 18679 diff --git a/checkpoint-1400/rng_state_3.pth b/checkpoint-1400/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..01281cc275eb299b8d53b2d8de48a243199d1e9d --- /dev/null +++ b/checkpoint-1400/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2d36476c1f68e57dbd03ff794033f730ef968f8d9e8aecf8299dc882a68cb2e +size 18679 diff --git a/checkpoint-1400/rng_state_4.pth b/checkpoint-1400/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..d7963fb140b33bd51c88c830699bbc944e909444 --- /dev/null +++ b/checkpoint-1400/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83eb914f8503bb55b84fbbf3b99e7ce79d625c6cc0f4d412abba661d5393af1a +size 18679 diff --git a/checkpoint-1400/scheduler.pt b/checkpoint-1400/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..9764563cd5365373aa1d6a505230cbd7d737c006 --- /dev/null +++ b/checkpoint-1400/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c5b9da5e9b4de58246d24ef97f5a656736c469da066e184cfd52c712accb4b1 +size 627 diff --git a/checkpoint-1400/trainer_state.json b/checkpoint-1400/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..518fb4b0d13cab6e2356dd0673b0ca80c4a80fbc --- /dev/null +++ b/checkpoint-1400/trainer_state.json @@ -0,0 +1,495 @@ +{ + "best_metric": 1.2265406847000122, + "best_model_checkpoint": "../llama2-9439-21sept/checkpoint-1400", + "epoch": 2.431259044862518, + "eval_steps": 200, + "global_step": 1400, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.03, + "learning_rate": 5.9999999999999995e-05, + "loss": 3.4609, + "step": 20 + }, + { + "epoch": 0.07, + "learning_rate": 0.00011999999999999999, + "loss": 2.8346, + "step": 40 + }, + { + "epoch": 0.1, + "learning_rate": 0.00017999999999999998, + "loss": 1.7597, + "step": 60 + }, + { + "epoch": 0.14, + "learning_rate": 0.00023999999999999998, + "loss": 1.542, + "step": 80 + }, + { + "epoch": 0.17, + "learning_rate": 0.0003, + "loss": 1.4567, + "step": 100 + }, + { + "epoch": 0.21, + "learning_rate": 0.0002963076923076923, + "loss": 1.4081, + "step": 120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0002926153846153846, + "loss": 1.3805, + "step": 140 + }, + { + "epoch": 0.28, + "learning_rate": 0.0002889230769230769, + "loss": 1.3605, + "step": 160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00028523076923076923, + "loss": 1.3329, + "step": 180 + }, + { + "epoch": 0.35, + "learning_rate": 0.0002815384615384615, + "loss": 1.3367, + "step": 200 + }, + { + "epoch": 0.35, + "eval_loss": 1.3450770378112793, + "eval_runtime": 6.306, + "eval_samples_per_second": 47.574, + "eval_steps_per_second": 1.269, + "step": 200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0002778461538461538, + "loss": 1.3152, + "step": 220 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002741538461538461, + "loss": 1.312, + "step": 240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027046153846153843, + "loss": 1.2883, + "step": 260 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026676923076923074, + "loss": 1.2843, + "step": 280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00026307692307692306, + "loss": 1.2609, + "step": 300 + }, + { + "epoch": 0.56, + "learning_rate": 0.00025938461538461537, + "loss": 1.2707, + "step": 320 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002556923076923077, + "loss": 1.2624, + "step": 340 + }, + { + "epoch": 0.63, + "learning_rate": 0.00025199999999999995, + "loss": 1.273, + "step": 360 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002483076923076923, + "loss": 1.251, + "step": 380 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002446153846153846, + "loss": 1.2648, + "step": 400 + }, + { + "epoch": 0.69, + "eval_loss": 1.292348861694336, + "eval_runtime": 6.304, + "eval_samples_per_second": 47.589, + "eval_steps_per_second": 1.269, + "step": 400 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002409230769230769, + "loss": 1.265, + "step": 420 + }, + { + "epoch": 0.76, + "learning_rate": 0.0002372307692307692, + "loss": 1.2516, + "step": 440 + }, + { + "epoch": 0.8, + "learning_rate": 0.00023353846153846151, + "loss": 1.2441, + "step": 460 + }, + { + "epoch": 0.83, + "learning_rate": 0.00022984615384615383, + "loss": 1.2204, + "step": 480 + }, + { + "epoch": 0.87, + "learning_rate": 0.00022615384615384614, + "loss": 1.2221, + "step": 500 + }, + { + "epoch": 0.9, + "learning_rate": 0.00022246153846153846, + "loss": 1.2246, + "step": 520 + }, + { + "epoch": 0.94, + "learning_rate": 0.00021876923076923074, + "loss": 1.2227, + "step": 540 + }, + { + "epoch": 0.97, + "learning_rate": 0.00021507692307692306, + "loss": 1.2124, + "step": 560 + }, + { + "epoch": 1.01, + "learning_rate": 0.00021138461538461537, + "loss": 1.2065, + "step": 580 + }, + { + "epoch": 1.04, + "learning_rate": 0.00020769230769230766, + "loss": 1.2106, + "step": 600 + }, + { + "epoch": 1.04, + "eval_loss": 1.2669302225112915, + "eval_runtime": 6.3128, + "eval_samples_per_second": 47.523, + "eval_steps_per_second": 1.267, + "step": 600 + }, + { + "epoch": 1.08, + "learning_rate": 0.000204, + "loss": 1.2046, + "step": 620 + }, + { + "epoch": 1.11, + "learning_rate": 0.00020030769230769229, + "loss": 1.2114, + "step": 640 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001966153846153846, + "loss": 1.2122, + "step": 660 + }, + { + "epoch": 1.18, + "learning_rate": 0.00019310769230769227, + "loss": 1.1963, + "step": 680 + }, + { + "epoch": 1.22, + "learning_rate": 0.00018941538461538461, + "loss": 1.1965, + "step": 700 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001857230769230769, + "loss": 1.1937, + "step": 720 + }, + { + "epoch": 1.29, + "learning_rate": 0.00018203076923076921, + "loss": 1.186, + "step": 740 + }, + { + "epoch": 1.32, + "learning_rate": 0.00017833846153846153, + "loss": 1.1732, + "step": 760 + }, + { + "epoch": 1.35, + "learning_rate": 0.00017464615384615381, + "loss": 1.1911, + "step": 780 + }, + { + "epoch": 1.39, + "learning_rate": 0.00017095384615384616, + "loss": 1.2153, + "step": 800 + }, + { + "epoch": 1.39, + "eval_loss": 1.2513903379440308, + "eval_runtime": 6.3069, + "eval_samples_per_second": 47.567, + "eval_steps_per_second": 1.268, + "step": 800 + }, + { + "epoch": 1.42, + "learning_rate": 0.00016726153846153844, + "loss": 1.1991, + "step": 820 + }, + { + "epoch": 1.46, + "learning_rate": 0.00016356923076923073, + "loss": 1.1934, + "step": 840 + }, + { + "epoch": 1.49, + "learning_rate": 0.00015987692307692307, + "loss": 1.2047, + "step": 860 + }, + { + "epoch": 1.53, + "learning_rate": 0.00015618461538461536, + "loss": 1.2204, + "step": 880 + }, + { + "epoch": 1.56, + "learning_rate": 0.0001524923076923077, + "loss": 1.1785, + "step": 900 + }, + { + "epoch": 1.6, + "learning_rate": 0.00014879999999999998, + "loss": 1.1756, + "step": 920 + }, + { + "epoch": 1.63, + "learning_rate": 0.0001451076923076923, + "loss": 1.214, + "step": 940 + }, + { + "epoch": 1.67, + "learning_rate": 0.0001414153846153846, + "loss": 1.1794, + "step": 960 + }, + { + "epoch": 1.7, + "learning_rate": 0.00013772307692307693, + "loss": 1.1946, + "step": 980 + }, + { + "epoch": 1.74, + "learning_rate": 0.0001340307692307692, + "loss": 1.175, + "step": 1000 + }, + { + "epoch": 1.74, + "eval_loss": 1.2406948804855347, + "eval_runtime": 6.3315, + "eval_samples_per_second": 47.382, + "eval_steps_per_second": 1.264, + "step": 1000 + }, + { + "epoch": 1.77, + "learning_rate": 0.00013033846153846153, + "loss": 1.178, + "step": 1020 + }, + { + "epoch": 1.81, + "learning_rate": 0.00012664615384615384, + "loss": 1.1859, + "step": 1040 + }, + { + "epoch": 1.84, + "learning_rate": 0.00012295384615384615, + "loss": 1.1801, + "step": 1060 + }, + { + "epoch": 1.88, + "learning_rate": 0.00011926153846153845, + "loss": 1.1743, + "step": 1080 + }, + { + "epoch": 1.91, + "learning_rate": 0.00011556923076923076, + "loss": 1.1596, + "step": 1100 + }, + { + "epoch": 1.95, + "learning_rate": 0.00011187692307692307, + "loss": 1.1495, + "step": 1120 + }, + { + "epoch": 1.98, + "learning_rate": 0.00010818461538461537, + "loss": 1.1808, + "step": 1140 + }, + { + "epoch": 2.01, + "learning_rate": 0.00010449230769230768, + "loss": 1.1743, + "step": 1160 + }, + { + "epoch": 2.05, + "learning_rate": 0.0001008, + "loss": 1.1354, + "step": 1180 + }, + { + "epoch": 2.08, + "learning_rate": 9.71076923076923e-05, + "loss": 1.1712, + "step": 1200 + }, + { + "epoch": 2.08, + "eval_loss": 1.2316911220550537, + "eval_runtime": 6.3191, + "eval_samples_per_second": 47.475, + "eval_steps_per_second": 1.266, + "step": 1200 + }, + { + "epoch": 2.12, + "learning_rate": 9.34153846153846e-05, + "loss": 1.1709, + "step": 1220 + }, + { + "epoch": 2.15, + "learning_rate": 8.972307692307691e-05, + "loss": 1.1667, + "step": 1240 + }, + { + "epoch": 2.19, + "learning_rate": 8.603076923076923e-05, + "loss": 1.1781, + "step": 1260 + }, + { + "epoch": 2.22, + "learning_rate": 8.233846153846154e-05, + "loss": 1.1628, + "step": 1280 + }, + { + "epoch": 2.26, + "learning_rate": 7.864615384615383e-05, + "loss": 1.1723, + "step": 1300 + }, + { + "epoch": 2.29, + "learning_rate": 7.495384615384615e-05, + "loss": 1.1578, + "step": 1320 + }, + { + "epoch": 2.33, + "learning_rate": 7.126153846153845e-05, + "loss": 1.1399, + "step": 1340 + }, + { + "epoch": 2.36, + "learning_rate": 6.756923076923077e-05, + "loss": 1.1612, + "step": 1360 + }, + { + "epoch": 2.4, + "learning_rate": 6.387692307692307e-05, + "loss": 1.1582, + "step": 1380 + }, + { + "epoch": 2.43, + "learning_rate": 6.0184615384615375e-05, + "loss": 1.1534, + "step": 1400 + }, + { + "epoch": 2.43, + "eval_loss": 1.2265406847000122, + "eval_runtime": 6.3176, + "eval_samples_per_second": 47.486, + "eval_steps_per_second": 1.266, + "step": 1400 + } + ], + "logging_steps": 20, + "max_steps": 1725, + "num_train_epochs": 3, + "save_steps": 200, + "total_flos": 3.348909438051287e+18, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1400/training_args.bin b/checkpoint-1400/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..14a45b06b3e8ec543e02c64e959e660395a27125 --- /dev/null +++ b/checkpoint-1400/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e5fa4a6be80dea6c8018bb49732568d9b22103b1e3942cf21f3b067cbdff444 +size 4027 diff --git a/checkpoint-1600/README.md b/checkpoint-1600/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5f4f5625039e17bc7326cc10fbbc7896bb06453b --- /dev/null +++ b/checkpoint-1600/README.md @@ -0,0 +1,21 @@ +--- +library_name: peft +--- +## Training procedure + + +The following `bitsandbytes` quantization config was used during training: +- quant_method: bitsandbytes +- load_in_8bit: True +- load_in_4bit: False +- llm_int8_threshold: 6.0 +- llm_int8_skip_modules: None +- llm_int8_enable_fp32_cpu_offload: False +- llm_int8_has_fp16_weight: False +- bnb_4bit_quant_type: fp4 +- bnb_4bit_use_double_quant: False +- bnb_4bit_compute_dtype: float32 +### Framework versions + + +- PEFT 0.6.0.dev0 diff --git a/checkpoint-1600/adapter_config.json b/checkpoint-1600/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6fddf8fb6febbb9b217e5849180ec2fc8aab3c0e --- /dev/null +++ b/checkpoint-1600/adapter_config.json @@ -0,0 +1,21 @@ +{ + "auto_mapping": null, + "base_model_name_or_path": "../chinese-llama-2-13b", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layers_pattern": null, + "layers_to_transform": null, + "lora_alpha": 8, + "lora_dropout": 0.05, + "modules_to_save": null, + "peft_type": "LORA", + "r": 8, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj" + ], + "task_type": "CAUSAL_LM" +} \ No newline at end of file diff --git a/checkpoint-1600/adapter_model.bin b/checkpoint-1600/adapter_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..8710093665dfdb9819e2f8817a1e25a4ccdd9935 --- /dev/null +++ b/checkpoint-1600/adapter_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5e1621f48d9ad8feb1d6d31050275f0aafd080c5c07153301fe2f48411f4406 +size 443 diff --git a/checkpoint-1600/optimizer.pt b/checkpoint-1600/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..790aa9c39053f0f0a6292baac751d83ec5c1ec6d --- /dev/null +++ b/checkpoint-1600/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:063a994a29578fb2287de3c40d6739c12e280d2c013c64b504c30c49b921d841 +size 52562757 diff --git a/checkpoint-1600/rng_state_0.pth b/checkpoint-1600/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..4ecfb579c9c18899c75ddd760aebcc3b1b3fddcb --- /dev/null +++ b/checkpoint-1600/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf98e784e6b8dc883e2b800a4e11a6bf86ba1127f10f4f1235ba37d2fff2ec1d +size 18679 diff --git a/checkpoint-1600/rng_state_1.pth b/checkpoint-1600/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..aa06567ce694f8371320584064b1bce08927f4d5 --- /dev/null +++ b/checkpoint-1600/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de58a6082f6549d7efc38756577f571cb6e854db8c6e315cf12c4d5166beb810 +size 18679 diff --git a/checkpoint-1600/rng_state_2.pth b/checkpoint-1600/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..26fd746dd3e65ba7148d4b83e22d6101e41e26ac --- /dev/null +++ b/checkpoint-1600/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd5ae166dffc6c6455bac816ad340b2c97433f56b532222b69bf2d7b7f170195 +size 18679 diff --git a/checkpoint-1600/rng_state_3.pth b/checkpoint-1600/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..e339b3097e6d8d17469bd7e127b56306c72bdf74 --- /dev/null +++ b/checkpoint-1600/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2567bbf472792e86322a470e9e62452e879e353cbc25534507c3f539691f3a94 +size 18679 diff --git a/checkpoint-1600/rng_state_4.pth b/checkpoint-1600/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..2f6515142110b3fe304816b3308c7eb719b04785 --- /dev/null +++ b/checkpoint-1600/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64dea77acac01ff60d3c4eb7bf7b6bce6a5b64093fcaa66ea3b7475ce7bf5a18 +size 18679 diff --git a/checkpoint-1600/scheduler.pt b/checkpoint-1600/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..237757b719a4fb3007a287c573b9fa4448436c3a --- /dev/null +++ b/checkpoint-1600/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b65eb6f72e0fedc2475341926889dc05fe463519f18fb1a3fcf45b742f89af60 +size 627 diff --git a/checkpoint-1600/trainer_state.json b/checkpoint-1600/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7c4b68f6756e12fb7560f77e416cacea8b612654 --- /dev/null +++ b/checkpoint-1600/trainer_state.json @@ -0,0 +1,563 @@ +{ + "best_metric": 1.2222874164581299, + "best_model_checkpoint": "../llama2-9439-21sept/checkpoint-1600", + "epoch": 2.7785817655571634, + "eval_steps": 200, + "global_step": 1600, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.03, + "learning_rate": 5.9999999999999995e-05, + "loss": 3.4609, + "step": 20 + }, + { + "epoch": 0.07, + "learning_rate": 0.00011999999999999999, + "loss": 2.8346, + "step": 40 + }, + { + "epoch": 0.1, + "learning_rate": 0.00017999999999999998, + "loss": 1.7597, + "step": 60 + }, + { + "epoch": 0.14, + "learning_rate": 0.00023999999999999998, + "loss": 1.542, + "step": 80 + }, + { + "epoch": 0.17, + "learning_rate": 0.0003, + "loss": 1.4567, + "step": 100 + }, + { + "epoch": 0.21, + "learning_rate": 0.0002963076923076923, + "loss": 1.4081, + "step": 120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0002926153846153846, + "loss": 1.3805, + "step": 140 + }, + { + "epoch": 0.28, + "learning_rate": 0.0002889230769230769, + "loss": 1.3605, + "step": 160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00028523076923076923, + "loss": 1.3329, + "step": 180 + }, + { + "epoch": 0.35, + "learning_rate": 0.0002815384615384615, + "loss": 1.3367, + "step": 200 + }, + { + "epoch": 0.35, + "eval_loss": 1.3450770378112793, + "eval_runtime": 6.306, + "eval_samples_per_second": 47.574, + "eval_steps_per_second": 1.269, + "step": 200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0002778461538461538, + "loss": 1.3152, + "step": 220 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002741538461538461, + "loss": 1.312, + "step": 240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027046153846153843, + "loss": 1.2883, + "step": 260 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026676923076923074, + "loss": 1.2843, + "step": 280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00026307692307692306, + "loss": 1.2609, + "step": 300 + }, + { + "epoch": 0.56, + "learning_rate": 0.00025938461538461537, + "loss": 1.2707, + "step": 320 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002556923076923077, + "loss": 1.2624, + "step": 340 + }, + { + "epoch": 0.63, + "learning_rate": 0.00025199999999999995, + "loss": 1.273, + "step": 360 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002483076923076923, + "loss": 1.251, + "step": 380 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002446153846153846, + "loss": 1.2648, + "step": 400 + }, + { + "epoch": 0.69, + "eval_loss": 1.292348861694336, + "eval_runtime": 6.304, + "eval_samples_per_second": 47.589, + "eval_steps_per_second": 1.269, + "step": 400 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002409230769230769, + "loss": 1.265, + "step": 420 + }, + { + "epoch": 0.76, + "learning_rate": 0.0002372307692307692, + "loss": 1.2516, + "step": 440 + }, + { + "epoch": 0.8, + "learning_rate": 0.00023353846153846151, + "loss": 1.2441, + "step": 460 + }, + { + "epoch": 0.83, + "learning_rate": 0.00022984615384615383, + "loss": 1.2204, + "step": 480 + }, + { + "epoch": 0.87, + "learning_rate": 0.00022615384615384614, + "loss": 1.2221, + "step": 500 + }, + { + "epoch": 0.9, + "learning_rate": 0.00022246153846153846, + "loss": 1.2246, + "step": 520 + }, + { + "epoch": 0.94, + "learning_rate": 0.00021876923076923074, + "loss": 1.2227, + "step": 540 + }, + { + "epoch": 0.97, + "learning_rate": 0.00021507692307692306, + "loss": 1.2124, + "step": 560 + }, + { + "epoch": 1.01, + "learning_rate": 0.00021138461538461537, + "loss": 1.2065, + "step": 580 + }, + { + "epoch": 1.04, + "learning_rate": 0.00020769230769230766, + "loss": 1.2106, + "step": 600 + }, + { + "epoch": 1.04, + "eval_loss": 1.2669302225112915, + "eval_runtime": 6.3128, + "eval_samples_per_second": 47.523, + "eval_steps_per_second": 1.267, + "step": 600 + }, + { + "epoch": 1.08, + "learning_rate": 0.000204, + "loss": 1.2046, + "step": 620 + }, + { + "epoch": 1.11, + "learning_rate": 0.00020030769230769229, + "loss": 1.2114, + "step": 640 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001966153846153846, + "loss": 1.2122, + "step": 660 + }, + { + "epoch": 1.18, + "learning_rate": 0.00019310769230769227, + "loss": 1.1963, + "step": 680 + }, + { + "epoch": 1.22, + "learning_rate": 0.00018941538461538461, + "loss": 1.1965, + "step": 700 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001857230769230769, + "loss": 1.1937, + "step": 720 + }, + { + "epoch": 1.29, + "learning_rate": 0.00018203076923076921, + "loss": 1.186, + "step": 740 + }, + { + "epoch": 1.32, + "learning_rate": 0.00017833846153846153, + "loss": 1.1732, + "step": 760 + }, + { + "epoch": 1.35, + "learning_rate": 0.00017464615384615381, + "loss": 1.1911, + "step": 780 + }, + { + "epoch": 1.39, + "learning_rate": 0.00017095384615384616, + "loss": 1.2153, + "step": 800 + }, + { + "epoch": 1.39, + "eval_loss": 1.2513903379440308, + "eval_runtime": 6.3069, + "eval_samples_per_second": 47.567, + "eval_steps_per_second": 1.268, + "step": 800 + }, + { + "epoch": 1.42, + "learning_rate": 0.00016726153846153844, + "loss": 1.1991, + "step": 820 + }, + { + "epoch": 1.46, + "learning_rate": 0.00016356923076923073, + "loss": 1.1934, + "step": 840 + }, + { + "epoch": 1.49, + "learning_rate": 0.00015987692307692307, + "loss": 1.2047, + "step": 860 + }, + { + "epoch": 1.53, + "learning_rate": 0.00015618461538461536, + "loss": 1.2204, + "step": 880 + }, + { + "epoch": 1.56, + "learning_rate": 0.0001524923076923077, + "loss": 1.1785, + "step": 900 + }, + { + "epoch": 1.6, + "learning_rate": 0.00014879999999999998, + "loss": 1.1756, + "step": 920 + }, + { + "epoch": 1.63, + "learning_rate": 0.0001451076923076923, + "loss": 1.214, + "step": 940 + }, + { + "epoch": 1.67, + "learning_rate": 0.0001414153846153846, + "loss": 1.1794, + "step": 960 + }, + { + "epoch": 1.7, + "learning_rate": 0.00013772307692307693, + "loss": 1.1946, + "step": 980 + }, + { + "epoch": 1.74, + "learning_rate": 0.0001340307692307692, + "loss": 1.175, + "step": 1000 + }, + { + "epoch": 1.74, + "eval_loss": 1.2406948804855347, + "eval_runtime": 6.3315, + "eval_samples_per_second": 47.382, + "eval_steps_per_second": 1.264, + "step": 1000 + }, + { + "epoch": 1.77, + "learning_rate": 0.00013033846153846153, + "loss": 1.178, + "step": 1020 + }, + { + "epoch": 1.81, + "learning_rate": 0.00012664615384615384, + "loss": 1.1859, + "step": 1040 + }, + { + "epoch": 1.84, + "learning_rate": 0.00012295384615384615, + "loss": 1.1801, + "step": 1060 + }, + { + "epoch": 1.88, + "learning_rate": 0.00011926153846153845, + "loss": 1.1743, + "step": 1080 + }, + { + "epoch": 1.91, + "learning_rate": 0.00011556923076923076, + "loss": 1.1596, + "step": 1100 + }, + { + "epoch": 1.95, + "learning_rate": 0.00011187692307692307, + "loss": 1.1495, + "step": 1120 + }, + { + "epoch": 1.98, + "learning_rate": 0.00010818461538461537, + "loss": 1.1808, + "step": 1140 + }, + { + "epoch": 2.01, + "learning_rate": 0.00010449230769230768, + "loss": 1.1743, + "step": 1160 + }, + { + "epoch": 2.05, + "learning_rate": 0.0001008, + "loss": 1.1354, + "step": 1180 + }, + { + "epoch": 2.08, + "learning_rate": 9.71076923076923e-05, + "loss": 1.1712, + "step": 1200 + }, + { + "epoch": 2.08, + "eval_loss": 1.2316911220550537, + "eval_runtime": 6.3191, + "eval_samples_per_second": 47.475, + "eval_steps_per_second": 1.266, + "step": 1200 + }, + { + "epoch": 2.12, + "learning_rate": 9.34153846153846e-05, + "loss": 1.1709, + "step": 1220 + }, + { + "epoch": 2.15, + "learning_rate": 8.972307692307691e-05, + "loss": 1.1667, + "step": 1240 + }, + { + "epoch": 2.19, + "learning_rate": 8.603076923076923e-05, + "loss": 1.1781, + "step": 1260 + }, + { + "epoch": 2.22, + "learning_rate": 8.233846153846154e-05, + "loss": 1.1628, + "step": 1280 + }, + { + "epoch": 2.26, + "learning_rate": 7.864615384615383e-05, + "loss": 1.1723, + "step": 1300 + }, + { + "epoch": 2.29, + "learning_rate": 7.495384615384615e-05, + "loss": 1.1578, + "step": 1320 + }, + { + "epoch": 2.33, + "learning_rate": 7.126153846153845e-05, + "loss": 1.1399, + "step": 1340 + }, + { + "epoch": 2.36, + "learning_rate": 6.756923076923077e-05, + "loss": 1.1612, + "step": 1360 + }, + { + "epoch": 2.4, + "learning_rate": 6.387692307692307e-05, + "loss": 1.1582, + "step": 1380 + }, + { + "epoch": 2.43, + "learning_rate": 6.0184615384615375e-05, + "loss": 1.1534, + "step": 1400 + }, + { + "epoch": 2.43, + "eval_loss": 1.2265406847000122, + "eval_runtime": 6.3176, + "eval_samples_per_second": 47.486, + "eval_steps_per_second": 1.266, + "step": 1400 + }, + { + "epoch": 2.47, + "learning_rate": 5.649230769230769e-05, + "loss": 1.1626, + "step": 1420 + }, + { + "epoch": 2.5, + "learning_rate": 5.279999999999999e-05, + "loss": 1.1619, + "step": 1440 + }, + { + "epoch": 2.54, + "learning_rate": 4.91076923076923e-05, + "loss": 1.168, + "step": 1460 + }, + { + "epoch": 2.57, + "learning_rate": 4.541538461538462e-05, + "loss": 1.1475, + "step": 1480 + }, + { + "epoch": 2.6, + "learning_rate": 4.172307692307692e-05, + "loss": 1.1863, + "step": 1500 + }, + { + "epoch": 2.64, + "learning_rate": 3.803076923076923e-05, + "loss": 1.1452, + "step": 1520 + }, + { + "epoch": 2.67, + "learning_rate": 3.433846153846154e-05, + "loss": 1.1555, + "step": 1540 + }, + { + "epoch": 2.71, + "learning_rate": 3.0646153846153845e-05, + "loss": 1.1113, + "step": 1560 + }, + { + "epoch": 2.74, + "learning_rate": 2.695384615384615e-05, + "loss": 1.1429, + "step": 1580 + }, + { + "epoch": 2.78, + "learning_rate": 2.326153846153846e-05, + "loss": 1.1609, + "step": 1600 + }, + { + "epoch": 2.78, + "eval_loss": 1.2222874164581299, + "eval_runtime": 6.3153, + "eval_samples_per_second": 47.504, + "eval_steps_per_second": 1.267, + "step": 1600 + } + ], + "logging_steps": 20, + "max_steps": 1725, + "num_train_epochs": 3, + "save_steps": 200, + "total_flos": 3.827325072058614e+18, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1600/training_args.bin b/checkpoint-1600/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..14a45b06b3e8ec543e02c64e959e660395a27125 --- /dev/null +++ b/checkpoint-1600/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e5fa4a6be80dea6c8018bb49732568d9b22103b1e3942cf21f3b067cbdff444 +size 4027 diff --git a/checkpoint-200/README.md b/checkpoint-200/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5f4f5625039e17bc7326cc10fbbc7896bb06453b --- /dev/null +++ b/checkpoint-200/README.md @@ -0,0 +1,21 @@ +--- +library_name: peft +--- +## Training procedure + + +The following `bitsandbytes` quantization config was used during training: +- quant_method: bitsandbytes +- load_in_8bit: True +- load_in_4bit: False +- llm_int8_threshold: 6.0 +- llm_int8_skip_modules: None +- llm_int8_enable_fp32_cpu_offload: False +- llm_int8_has_fp16_weight: False +- bnb_4bit_quant_type: fp4 +- bnb_4bit_use_double_quant: False +- bnb_4bit_compute_dtype: float32 +### Framework versions + + +- PEFT 0.6.0.dev0 diff --git a/checkpoint-200/adapter_config.json b/checkpoint-200/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6fddf8fb6febbb9b217e5849180ec2fc8aab3c0e --- /dev/null +++ b/checkpoint-200/adapter_config.json @@ -0,0 +1,21 @@ +{ + "auto_mapping": null, + "base_model_name_or_path": "../chinese-llama-2-13b", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layers_pattern": null, + "layers_to_transform": null, + "lora_alpha": 8, + "lora_dropout": 0.05, + "modules_to_save": null, + "peft_type": "LORA", + "r": 8, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj" + ], + "task_type": "CAUSAL_LM" +} \ No newline at end of file diff --git a/checkpoint-200/adapter_model.bin b/checkpoint-200/adapter_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..8710093665dfdb9819e2f8817a1e25a4ccdd9935 --- /dev/null +++ b/checkpoint-200/adapter_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5e1621f48d9ad8feb1d6d31050275f0aafd080c5c07153301fe2f48411f4406 +size 443 diff --git a/checkpoint-200/optimizer.pt b/checkpoint-200/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..f4b1344027a1699514c2a2ebaaba6901db306f1a --- /dev/null +++ b/checkpoint-200/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9048ceb424f75cf82a0cca4065bdedbe24913cc4d2cca9d7438c22b3cdd33db7 +size 52562757 diff --git a/checkpoint-200/rng_state_0.pth b/checkpoint-200/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..25fe993c3974725f7d8d6d50de238e15c2639aa8 --- /dev/null +++ b/checkpoint-200/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c484e25fb495d8d891e84989a975e7b57ff72b7831d5d998387b0b709e7c1c4 +size 18679 diff --git a/checkpoint-200/rng_state_1.pth b/checkpoint-200/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..6d86e1a446ad6c685e37701263034c2a7223aa37 --- /dev/null +++ b/checkpoint-200/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:525f1f5c716339809ca5aa2104b34c1f89dd5812028d65e585943f5c7560780c +size 18679 diff --git a/checkpoint-200/rng_state_2.pth b/checkpoint-200/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..78f1bcd87fbdad867ce274c1d444ab209190490b --- /dev/null +++ b/checkpoint-200/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75be087fbb4eb9e1235ffe7e2800ec89a9c78570a749b4743d7fd2fa23bd220f +size 18679 diff --git a/checkpoint-200/rng_state_3.pth b/checkpoint-200/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..498fe3243d67698bc8a8a42581abf518559d960b --- /dev/null +++ b/checkpoint-200/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:207b9bc11d952068e35a4dc46f635432d1adeb20d51a89a86c6746098ff99ae7 +size 18679 diff --git a/checkpoint-200/rng_state_4.pth b/checkpoint-200/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..76e3995aab9805142839d3b5d68a5651a0129440 --- /dev/null +++ b/checkpoint-200/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eeca9fef300ecee1811083dd5e06f1c1a161d1269bf822f9e2327e1c88501f31 +size 18679 diff --git a/checkpoint-200/scheduler.pt b/checkpoint-200/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..43e1f2517ff21fed430574d7fec683d65694ea01 --- /dev/null +++ b/checkpoint-200/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:094fa80bf1b24e273e14df73940cdd323603b6f804fb90789a0674f65e04d856 +size 627 diff --git a/checkpoint-200/trainer_state.json b/checkpoint-200/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bed6a2434910ad73fbcedfd1ab17d971723726ad --- /dev/null +++ b/checkpoint-200/trainer_state.json @@ -0,0 +1,87 @@ +{ + "best_metric": 1.3450770378112793, + "best_model_checkpoint": "../llama2-9439-21sept/checkpoint-200", + "epoch": 0.3473227206946454, + "eval_steps": 200, + "global_step": 200, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.03, + "learning_rate": 5.9999999999999995e-05, + "loss": 3.4609, + "step": 20 + }, + { + "epoch": 0.07, + "learning_rate": 0.00011999999999999999, + "loss": 2.8346, + "step": 40 + }, + { + "epoch": 0.1, + "learning_rate": 0.00017999999999999998, + "loss": 1.7597, + "step": 60 + }, + { + "epoch": 0.14, + "learning_rate": 0.00023999999999999998, + "loss": 1.542, + "step": 80 + }, + { + "epoch": 0.17, + "learning_rate": 0.0003, + "loss": 1.4567, + "step": 100 + }, + { + "epoch": 0.21, + "learning_rate": 0.0002963076923076923, + "loss": 1.4081, + "step": 120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0002926153846153846, + "loss": 1.3805, + "step": 140 + }, + { + "epoch": 0.28, + "learning_rate": 0.0002889230769230769, + "loss": 1.3605, + "step": 160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00028523076923076923, + "loss": 1.3329, + "step": 180 + }, + { + "epoch": 0.35, + "learning_rate": 0.0002815384615384615, + "loss": 1.3367, + "step": 200 + }, + { + "epoch": 0.35, + "eval_loss": 1.3450770378112793, + "eval_runtime": 6.306, + "eval_samples_per_second": 47.574, + "eval_steps_per_second": 1.269, + "step": 200 + } + ], + "logging_steps": 20, + "max_steps": 1725, + "num_train_epochs": 3, + "save_steps": 200, + "total_flos": 4.784156340073267e+17, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-200/training_args.bin b/checkpoint-200/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..14a45b06b3e8ec543e02c64e959e660395a27125 --- /dev/null +++ b/checkpoint-200/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e5fa4a6be80dea6c8018bb49732568d9b22103b1e3942cf21f3b067cbdff444 +size 4027 diff --git a/checkpoint-400/README.md b/checkpoint-400/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5f4f5625039e17bc7326cc10fbbc7896bb06453b --- /dev/null +++ b/checkpoint-400/README.md @@ -0,0 +1,21 @@ +--- +library_name: peft +--- +## Training procedure + + +The following `bitsandbytes` quantization config was used during training: +- quant_method: bitsandbytes +- load_in_8bit: True +- load_in_4bit: False +- llm_int8_threshold: 6.0 +- llm_int8_skip_modules: None +- llm_int8_enable_fp32_cpu_offload: False +- llm_int8_has_fp16_weight: False +- bnb_4bit_quant_type: fp4 +- bnb_4bit_use_double_quant: False +- bnb_4bit_compute_dtype: float32 +### Framework versions + + +- PEFT 0.6.0.dev0 diff --git a/checkpoint-400/adapter_config.json b/checkpoint-400/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6fddf8fb6febbb9b217e5849180ec2fc8aab3c0e --- /dev/null +++ b/checkpoint-400/adapter_config.json @@ -0,0 +1,21 @@ +{ + "auto_mapping": null, + "base_model_name_or_path": "../chinese-llama-2-13b", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layers_pattern": null, + "layers_to_transform": null, + "lora_alpha": 8, + "lora_dropout": 0.05, + "modules_to_save": null, + "peft_type": "LORA", + "r": 8, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj" + ], + "task_type": "CAUSAL_LM" +} \ No newline at end of file diff --git a/checkpoint-400/adapter_model.bin b/checkpoint-400/adapter_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..8710093665dfdb9819e2f8817a1e25a4ccdd9935 --- /dev/null +++ b/checkpoint-400/adapter_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5e1621f48d9ad8feb1d6d31050275f0aafd080c5c07153301fe2f48411f4406 +size 443 diff --git a/checkpoint-400/optimizer.pt b/checkpoint-400/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ce64c7e3c38f8af954bb4cf657b346aaa2b1c0d --- /dev/null +++ b/checkpoint-400/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1087cf116bff8b018ad79ef6a645a8d75ecbaea363be8ba18e76557259fe1190 +size 52562757 diff --git a/checkpoint-400/rng_state_0.pth b/checkpoint-400/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..4cf0928969d03a2f8d09b2374437973027a688bb --- /dev/null +++ b/checkpoint-400/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d8932e53b2965dea55d3e0bc12bbd7bae98a67c1aedaa640b18ac92bd090791 +size 18679 diff --git a/checkpoint-400/rng_state_1.pth b/checkpoint-400/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..d196dcfc468e8b23c1fa72450389f068ae4b5741 --- /dev/null +++ b/checkpoint-400/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1691b50a15cedae04437854745dd315183306f4648de8fbc892dfa0ea0d72a09 +size 18679 diff --git a/checkpoint-400/rng_state_2.pth b/checkpoint-400/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..971a73d44461d761402ca13ea001aa82ec4a7058 --- /dev/null +++ b/checkpoint-400/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b42314d590b087051172c2e5fd827fcfd7dbd52b889242ee7a559d8e03439d5 +size 18679 diff --git a/checkpoint-400/rng_state_3.pth b/checkpoint-400/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..e6436cc46c92be1e0c73a1ae5fc4eace8d645b63 --- /dev/null +++ b/checkpoint-400/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f694389117122efbd4bc25c0900947194703ddb5394905c33b64b4a69bf9c2d7 +size 18679 diff --git a/checkpoint-400/rng_state_4.pth b/checkpoint-400/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..06cc7209c1f601cffec4998110ac1db069dd084b --- /dev/null +++ b/checkpoint-400/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f51ff33d6aa834a33ceae1131e45238ac0800eeada04483818b8aaaf914eb2aa +size 18679 diff --git a/checkpoint-400/scheduler.pt b/checkpoint-400/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..c939ddf87db8d122d18d9ef7572c14c6f195452f --- /dev/null +++ b/checkpoint-400/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad3f9c7228f51da9bfe9897101411228dc45c5b4aa01a0f499f6d53489758375 +size 627 diff --git a/checkpoint-400/trainer_state.json b/checkpoint-400/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b3fa45400e0772b8fb164e2e82f6d7395e6776b0 --- /dev/null +++ b/checkpoint-400/trainer_state.json @@ -0,0 +1,155 @@ +{ + "best_metric": 1.292348861694336, + "best_model_checkpoint": "../llama2-9439-21sept/checkpoint-400", + "epoch": 0.6946454413892909, + "eval_steps": 200, + "global_step": 400, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.03, + "learning_rate": 5.9999999999999995e-05, + "loss": 3.4609, + "step": 20 + }, + { + "epoch": 0.07, + "learning_rate": 0.00011999999999999999, + "loss": 2.8346, + "step": 40 + }, + { + "epoch": 0.1, + "learning_rate": 0.00017999999999999998, + "loss": 1.7597, + "step": 60 + }, + { + "epoch": 0.14, + "learning_rate": 0.00023999999999999998, + "loss": 1.542, + "step": 80 + }, + { + "epoch": 0.17, + "learning_rate": 0.0003, + "loss": 1.4567, + "step": 100 + }, + { + "epoch": 0.21, + "learning_rate": 0.0002963076923076923, + "loss": 1.4081, + "step": 120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0002926153846153846, + "loss": 1.3805, + "step": 140 + }, + { + "epoch": 0.28, + "learning_rate": 0.0002889230769230769, + "loss": 1.3605, + "step": 160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00028523076923076923, + "loss": 1.3329, + "step": 180 + }, + { + "epoch": 0.35, + "learning_rate": 0.0002815384615384615, + "loss": 1.3367, + "step": 200 + }, + { + "epoch": 0.35, + "eval_loss": 1.3450770378112793, + "eval_runtime": 6.306, + "eval_samples_per_second": 47.574, + "eval_steps_per_second": 1.269, + "step": 200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0002778461538461538, + "loss": 1.3152, + "step": 220 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002741538461538461, + "loss": 1.312, + "step": 240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027046153846153843, + "loss": 1.2883, + "step": 260 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026676923076923074, + "loss": 1.2843, + "step": 280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00026307692307692306, + "loss": 1.2609, + "step": 300 + }, + { + "epoch": 0.56, + "learning_rate": 0.00025938461538461537, + "loss": 1.2707, + "step": 320 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002556923076923077, + "loss": 1.2624, + "step": 340 + }, + { + "epoch": 0.63, + "learning_rate": 0.00025199999999999995, + "loss": 1.273, + "step": 360 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002483076923076923, + "loss": 1.251, + "step": 380 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002446153846153846, + "loss": 1.2648, + "step": 400 + }, + { + "epoch": 0.69, + "eval_loss": 1.292348861694336, + "eval_runtime": 6.304, + "eval_samples_per_second": 47.589, + "eval_steps_per_second": 1.269, + "step": 400 + } + ], + "logging_steps": 20, + "max_steps": 1725, + "num_train_epochs": 3, + "save_steps": 200, + "total_flos": 9.568312680146534e+17, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-400/training_args.bin b/checkpoint-400/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..14a45b06b3e8ec543e02c64e959e660395a27125 --- /dev/null +++ b/checkpoint-400/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e5fa4a6be80dea6c8018bb49732568d9b22103b1e3942cf21f3b067cbdff444 +size 4027 diff --git a/checkpoint-600/README.md b/checkpoint-600/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5f4f5625039e17bc7326cc10fbbc7896bb06453b --- /dev/null +++ b/checkpoint-600/README.md @@ -0,0 +1,21 @@ +--- +library_name: peft +--- +## Training procedure + + +The following `bitsandbytes` quantization config was used during training: +- quant_method: bitsandbytes +- load_in_8bit: True +- load_in_4bit: False +- llm_int8_threshold: 6.0 +- llm_int8_skip_modules: None +- llm_int8_enable_fp32_cpu_offload: False +- llm_int8_has_fp16_weight: False +- bnb_4bit_quant_type: fp4 +- bnb_4bit_use_double_quant: False +- bnb_4bit_compute_dtype: float32 +### Framework versions + + +- PEFT 0.6.0.dev0 diff --git a/checkpoint-600/adapter_config.json b/checkpoint-600/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6fddf8fb6febbb9b217e5849180ec2fc8aab3c0e --- /dev/null +++ b/checkpoint-600/adapter_config.json @@ -0,0 +1,21 @@ +{ + "auto_mapping": null, + "base_model_name_or_path": "../chinese-llama-2-13b", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layers_pattern": null, + "layers_to_transform": null, + "lora_alpha": 8, + "lora_dropout": 0.05, + "modules_to_save": null, + "peft_type": "LORA", + "r": 8, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj" + ], + "task_type": "CAUSAL_LM" +} \ No newline at end of file diff --git a/checkpoint-600/adapter_model.bin b/checkpoint-600/adapter_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..8710093665dfdb9819e2f8817a1e25a4ccdd9935 --- /dev/null +++ b/checkpoint-600/adapter_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5e1621f48d9ad8feb1d6d31050275f0aafd080c5c07153301fe2f48411f4406 +size 443 diff --git a/checkpoint-600/optimizer.pt b/checkpoint-600/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..484db44b26b2ae20e3252bfa5a826af47c6a53ab --- /dev/null +++ b/checkpoint-600/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:339169a2585a6676c9e38c79555f5a21dddd5c0724f4acb77e71249549153181 +size 52562757 diff --git a/checkpoint-600/rng_state_0.pth b/checkpoint-600/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..c2e6c72f204a5b7c98b658f79ee0a03a8f72f1a9 --- /dev/null +++ b/checkpoint-600/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c73594f7e6202837eb0f41ece0c6b736946f660243ffd9b45d9a25e4962b159b +size 18679 diff --git a/checkpoint-600/rng_state_1.pth b/checkpoint-600/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..0eabb3bf8a16216752c2b3adca7d2547ac92ed6f --- /dev/null +++ b/checkpoint-600/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8e35072e7cd2131548921501749d2eff6b0374edd23a6d439fbf58a3e6fd5a2 +size 18679 diff --git a/checkpoint-600/rng_state_2.pth b/checkpoint-600/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..424df528e5c6941da5daafbcee8a5841001c6ed5 --- /dev/null +++ b/checkpoint-600/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe47cdcf72580764c764034de798705e7eee8e5a1234a9054490c29a2eb9d077 +size 18679 diff --git a/checkpoint-600/rng_state_3.pth b/checkpoint-600/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..c6c735205b4dc174971b3823d795a08961c574ee --- /dev/null +++ b/checkpoint-600/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd5c6168af2a6cc331c041720fe79492847848fe461bee0ba68ff70de0604659 +size 18679 diff --git a/checkpoint-600/rng_state_4.pth b/checkpoint-600/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..d071650e4e68fc2282a7e7da8f12d517371e2b52 --- /dev/null +++ b/checkpoint-600/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5ef82ff667df23008ad0c632582ec69d90062c27a0be052656898dd0baf23f3 +size 18679 diff --git a/checkpoint-600/scheduler.pt b/checkpoint-600/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..0bf9ac1767e452c37569e46d5478a6f7fb607035 --- /dev/null +++ b/checkpoint-600/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7fade71b9f5122299938bd8bf083e2025c89d76539737287c3eb021e5ed2824 +size 627 diff --git a/checkpoint-600/trainer_state.json b/checkpoint-600/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b152bbce33058cbdfbf323c66462aa5cec135502 --- /dev/null +++ b/checkpoint-600/trainer_state.json @@ -0,0 +1,223 @@ +{ + "best_metric": 1.2669302225112915, + "best_model_checkpoint": "../llama2-9439-21sept/checkpoint-600", + "epoch": 1.0419681620839363, + "eval_steps": 200, + "global_step": 600, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.03, + "learning_rate": 5.9999999999999995e-05, + "loss": 3.4609, + "step": 20 + }, + { + "epoch": 0.07, + "learning_rate": 0.00011999999999999999, + "loss": 2.8346, + "step": 40 + }, + { + "epoch": 0.1, + "learning_rate": 0.00017999999999999998, + "loss": 1.7597, + "step": 60 + }, + { + "epoch": 0.14, + "learning_rate": 0.00023999999999999998, + "loss": 1.542, + "step": 80 + }, + { + "epoch": 0.17, + "learning_rate": 0.0003, + "loss": 1.4567, + "step": 100 + }, + { + "epoch": 0.21, + "learning_rate": 0.0002963076923076923, + "loss": 1.4081, + "step": 120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0002926153846153846, + "loss": 1.3805, + "step": 140 + }, + { + "epoch": 0.28, + "learning_rate": 0.0002889230769230769, + "loss": 1.3605, + "step": 160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00028523076923076923, + "loss": 1.3329, + "step": 180 + }, + { + "epoch": 0.35, + "learning_rate": 0.0002815384615384615, + "loss": 1.3367, + "step": 200 + }, + { + "epoch": 0.35, + "eval_loss": 1.3450770378112793, + "eval_runtime": 6.306, + "eval_samples_per_second": 47.574, + "eval_steps_per_second": 1.269, + "step": 200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0002778461538461538, + "loss": 1.3152, + "step": 220 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002741538461538461, + "loss": 1.312, + "step": 240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027046153846153843, + "loss": 1.2883, + "step": 260 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026676923076923074, + "loss": 1.2843, + "step": 280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00026307692307692306, + "loss": 1.2609, + "step": 300 + }, + { + "epoch": 0.56, + "learning_rate": 0.00025938461538461537, + "loss": 1.2707, + "step": 320 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002556923076923077, + "loss": 1.2624, + "step": 340 + }, + { + "epoch": 0.63, + "learning_rate": 0.00025199999999999995, + "loss": 1.273, + "step": 360 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002483076923076923, + "loss": 1.251, + "step": 380 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002446153846153846, + "loss": 1.2648, + "step": 400 + }, + { + "epoch": 0.69, + "eval_loss": 1.292348861694336, + "eval_runtime": 6.304, + "eval_samples_per_second": 47.589, + "eval_steps_per_second": 1.269, + "step": 400 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002409230769230769, + "loss": 1.265, + "step": 420 + }, + { + "epoch": 0.76, + "learning_rate": 0.0002372307692307692, + "loss": 1.2516, + "step": 440 + }, + { + "epoch": 0.8, + "learning_rate": 0.00023353846153846151, + "loss": 1.2441, + "step": 460 + }, + { + "epoch": 0.83, + "learning_rate": 0.00022984615384615383, + "loss": 1.2204, + "step": 480 + }, + { + "epoch": 0.87, + "learning_rate": 0.00022615384615384614, + "loss": 1.2221, + "step": 500 + }, + { + "epoch": 0.9, + "learning_rate": 0.00022246153846153846, + "loss": 1.2246, + "step": 520 + }, + { + "epoch": 0.94, + "learning_rate": 0.00021876923076923074, + "loss": 1.2227, + "step": 540 + }, + { + "epoch": 0.97, + "learning_rate": 0.00021507692307692306, + "loss": 1.2124, + "step": 560 + }, + { + "epoch": 1.01, + "learning_rate": 0.00021138461538461537, + "loss": 1.2065, + "step": 580 + }, + { + "epoch": 1.04, + "learning_rate": 0.00020769230769230766, + "loss": 1.2106, + "step": 600 + }, + { + "epoch": 1.04, + "eval_loss": 1.2669302225112915, + "eval_runtime": 6.3128, + "eval_samples_per_second": 47.523, + "eval_steps_per_second": 1.267, + "step": 600 + } + ], + "logging_steps": 20, + "max_steps": 1725, + "num_train_epochs": 3, + "save_steps": 200, + "total_flos": 1.4352469020219802e+18, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-600/training_args.bin b/checkpoint-600/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..14a45b06b3e8ec543e02c64e959e660395a27125 --- /dev/null +++ b/checkpoint-600/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e5fa4a6be80dea6c8018bb49732568d9b22103b1e3942cf21f3b067cbdff444 +size 4027 diff --git a/checkpoint-800/README.md b/checkpoint-800/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5f4f5625039e17bc7326cc10fbbc7896bb06453b --- /dev/null +++ b/checkpoint-800/README.md @@ -0,0 +1,21 @@ +--- +library_name: peft +--- +## Training procedure + + +The following `bitsandbytes` quantization config was used during training: +- quant_method: bitsandbytes +- load_in_8bit: True +- load_in_4bit: False +- llm_int8_threshold: 6.0 +- llm_int8_skip_modules: None +- llm_int8_enable_fp32_cpu_offload: False +- llm_int8_has_fp16_weight: False +- bnb_4bit_quant_type: fp4 +- bnb_4bit_use_double_quant: False +- bnb_4bit_compute_dtype: float32 +### Framework versions + + +- PEFT 0.6.0.dev0 diff --git a/checkpoint-800/adapter_config.json b/checkpoint-800/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6fddf8fb6febbb9b217e5849180ec2fc8aab3c0e --- /dev/null +++ b/checkpoint-800/adapter_config.json @@ -0,0 +1,21 @@ +{ + "auto_mapping": null, + "base_model_name_or_path": "../chinese-llama-2-13b", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layers_pattern": null, + "layers_to_transform": null, + "lora_alpha": 8, + "lora_dropout": 0.05, + "modules_to_save": null, + "peft_type": "LORA", + "r": 8, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj" + ], + "task_type": "CAUSAL_LM" +} \ No newline at end of file diff --git a/checkpoint-800/adapter_model.bin b/checkpoint-800/adapter_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..8710093665dfdb9819e2f8817a1e25a4ccdd9935 --- /dev/null +++ b/checkpoint-800/adapter_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5e1621f48d9ad8feb1d6d31050275f0aafd080c5c07153301fe2f48411f4406 +size 443 diff --git a/checkpoint-800/optimizer.pt b/checkpoint-800/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..a85deff0a7d45fd33c96acb8e06177eedaa1048c --- /dev/null +++ b/checkpoint-800/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fe37b59cc070bd1fd4ad267f92c619227aa0b57c87d55c59b367590f89567a4 +size 52562757 diff --git a/checkpoint-800/rng_state_0.pth b/checkpoint-800/rng_state_0.pth new file mode 100644 index 0000000000000000000000000000000000000000..503ea06e308eff773a5997682a539b36de82959a --- /dev/null +++ b/checkpoint-800/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9847d3cc681babaa7a9081bc0ce35de75e2e393c1c3d55b7122cdf01342ba91d +size 18679 diff --git a/checkpoint-800/rng_state_1.pth b/checkpoint-800/rng_state_1.pth new file mode 100644 index 0000000000000000000000000000000000000000..3d8ceafc0a1e93565f4e432f6238762b2229feef --- /dev/null +++ b/checkpoint-800/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7efa763cd904deacd9f3a0a587acb6512182d80cd6bf01def49b018efc6f6d2f +size 18679 diff --git a/checkpoint-800/rng_state_2.pth b/checkpoint-800/rng_state_2.pth new file mode 100644 index 0000000000000000000000000000000000000000..856613994081c9ef42fbac871884374c220ca99f --- /dev/null +++ b/checkpoint-800/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fa3a35bc53a06c620218ce0c9de8e7fa3d97362ee3074fdebc7aff7ec86ba79 +size 18679 diff --git a/checkpoint-800/rng_state_3.pth b/checkpoint-800/rng_state_3.pth new file mode 100644 index 0000000000000000000000000000000000000000..c44bd1d0c47ec4969928e3350f8212166871ceb1 --- /dev/null +++ b/checkpoint-800/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b28836299385f929e8a336e4ab1c416b6c7f21e3c1d8ea558c6d27f0a6c3de6 +size 18679 diff --git a/checkpoint-800/rng_state_4.pth b/checkpoint-800/rng_state_4.pth new file mode 100644 index 0000000000000000000000000000000000000000..97bab8e30c2ac369efc777ea0c8c491c5cec099c --- /dev/null +++ b/checkpoint-800/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccf432a238bddfd6a047c82aaa3f361ae128d48fee2e7b66a99d51b4b6b48950 +size 18679 diff --git a/checkpoint-800/scheduler.pt b/checkpoint-800/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..895fc70db617899f656a4509cbd733ceb242a620 --- /dev/null +++ b/checkpoint-800/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cdbeffcfb86ce2af2fe5ffd938f4bfe7926a14c8e8d98302487126161ee8896 +size 627 diff --git a/checkpoint-800/trainer_state.json b/checkpoint-800/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8d0393589a30b8952cf6e46cd89d49f8be72e206 --- /dev/null +++ b/checkpoint-800/trainer_state.json @@ -0,0 +1,291 @@ +{ + "best_metric": 1.2513903379440308, + "best_model_checkpoint": "../llama2-9439-21sept/checkpoint-800", + "epoch": 1.3892908827785817, + "eval_steps": 200, + "global_step": 800, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.03, + "learning_rate": 5.9999999999999995e-05, + "loss": 3.4609, + "step": 20 + }, + { + "epoch": 0.07, + "learning_rate": 0.00011999999999999999, + "loss": 2.8346, + "step": 40 + }, + { + "epoch": 0.1, + "learning_rate": 0.00017999999999999998, + "loss": 1.7597, + "step": 60 + }, + { + "epoch": 0.14, + "learning_rate": 0.00023999999999999998, + "loss": 1.542, + "step": 80 + }, + { + "epoch": 0.17, + "learning_rate": 0.0003, + "loss": 1.4567, + "step": 100 + }, + { + "epoch": 0.21, + "learning_rate": 0.0002963076923076923, + "loss": 1.4081, + "step": 120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0002926153846153846, + "loss": 1.3805, + "step": 140 + }, + { + "epoch": 0.28, + "learning_rate": 0.0002889230769230769, + "loss": 1.3605, + "step": 160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00028523076923076923, + "loss": 1.3329, + "step": 180 + }, + { + "epoch": 0.35, + "learning_rate": 0.0002815384615384615, + "loss": 1.3367, + "step": 200 + }, + { + "epoch": 0.35, + "eval_loss": 1.3450770378112793, + "eval_runtime": 6.306, + "eval_samples_per_second": 47.574, + "eval_steps_per_second": 1.269, + "step": 200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0002778461538461538, + "loss": 1.3152, + "step": 220 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002741538461538461, + "loss": 1.312, + "step": 240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027046153846153843, + "loss": 1.2883, + "step": 260 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026676923076923074, + "loss": 1.2843, + "step": 280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00026307692307692306, + "loss": 1.2609, + "step": 300 + }, + { + "epoch": 0.56, + "learning_rate": 0.00025938461538461537, + "loss": 1.2707, + "step": 320 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002556923076923077, + "loss": 1.2624, + "step": 340 + }, + { + "epoch": 0.63, + "learning_rate": 0.00025199999999999995, + "loss": 1.273, + "step": 360 + }, + { + "epoch": 0.66, + "learning_rate": 0.0002483076923076923, + "loss": 1.251, + "step": 380 + }, + { + "epoch": 0.69, + "learning_rate": 0.0002446153846153846, + "loss": 1.2648, + "step": 400 + }, + { + "epoch": 0.69, + "eval_loss": 1.292348861694336, + "eval_runtime": 6.304, + "eval_samples_per_second": 47.589, + "eval_steps_per_second": 1.269, + "step": 400 + }, + { + "epoch": 0.73, + "learning_rate": 0.0002409230769230769, + "loss": 1.265, + "step": 420 + }, + { + "epoch": 0.76, + "learning_rate": 0.0002372307692307692, + "loss": 1.2516, + "step": 440 + }, + { + "epoch": 0.8, + "learning_rate": 0.00023353846153846151, + "loss": 1.2441, + "step": 460 + }, + { + "epoch": 0.83, + "learning_rate": 0.00022984615384615383, + "loss": 1.2204, + "step": 480 + }, + { + "epoch": 0.87, + "learning_rate": 0.00022615384615384614, + "loss": 1.2221, + "step": 500 + }, + { + "epoch": 0.9, + "learning_rate": 0.00022246153846153846, + "loss": 1.2246, + "step": 520 + }, + { + "epoch": 0.94, + "learning_rate": 0.00021876923076923074, + "loss": 1.2227, + "step": 540 + }, + { + "epoch": 0.97, + "learning_rate": 0.00021507692307692306, + "loss": 1.2124, + "step": 560 + }, + { + "epoch": 1.01, + "learning_rate": 0.00021138461538461537, + "loss": 1.2065, + "step": 580 + }, + { + "epoch": 1.04, + "learning_rate": 0.00020769230769230766, + "loss": 1.2106, + "step": 600 + }, + { + "epoch": 1.04, + "eval_loss": 1.2669302225112915, + "eval_runtime": 6.3128, + "eval_samples_per_second": 47.523, + "eval_steps_per_second": 1.267, + "step": 600 + }, + { + "epoch": 1.08, + "learning_rate": 0.000204, + "loss": 1.2046, + "step": 620 + }, + { + "epoch": 1.11, + "learning_rate": 0.00020030769230769229, + "loss": 1.2114, + "step": 640 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001966153846153846, + "loss": 1.2122, + "step": 660 + }, + { + "epoch": 1.18, + "learning_rate": 0.00019310769230769227, + "loss": 1.1963, + "step": 680 + }, + { + "epoch": 1.22, + "learning_rate": 0.00018941538461538461, + "loss": 1.1965, + "step": 700 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001857230769230769, + "loss": 1.1937, + "step": 720 + }, + { + "epoch": 1.29, + "learning_rate": 0.00018203076923076921, + "loss": 1.186, + "step": 740 + }, + { + "epoch": 1.32, + "learning_rate": 0.00017833846153846153, + "loss": 1.1732, + "step": 760 + }, + { + "epoch": 1.35, + "learning_rate": 0.00017464615384615381, + "loss": 1.1911, + "step": 780 + }, + { + "epoch": 1.39, + "learning_rate": 0.00017095384615384616, + "loss": 1.2153, + "step": 800 + }, + { + "epoch": 1.39, + "eval_loss": 1.2513903379440308, + "eval_runtime": 6.3069, + "eval_samples_per_second": 47.567, + "eval_steps_per_second": 1.268, + "step": 800 + } + ], + "logging_steps": 20, + "max_steps": 1725, + "num_train_epochs": 3, + "save_steps": 200, + "total_flos": 1.913662536029307e+18, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-800/training_args.bin b/checkpoint-800/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..14a45b06b3e8ec543e02c64e959e660395a27125 --- /dev/null +++ b/checkpoint-800/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e5fa4a6be80dea6c8018bb49732568d9b22103b1e3942cf21f3b067cbdff444 +size 4027