Delete QwenTT-0.5B-INT8

QwenTT-0.5B-INT8/README.md

@@ -1,59 +0,0 @@
----
-license: other
-library_name: peft
-tags:
-- llama-factory
-- lora
-- generated_from_trainer
-base_model: Qwen/Qwen1.5-0.5B-Chat
-model-index:
-- name: QwenTT-0.5B-INT8
-  results: []
----
-
-<!-- This model card has been generated automatically according to the information the Trainer had access to. You
-should probably proofread and complete it, then remove this comment. -->
-
-# QwenTT-0.5B-INT8
-
-This model is a fine-tuned version of [Qwen/Qwen1.5-0.5B-Chat](https://huggingface.co/Qwen/Qwen1.5-0.5B-Chat) on the identity dataset.
-
-## Model description
-
-More information needed
-
-## Intended uses & limitations
-
-More information needed
-
-## Training and evaluation data
-
-More information needed
-
-## Training procedure
-
-### Training hyperparameters
-
-The following hyperparameters were used during training:
-- learning_rate: 5e-05
-- train_batch_size: 2
-- eval_batch_size: 8
-- seed: 42
-- gradient_accumulation_steps: 8
-- total_train_batch_size: 16
-- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
-- lr_scheduler_type: cosine
-- num_epochs: 3.0
-- mixed_precision_training: Native AMP
-
-### Training results
-
-
-
-### Framework versions
-
-- PEFT 0.11.1
-- Transformers 4.40.2
-- Pytorch 2.2.1+cu121
-- Datasets 2.19.1
-- Tokenizers 0.19.1

QwenTT-0.5B-INT8/added_tokens.json

@@ -1,5 +0,0 @@
-{
-  "<|endoftext|>": 151643,
-  "<|im_end|>": 151645,
-  "<|im_start|>": 151644
-}

QwenTT-0.5B-INT8/all_results.json

@@ -1,8 +0,0 @@
-{
-    "epoch": 2.608695652173913,
-    "total_flos": 27871774801920.0,
-    "train_loss": 3.3604891459147135,
-    "train_runtime": 81.3934,
-    "train_samples_per_second": 3.354,
-    "train_steps_per_second": 0.184
-}

QwenTT-0.5B-INT8/config.json

@@ -1,29 +0,0 @@
-{
-  "alpha_pattern": {},
-  "auto_mapping": null,
-  "base_model_name_or_path": "Qwen/Qwen1.5-0.5B-Chat",
-  "bias": "none",
-  "fan_in_fan_out": false,
-  "inference_mode": true,
-  "init_lora_weights": true,
-  "layer_replication": null,
-  "layers_pattern": null,
-  "layers_to_transform": null,
-  "loftq_config": {},
-  "lora_alpha": 16,
-  "lora_dropout": 0,
-  "megatron_config": null,
-  "megatron_core": "megatron.core",
-  "modules_to_save": null,
-  "peft_type": "LORA",
-  "r": 8,
-  "rank_pattern": {},
-  "revision": null,
-  "target_modules": [
-    "v_proj",
-    "q_proj"
-  ],
-  "task_type": "CAUSAL_LM",
-  "use_dora": false,
-  "use_rslora": false
-}

QwenTT-0.5B-INT8/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:bbe5be185b3c4eac11027a7757c117526bec685ef0b8f1b04cd85732532d1060
-size 3158328

QwenTT-0.5B-INT8/running_log.txt

@@ -1,165 +0,0 @@
-05/19/2024 22:55:50 - INFO - transformers.tokenization_utils_base - loading file vocab.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen1.5-0.5B-Chat/snapshots/4d14e384a4b037942bb3f3016665157c8bcb70ea/vocab.json
-
-05/19/2024 22:55:50 - INFO - transformers.tokenization_utils_base - loading file merges.txt from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen1.5-0.5B-Chat/snapshots/4d14e384a4b037942bb3f3016665157c8bcb70ea/merges.txt
-
-05/19/2024 22:55:50 - INFO - transformers.tokenization_utils_base - loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen1.5-0.5B-Chat/snapshots/4d14e384a4b037942bb3f3016665157c8bcb70ea/tokenizer.json
-
-05/19/2024 22:55:50 - INFO - transformers.tokenization_utils_base - loading file added_tokens.json from cache at None
-
-05/19/2024 22:55:50 - INFO - transformers.tokenization_utils_base - loading file special_tokens_map.json from cache at None
-
-05/19/2024 22:55:50 - INFO - transformers.tokenization_utils_base - loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen1.5-0.5B-Chat/snapshots/4d14e384a4b037942bb3f3016665157c8bcb70ea/tokenizer_config.json
-
-05/19/2024 22:55:51 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
-
-05/19/2024 22:55:51 - INFO - llamafactory.data.template - Replace eos token: <|im_end|>
-
-05/19/2024 22:55:51 - INFO - llamafactory.data.loader - Loading dataset identity.json...
-
-05/19/2024 22:55:58 - INFO - transformers.configuration_utils - loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen1.5-0.5B-Chat/snapshots/4d14e384a4b037942bb3f3016665157c8bcb70ea/config.json
-
-05/19/2024 22:55:58 - INFO - transformers.configuration_utils - Model config Qwen2Config {
-  "_name_or_path": "Qwen/Qwen1.5-0.5B-Chat",
-  "architectures": [
-    "Qwen2ForCausalLM"
-  ],
-  "attention_dropout": 0.0,
-  "bos_token_id": 151643,
-  "eos_token_id": 151645,
-  "hidden_act": "silu",
-  "hidden_size": 1024,
-  "initializer_range": 0.02,
-  "intermediate_size": 2816,
-  "max_position_embeddings": 32768,
-  "max_window_layers": 21,
-  "model_type": "qwen2",
-  "num_attention_heads": 16,
-  "num_hidden_layers": 24,
-  "num_key_value_heads": 16,
-  "rms_norm_eps": 1e-06,
-  "rope_theta": 1000000.0,
-  "sliding_window": 32768,
-  "tie_word_embeddings": true,
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.2",
-  "use_cache": true,
-  "use_sliding_window": false,
-  "vocab_size": 151936
-}
-
-
-05/19/2024 22:55:58 - INFO - llamafactory.model.utils.quantization - Quantizing model to 8 bit.
-
-05/19/2024 22:55:58 - INFO - transformers.modeling_utils - loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen1.5-0.5B-Chat/snapshots/4d14e384a4b037942bb3f3016665157c8bcb70ea/model.safetensors
-
-05/19/2024 22:55:58 - INFO - transformers.modeling_utils - Instantiating Qwen2ForCausalLM model under default dtype torch.float16.
-
-05/19/2024 22:55:58 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig {
-  "bos_token_id": 151643,
-  "eos_token_id": 151645
-}
-
-
-05/19/2024 22:56:02 - INFO - transformers.modeling_utils - All model checkpoint weights were used when initializing Qwen2ForCausalLM.
-
-
-05/19/2024 22:56:02 - INFO - transformers.modeling_utils - All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen1.5-0.5B-Chat.
-If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.
-
-05/19/2024 22:56:02 - INFO - transformers.generation.configuration_utils - loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen1.5-0.5B-Chat/snapshots/4d14e384a4b037942bb3f3016665157c8bcb70ea/generation_config.json
-
-05/19/2024 22:56:02 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig {
-  "bos_token_id": 151643,
-  "do_sample": true,
-  "eos_token_id": [
-    151645,
-    151643
-  ],
-  "pad_token_id": 151643,
-  "repetition_penalty": 1.1,
-  "top_p": 0.8
-}
-
-
-05/19/2024 22:56:02 - INFO - llamafactory.model.utils.checkpointing - Gradient checkpointing enabled.
-
-05/19/2024 22:56:02 - INFO - llamafactory.model.utils.attention - Using torch SDPA for faster training and inference.
-
-05/19/2024 22:56:02 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
-
-05/19/2024 22:56:02 - INFO - llamafactory.model.adapter - Fine-tuning method: LoRA
-
-05/19/2024 22:56:02 - INFO - llamafactory.model.loader - trainable params: 786432 || all params: 464774144 || trainable%: 0.1692
-
-05/19/2024 22:56:02 - INFO - transformers.trainer - Using auto half precision backend
-
-05/19/2024 22:56:03 - INFO - transformers.trainer - ***** Running training *****
-
-05/19/2024 22:56:03 - INFO - transformers.trainer -   Num examples = 91
-
-05/19/2024 22:56:03 - INFO - transformers.trainer -   Num Epochs = 3
-
-05/19/2024 22:56:03 - INFO - transformers.trainer -   Instantaneous batch size per device = 2
-
-05/19/2024 22:56:03 - INFO - transformers.trainer -   Total train batch size (w. parallel, distributed & accumulation) = 16
-
-05/19/2024 22:56:03 - INFO - transformers.trainer -   Gradient Accumulation steps = 8
-
-05/19/2024 22:56:03 - INFO - transformers.trainer -   Total optimization steps = 15
-
-05/19/2024 22:56:03 - INFO - transformers.trainer -   Number of trainable parameters = 786,432
-
-05/19/2024 22:56:30 - INFO - llamafactory.extras.callbacks - {'loss': 3.4258, 'learning_rate': 3.7500e-05, 'epoch': 0.87}
-
-05/19/2024 22:56:57 - INFO - llamafactory.extras.callbacks - {'loss': 3.3578, 'learning_rate': 1.2500e-05, 'epoch': 1.74}
-
-05/19/2024 22:57:24 - INFO - llamafactory.extras.callbacks - {'loss': 3.2979, 'learning_rate': 0.0000e+00, 'epoch': 2.61}
-
-05/19/2024 22:57:24 - INFO - transformers.trainer - 
-
-Training completed. Do not forget to share your model on huggingface.co/models =)
-
-
-
-05/19/2024 22:57:24 - INFO - transformers.trainer - Saving model checkpoint to saves/Qwen1.5-0.5B-Chat/lora/QwenTT-0.5B-INT8
-
-05/19/2024 22:57:25 - INFO - transformers.configuration_utils - loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--Qwen--Qwen1.5-0.5B-Chat/snapshots/4d14e384a4b037942bb3f3016665157c8bcb70ea/config.json
-
-05/19/2024 22:57:25 - INFO - transformers.configuration_utils - Model config Qwen2Config {
-  "architectures": [
-    "Qwen2ForCausalLM"
-  ],
-  "attention_dropout": 0.0,
-  "bos_token_id": 151643,
-  "eos_token_id": 151645,
-  "hidden_act": "silu",
-  "hidden_size": 1024,
-  "initializer_range": 0.02,
-  "intermediate_size": 2816,
-  "max_position_embeddings": 32768,
-  "max_window_layers": 21,
-  "model_type": "qwen2",
-  "num_attention_heads": 16,
-  "num_hidden_layers": 24,
-  "num_key_value_heads": 16,
-  "rms_norm_eps": 1e-06,
-  "rope_theta": 1000000.0,
-  "sliding_window": 32768,
-  "tie_word_embeddings": true,
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.2",
-  "use_cache": true,
-  "use_sliding_window": false,
-  "vocab_size": 151936
-}
-
-
-05/19/2024 22:57:25 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in saves/Qwen1.5-0.5B-Chat/lora/QwenTT-0.5B-INT8/tokenizer_config.json
-
-05/19/2024 22:57:25 - INFO - transformers.tokenization_utils_base - Special tokens file saved in saves/Qwen1.5-0.5B-Chat/lora/QwenTT-0.5B-INT8/special_tokens_map.json
-
-05/19/2024 22:57:25 - WARNING - llamafactory.extras.ploting - No metric eval_loss to plot.
-
-05/19/2024 22:57:25 - INFO - transformers.modelcard - Dropping the following result as it does not have all the necessary fields:
-{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
-

QwenTT-0.5B-INT8/special_tokens_map.json

@@ -1,20 +0,0 @@
-{
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>"
-  ],
-  "eos_token": {
-    "content": "<|im_end|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}

QwenTT-0.5B-INT8/tokenizer_config.json

@@ -1,44 +0,0 @@
-{
-  "add_prefix_space": false,
-  "added_tokens_decoder": {
-    "151643": {
-      "content": "<|endoftext|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151644": {
-      "content": "<|im_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151645": {
-      "content": "<|im_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>"
-  ],
-  "bos_token": null,
-  "chat_template": "{% set system_message = 'You are a helpful assistant.' %}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ '<|im_start|>system\n' + system_message + '<|im_end|>\n' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user\n' + content + '<|im_end|>\n<|im_start|>assistant\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '\n' }}{% endif %}{% endfor %}",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "<|im_end|>",
-  "errors": "replace",
-  "model_max_length": 32768,
-  "pad_token": "<|endoftext|>",
-  "padding_side": "right",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
-}

QwenTT-0.5B-INT8/train_results.json

@@ -1,8 +0,0 @@
-{
-    "epoch": 2.608695652173913,
-    "total_flos": 27871774801920.0,
-    "train_loss": 3.3604891459147135,
-    "train_runtime": 81.3934,
-    "train_samples_per_second": 3.354,
-    "train_steps_per_second": 0.184
-}

QwenTT-0.5B-INT8/trainer_config.yaml

@@ -1,31 +0,0 @@
-cutoff_len: 1024
-dataset: identity
-dataset_dir: data
-do_train: true
-finetuning_type: lora
-flash_attn: auto
-fp16: true
-gradient_accumulation_steps: 8
-learning_rate: 5.0e-05
-logging_steps: 5
-lora_alpha: 16
-lora_dropout: 0
-lora_rank: 8
-lora_target: q_proj,v_proj
-lr_scheduler_type: cosine
-max_grad_norm: 1.0
-max_samples: 100000
-model_name_or_path: Qwen/Qwen1.5-0.5B-Chat
-num_train_epochs: 3.0
-optim: adamw_torch
-output_dir: saves/Qwen1.5-0.5B-Chat/lora/QwenTT-0.5B-INT8
-packing: false
-per_device_train_batch_size: 2
-plot_loss: true
-preprocessing_num_workers: 16
-quantization_bit: 8
-report_to: none
-save_steps: 100
-stage: sft
-template: qwen
-warmup_steps: 0

QwenTT-0.5B-INT8/trainer_log.jsonl

@@ -1,4 +0,0 @@
-{"current_steps": 5, "total_steps": 15, "loss": 3.4258, "learning_rate": 3.7500000000000003e-05, "epoch": 0.8695652173913043, "percentage": 33.33, "elapsed_time": "0:00:27", "remaining_time": "0:00:54"}
-{"current_steps": 10, "total_steps": 15, "loss": 3.3578, "learning_rate": 1.2500000000000006e-05, "epoch": 1.7391304347826086, "percentage": 66.67, "elapsed_time": "0:00:54", "remaining_time": "0:00:27"}
-{"current_steps": 15, "total_steps": 15, "loss": 3.2979, "learning_rate": 0.0, "epoch": 2.608695652173913, "percentage": 100.0, "elapsed_time": "0:01:21", "remaining_time": "0:00:00"}
-{"current_steps": 15, "total_steps": 15, "epoch": 2.608695652173913, "percentage": 100.0, "elapsed_time": "0:01:21", "remaining_time": "0:00:00"}

QwenTT-0.5B-INT8/trainer_state.json

@@ -1,51 +0,0 @@
-{
-  "best_metric": null,
-  "best_model_checkpoint": null,
-  "epoch": 2.608695652173913,
-  "eval_steps": 500,
-  "global_step": 15,
-  "is_hyper_param_search": false,
-  "is_local_process_zero": true,
-  "is_world_process_zero": true,
-  "log_history": [
-    {
-      "epoch": 0.8695652173913043,
-      "grad_norm": 1.6034659147262573,
-      "learning_rate": 3.7500000000000003e-05,
-      "loss": 3.4258,
-      "step": 5
-    },
-    {
-      "epoch": 1.7391304347826086,
-      "grad_norm": 1.0319217443466187,
-      "learning_rate": 1.2500000000000006e-05,
-      "loss": 3.3578,
-      "step": 10
-    },
-    {
-      "epoch": 2.608695652173913,
-      "grad_norm": 1.2878139019012451,
-      "learning_rate": 0.0,
-      "loss": 3.2979,
-      "step": 15
-    },
-    {
-      "epoch": 2.608695652173913,
-      "step": 15,
-      "total_flos": 27871774801920.0,
-      "train_loss": 3.3604891459147135,
-      "train_runtime": 81.3934,
-      "train_samples_per_second": 3.354,
-      "train_steps_per_second": 0.184
-    }
-  ],
-  "logging_steps": 5,
-  "max_steps": 15,
-  "num_input_tokens_seen": 0,
-  "num_train_epochs": 3,
-  "save_steps": 100,
-  "total_flos": 27871774801920.0,
-  "train_batch_size": 2,
-  "trial_name": null,
-  "trial_params": null
-}

QwenTT-0.5B-INT8/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:44c4d8f46ae72d6f97f06fdd5f7f5b232339172b58af2232de6515d3688c44bf
-size 5176