Text Generation
PEFT
Russian
English
mistral
custom_code
Paul Rock commited on
Commit
a33d530
1 Parent(s): aa68cf9

Initial commit

Browse files
.gitignore ADDED
@@ -0,0 +1 @@
 
 
1
+ /.idea/
README.md CHANGED
@@ -1,3 +1,12 @@
1
  ---
2
  license: mit
 
 
 
 
 
 
 
 
 
3
  ---
 
1
  ---
2
  license: mit
3
+ datasets:
4
+ - d0rj/gsm8k-ru
5
+ - d0rj/alpaca-cleaned-ru
6
+ - IlyaGusev/ru_turbo_alpaca
7
+ - IlyaGusev/ru_turbo_alpaca_evol_instruct
8
+ language:
9
+ - ru
10
+ library_name: peft
11
+ pipeline_tag: text-generation
12
  ---
adapter_config.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "/home/pasha/Documents/Repository/gpt/saiga-custom/Yarn-Mistral-7b-128k",
5
+ "bias": "none",
6
+ "fan_in_fan_out": false,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layers_pattern": null,
10
+ "layers_to_transform": null,
11
+ "loftq_config": {},
12
+ "lora_alpha": 16,
13
+ "lora_dropout": 0.05,
14
+ "megatron_config": null,
15
+ "megatron_core": "megatron.core",
16
+ "modules_to_save": null,
17
+ "peft_type": "LORA",
18
+ "r": 16,
19
+ "rank_pattern": {},
20
+ "revision": null,
21
+ "target_modules": [
22
+ "q_proj",
23
+ "o_proj",
24
+ "k_proj",
25
+ "v_proj"
26
+ ],
27
+ "task_type": "CAUSAL_LM"
28
+ }
adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:972e34d18e05e346f738031a80b365cb82c29c0b9d6e0674ee4899ff5541d2fe
3
+ size 54618762
chat.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "system_prompt": "Ты — PavelGPT, русскоязычный автоматический ассистент. Ты разговариваешь с людьми и помогаешь им.",
3
+ "system_message_template": "<s>{role}\n{content}</s>\n",
4
+ "user_message_template": "<s>{role}\n{content}</s>\n",
5
+ "bot_message_template": "<s>{role}\n{content}</s>\n",
6
+ "user_role": "user",
7
+ "bot_role": "bot",
8
+ "system_role": "system",
9
+ "suffix": ""
10
+ }
config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "NousResearch/Yarn-Mistral-7b-128k",
3
+ "architectures": [
4
+ "MistralForCausalLM"
5
+ ],
6
+ "auto_map": {
7
+ "AutoConfig": "configuration_mistral.MistralConfig",
8
+ "AutoModelForCausalLM": "modeling_mistral_yarn.MistralForCausalLM"
9
+ },
10
+ "bos_token_id": 1,
11
+ "eos_token_id": 2,
12
+ "hidden_act": "silu",
13
+ "hidden_size": 4096,
14
+ "initializer_range": 0.02,
15
+ "intermediate_size": 14336,
16
+ "max_position_embeddings": 32768,
17
+ "max_sequence_length": 131072,
18
+ "model_type": "mistral",
19
+ "num_attention_heads": 32,
20
+ "num_hidden_layers": 32,
21
+ "num_key_value_heads": 8,
22
+ "rms_norm_eps": 1e-05,
23
+ "rope_scaling": {
24
+ "factor": 16.0,
25
+ "finetuned": true,
26
+ "original_max_position_embeddings": 8192,
27
+ "type": "yarn"
28
+ },
29
+ "rope_theta": 10000.0,
30
+ "sliding_window": 131072,
31
+ "tie_word_embeddings": false,
32
+ "torch_dtype": "bfloat16",
33
+ "transformers_version": "4.35.0.dev0",
34
+ "use_cache": true,
35
+ "vocab_size": 32000
36
+ }
generation_config.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "pad_token_id": 0,
3
+ "unk_token_id": 0,
4
+ "bos_token_id": 1,
5
+ "eos_token_id": 2,
6
+ "temperature": 0.2,
7
+ "top_p": 0.2,
8
+ "top_k": 20,
9
+ "do_sample": true,
10
+ "max_new_tokens": 4096,
11
+ "repetition_penalty": 1.2,
12
+ "no_repeat_ngram_size": 15
13
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "<unk>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "sep_token": {
24
+ "content": "<s>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ },
30
+ "unk_token": {
31
+ "content": "<unk>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false
36
+ }
37
+ }
test_lora.py ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import logging
3
+ from peft import PeftModel, PeftConfig
4
+ from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
5
+
6
+ # MODEL_NAME = "IlyaGusev/gigasaiga_lora"
7
+ # MODEL_NAME = "evilfreelancer/ruGPT-3.5-13B-lora"
8
+ # MODEL_NAME = "evilfreelancer/saiga_mistral_7b_128k_lora"
9
+ # MODEL_NAME = "./yarn_mistral_7b_128k"
10
+ MODEL_NAME = "./yarn_mistral_7b_128k_yakovlev"
11
+
12
+ DEFAULT_MESSAGE_TEMPLATE = "<s>{role}\n{content}</s>\n"
13
+ # DEFAULT_SYSTEM_PROMPT = """
14
+ # Ты — Saiga 2, русскоязычный автоматический ассистент. Ты разговариваешь с людьми и помогаешь им.
15
+ # """
16
+ DEFAULT_SYSTEM_PROMPT = """
17
+ Ты - Иван Яковлевич Яковлев. Ты был просветителем, православным миссионером, российский педагогом, организатором народных школ, инспектором (руководителем) чувашских школ Казанского учебного округа (1875-1903) и созданной им Симбирской чувашской учительской школы (1868-1919). Я создал современный чувашский алфавит (1870-е годы) и учебники чувашского и русского языков для чувашей, был писателем, переводчиком, фольклористом. Действительный статский советник (1904).
18
+ """
19
+
20
+
21
+ class Conversation:
22
+ def __init__(
23
+ self,
24
+ message_template=DEFAULT_MESSAGE_TEMPLATE,
25
+ system_prompt=DEFAULT_SYSTEM_PROMPT,
26
+ start_token_id=2,
27
+ # Bot token may be a list or single int
28
+ bot_token_id=10093, # yarn_mistral_7b_128k
29
+ # bot_token_id=46787, # rugpt35_13b
30
+ # int (amount of questions and answers) or None (unlimited)
31
+ history_limit=None,
32
+ ):
33
+ self.logger = logging.getLogger('Conversation')
34
+ self.message_template = message_template
35
+ self.start_token_id = start_token_id
36
+ self.bot_token_id = bot_token_id
37
+ self.history_limit = history_limit
38
+ self.messages = [{
39
+ "role": "system",
40
+ "content": system_prompt
41
+ }]
42
+
43
+ def get_start_token_id(self):
44
+ return self.start_token_id
45
+
46
+ def get_bot_token_id(self):
47
+ return self.bot_token_id
48
+
49
+ def add_message(self, role, message):
50
+ self.messages.append({
51
+ "role": role,
52
+ "content": message
53
+ })
54
+ self.trim_history()
55
+
56
+ def add_user_message(self, message):
57
+ self.add_message("user", message)
58
+
59
+ def add_bot_message(self, message):
60
+ self.add_message("assistant", message)
61
+
62
+ def trim_history(self):
63
+ if self.history_limit is not None and len(self.messages) > self.history_limit + 1:
64
+ overflow = len(self.messages) - (self.history_limit + 1)
65
+ self.messages = [self.messages[0]] + self.messages[overflow + 1:] # remove old messages except system
66
+
67
+ def get_prompt(self, tokenizer):
68
+ final_text = ""
69
+ # print(self.messages)
70
+ for message in self.messages:
71
+ message_text = self.message_template.format(**message)
72
+ final_text += message_text
73
+
74
+ # Bot token id may be an array
75
+ if isinstance(self.bot_token_id, (list, tuple)):
76
+ final_text += tokenizer.decode([self.start_token_id] + self.bot_token_id)
77
+ else:
78
+ final_text += tokenizer.decode([self.start_token_id, self.bot_token_id])
79
+
80
+ return final_text.strip()
81
+
82
+
83
+ def generate(model, tokenizer, prompt, generation_config):
84
+ data = tokenizer(prompt, return_tensors="pt")
85
+ data = {k: v.to(model.device) for k, v in data.items()}
86
+ output_ids = model.generate(
87
+ **data,
88
+ max_length=10240,
89
+ generation_config=generation_config
90
+ )[0]
91
+ output_ids = output_ids[len(data["input_ids"][0]):]
92
+ output = tokenizer.decode(output_ids, skip_special_tokens=True)
93
+ return output.strip()
94
+
95
+
96
+ config = PeftConfig.from_pretrained(MODEL_NAME)
97
+ model = AutoModelForCausalLM.from_pretrained(
98
+ config.base_model_name_or_path,
99
+ load_in_8bit=True,
100
+ torch_dtype=torch.float16,
101
+ device_map="auto",
102
+ use_flash_attention_2=True,
103
+ )
104
+ model = PeftModel.from_pretrained(
105
+ model,
106
+ MODEL_NAME,
107
+ torch_dtype=torch.float16
108
+ )
109
+ model.eval()
110
+
111
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, use_fast=False)
112
+ generation_config = GenerationConfig.from_pretrained(MODEL_NAME)
113
+ print(generation_config)
114
+
115
+ conversation = Conversation()
116
+ while True:
117
+ user_message = input("User: ")
118
+
119
+ # Reset chat command
120
+ if user_message.strip() == "/reset":
121
+ conversation = Conversation()
122
+ print("History reset completed!")
123
+ continue
124
+
125
+ # Skip empty messages from user
126
+ if user_message.strip() == "":
127
+ continue
128
+
129
+ conversation.add_user_message(user_message)
130
+ prompt = conversation.get_prompt(tokenizer)
131
+ output = generate(
132
+ model=model,
133
+ tokenizer=tokenizer,
134
+ prompt=prompt,
135
+ generation_config=generation_config
136
+ )
137
+ conversation.add_bot_message(output)
138
+ print("Bot:", output)
139
+ print()
140
+ print("==============================")
141
+ print()
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
3
+ size 493443
tokenizer_config.json ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ }
29
+ },
30
+ "bos_token": "<s>",
31
+ "clean_up_tokenization_spaces": false,
32
+ "eos_token": "</s>",
33
+ "legacy": true,
34
+ "model_max_length": 32768,
35
+ "pad_token": "<unk>",
36
+ "padding_side": "left",
37
+ "sep_token": "<s>",
38
+ "sp_model_kwargs": {},
39
+ "spaces_between_special_tokens": false,
40
+ "tokenizer_class": "LlamaTokenizer",
41
+ "unk_token": "<unk>",
42
+ "use_default_system_prompt": false
43
+ }