Commit
•
47ef1ed
1
Parent(s):
216ae70
Upload 8 files
Browse files- README.md +45 -3
- config.json +28 -0
- generation_config.json +7 -0
- model.safetensors +3 -0
- special_tokens_map.json +30 -0
- tokenizer.json +0 -0
- tokenizer.model +3 -0
- tokenizer_config.json +41 -0
README.md
CHANGED
@@ -1,3 +1,45 @@
|
|
1 |
-
---
|
2 |
-
license:
|
3 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
license: apache-2.0
|
3 |
+
language:
|
4 |
+
- en
|
5 |
+
---
|
6 |
+
<div align="center">
|
7 |
+
|
8 |
+
# KobbleTinyV2-1.1B
|
9 |
+
</div>
|
10 |
+
|
11 |
+
This is a finetune of https://huggingface.co/TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T trained on a small 50mb subset of the Kobble Dataset.
|
12 |
+
Training was done in under 2 hours on a single Nvidia RTX 2060 Mobile GPU with qLora (LR 1.5e-4, rank 8, alpha 16, batch size 2, gradient acc. 4, 2048 ctx).
|
13 |
+
|
14 |
+
You can obtain the GGUF quantization of this model here: https://huggingface.co/concedo/KobbleTinyV2-1.1B-GGUF
|
15 |
+
|
16 |
+
Update: KobbleTiny has been upgraded to V2! The old V1 is [still available at this link](https://huggingface.co/concedo/KobbleTiny/tree/eb0c96864bfecfd6ac9ece1a42c4654b4997eb72).
|
17 |
+
|
18 |
+
<video width="320" controls autoplay src="https://cdn-uploads.huggingface.co/production/uploads/63cd4b6d1c8a5d1d7d76a778/zjHfohCnEu2Y9CWSWgf0n.mp4"></video>
|
19 |
+
|
20 |
+
Try it live now: https://concedo-koboldcpp-kobbletiny.hf.space/
|
21 |
+
|
22 |
+
## Dataset and Objectives
|
23 |
+
|
24 |
+
The Kobble Dataset is a semi-private aggregated dataset made from multiple online sources and web scrapes.
|
25 |
+
It contains content chosen and formatted specifically to work with KoboldAI software and Kobold Lite.
|
26 |
+
|
27 |
+
#### Dataset Categories:
|
28 |
+
- Instruct: Single turn instruct examples presented in the Alpaca format, with an emphasis on uncensored and unrestricted responses.
|
29 |
+
- Chat: Two participant roleplay conversation logs in a multi-turn raw chat format that KoboldAI uses.
|
30 |
+
- Story: Unstructured fiction excerpts, including literature containing various erotic and provocative content.
|
31 |
+
|
32 |
+
<!-- prompt-template start -->
|
33 |
+
## Prompt template: Alpaca
|
34 |
+
|
35 |
+
```
|
36 |
+
### Instruction:
|
37 |
+
{prompt}
|
38 |
+
|
39 |
+
### Response:
|
40 |
+
```
|
41 |
+
|
42 |
+
<!-- prompt-template end -->
|
43 |
+
|
44 |
+
**Note:** *No assurances will be provided about the **origins, safety, or copyright status** of this model, or of **any content** within the Kobble dataset.*
|
45 |
+
*If you belong to a country or organization that has strict AI laws or restrictions against unlabelled or unrestricted content, you are advised not to use this model.*
|
config.json
ADDED
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"architectures": [
|
3 |
+
"LlamaForCausalLM"
|
4 |
+
],
|
5 |
+
"attention_bias": false,
|
6 |
+
"attention_dropout": 0.0,
|
7 |
+
"bos_token_id": 1,
|
8 |
+
"eos_token_id": 2,
|
9 |
+
"hidden_act": "silu",
|
10 |
+
"hidden_size": 2048,
|
11 |
+
"initializer_range": 0.02,
|
12 |
+
"intermediate_size": 5632,
|
13 |
+
"max_position_embeddings": 2048,
|
14 |
+
"model_type": "llama",
|
15 |
+
"num_attention_heads": 32,
|
16 |
+
"num_hidden_layers": 22,
|
17 |
+
"num_key_value_heads": 4,
|
18 |
+
"pad_token_id": 0,
|
19 |
+
"pretraining_tp": 1,
|
20 |
+
"rms_norm_eps": 1e-05,
|
21 |
+
"rope_scaling": null,
|
22 |
+
"rope_theta": 10000.0,
|
23 |
+
"tie_word_embeddings": false,
|
24 |
+
"torch_dtype": "float16",
|
25 |
+
"transformers_version": "4.40.0",
|
26 |
+
"use_cache": true,
|
27 |
+
"vocab_size": 32000
|
28 |
+
}
|
generation_config.json
ADDED
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"bos_token_id": 1,
|
3 |
+
"eos_token_id": 2,
|
4 |
+
"max_length": 2048,
|
5 |
+
"pad_token_id": 0,
|
6 |
+
"transformers_version": "4.40.0"
|
7 |
+
}
|
model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:b5dd8a25c23133b308eff29d17bccd9a282bdb9c7e692f2b2a3a0f9102ab8408
|
3 |
+
size 2200119664
|
special_tokens_map.json
ADDED
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"bos_token": {
|
3 |
+
"content": "<s>",
|
4 |
+
"lstrip": false,
|
5 |
+
"normalized": false,
|
6 |
+
"rstrip": false,
|
7 |
+
"single_word": false
|
8 |
+
},
|
9 |
+
"eos_token": {
|
10 |
+
"content": "</s>",
|
11 |
+
"lstrip": false,
|
12 |
+
"normalized": false,
|
13 |
+
"rstrip": false,
|
14 |
+
"single_word": false
|
15 |
+
},
|
16 |
+
"pad_token": {
|
17 |
+
"content": "<unk>",
|
18 |
+
"lstrip": false,
|
19 |
+
"normalized": false,
|
20 |
+
"rstrip": false,
|
21 |
+
"single_word": false
|
22 |
+
},
|
23 |
+
"unk_token": {
|
24 |
+
"content": "<unk>",
|
25 |
+
"lstrip": false,
|
26 |
+
"normalized": false,
|
27 |
+
"rstrip": false,
|
28 |
+
"single_word": false
|
29 |
+
}
|
30 |
+
}
|
tokenizer.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
tokenizer.model
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
|
3 |
+
size 499723
|
tokenizer_config.json
ADDED
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"add_bos_token": true,
|
3 |
+
"add_eos_token": false,
|
4 |
+
"added_tokens_decoder": {
|
5 |
+
"0": {
|
6 |
+
"content": "<unk>",
|
7 |
+
"lstrip": false,
|
8 |
+
"normalized": false,
|
9 |
+
"rstrip": false,
|
10 |
+
"single_word": false,
|
11 |
+
"special": true
|
12 |
+
},
|
13 |
+
"1": {
|
14 |
+
"content": "<s>",
|
15 |
+
"lstrip": false,
|
16 |
+
"normalized": false,
|
17 |
+
"rstrip": false,
|
18 |
+
"single_word": false,
|
19 |
+
"special": true
|
20 |
+
},
|
21 |
+
"2": {
|
22 |
+
"content": "</s>",
|
23 |
+
"lstrip": false,
|
24 |
+
"normalized": false,
|
25 |
+
"rstrip": false,
|
26 |
+
"single_word": false,
|
27 |
+
"special": true
|
28 |
+
}
|
29 |
+
},
|
30 |
+
"bos_token": "<s>",
|
31 |
+
"clean_up_tokenization_spaces": false,
|
32 |
+
"eos_token": "</s>",
|
33 |
+
"legacy": false,
|
34 |
+
"model_max_length": 2048,
|
35 |
+
"pad_token": "<unk>",
|
36 |
+
"padding_side": "right",
|
37 |
+
"sp_model_kwargs": {},
|
38 |
+
"tokenizer_class": "LlamaTokenizer",
|
39 |
+
"unk_token": "<unk>",
|
40 |
+
"use_default_system_prompt": false
|
41 |
+
}
|