Upload folder using huggingface_hub

README.md

@@ -1,19 +1,17 @@
 ---
 license: apache-2.0
-base_model: TinyLlama/TinyLlama-1.1B-Chat-v1.0
 tags:
 - TinyLlama
 - QLoRA
 - Politics
 - EU
 - sft
-language:
-- en
+base_model: TinyLlama/TinyLlama-1.1B-Chat-v1.0
 ---
 
 # TinyParlaMintLlama-1.1B
 
-TinyParlaMintLlama-1.1B is a QLoRA SFT fine-tune of [TinyLlama/TinyLlama-1.1B-Chat-v1.0](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0) using a sample of a concentrated version of the English [ParlaMint] (https://www.clarin.si/repository/xmlui/handle/11356/1864) Dataset. The model was fine-tuned for ~12h on one A100 40GB on ~125M tokens.
+TinyParlaMintLlama-1.1B is a SFT fine-tune of [TinyLlama/TinyLlama-1.1B-Chat-v1.0](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0) using a sample of a concentrated version of the English [ParlaMint] (https://www.clarin.si/repository/xmlui/handle/11356/1864) Dataset using QLoRA. The model was fine-tuned for ~12h on one A100 40GB on ~100M tokens.
 
 The goal of this project is to study the potential for improving the domain-specific (in this case political) knowledge of small (<3B) LLMs by concentrating the training datasets TF-IDF in respect to the underlying Topics found in the origianl Dataset.
 
@@ -24,7 +22,7 @@ The used training data contains speeches from the **Austrian**, **Danish**, **Fr
 
 ```python
 !pip install -qU transformers accelerate
-from transformers import AutoTokenizer
+from transformers import AutoTokenizer, AutoModelForCausalLM
 import transformers
 import torch
 model = "h4rz3rk4s3/TinyParlaMintLlama-1.1B"
@@ -38,10 +36,16 @@ messages = [
 
 tokenizer = AutoTokenizer.from_pretrained(model)
 prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+model = AutoModelForCausalLM.from_pretrained(
+    model, trust_remote_code=True, device_map={"": Accelerator().process_index}
+)
+
 pipeline = transformers.pipeline(
     "text-generation",
+    tokenizer=tokenizer,
     model=model,
-    device_map="auto",
+    torch_dtype=torch.float16,
+    device_map={"": Accelerator().process_index},
 )
 outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
 print(outputs[0]["generated_text"])

config.json

@@ -16,27 +16,12 @@
   "num_attention_heads": 32,
   "num_hidden_layers": 22,
   "num_key_value_heads": 4,
-  "pretraining_tp": 1,
-  "quantization_config": {
-    "_load_in_4bit": true,
-    "_load_in_8bit": false,
-    "bnb_4bit_compute_dtype": "bfloat16",
-    "bnb_4bit_quant_type": "nf4",
-    "bnb_4bit_use_double_quant": false,
-    "llm_int8_enable_fp32_cpu_offload": false,
-    "llm_int8_has_fp16_weight": false,
-    "llm_int8_skip_modules": null,
-    "llm_int8_threshold": 6.0,
-    "load_in_4bit": true,
-    "load_in_8bit": false,
-    "quant_method": "bitsandbytes"
-  },
   "rms_norm_eps": 1e-05,
   "rope_scaling": null,
   "rope_theta": 10000.0,
   "tie_word_embeddings": false,
-  "torch_dtype": "float16",
-  "transformers_version": "4.38.0.dev0",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.38.0",
   "use_cache": true,
   "vocab_size": 32000
-}
+}

generation_config.json

@@ -3,5 +3,5 @@
   "eos_token_id": 2,
   "max_length": 2048,
   "pad_token_id": 0,
-  "transformers_version": "4.38.0.dev0"
+  "transformers_version": "4.38.0"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:59b91068db6d7cf982aa85cdb5a1e69db86a439420c79dffa7a5d04f34f45419
-size 807426286
+oid sha256:db30ca6110845908b747a44327b67440a0722e591009392152a815f5de622bfc
+size 2200119864