add app.py and requirments.txt

README.md

@@ -10,4 +10,3 @@ pinned: false
 license: apache-2.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,88 @@
+import gradio as gr
+import torch
+import os
+import sys
+import time
+import json
+from typing import List
+
+from transformers import (
+    LlamaTokenizer,
+    LlamaForCausalLM,
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    LlamaConfig
+)
+from peft import PeftModel
+from accelerate import disk_offload
+
+model = AutoModelForCausalLM.from_pretrained(
+    "Johntad110/llama-2-7b-amharic-tokenizer",
+    return_dict=True,
+    load_in_8bit=True,
+    device_map="auto",
+    low_cpu_mem_usage=True,
+    attn_implementation="sdpa"
+)
+
+tokenizer = LlamaTokenizer.from_pretrained(
+    "Johntad110/llama-2-7b-amharic-tokenizer"
+)
+
+embedding_size = model.get_input_embeddings().weight.shape[0]
+if len(tokenizer) != embedding_size:
+    model.resize_token_embeddings(len(tokenizer))
+
+model = PeftModel.from_pretrained(model, "Johntad110/llama-2-amharic-peft")
+
+model.eval()  # Set model to evaluation mode
+
+
+def generate_text(
+    prompt: str,
+    max_new_tokens: int = None,
+    seed: int = 42,
+    do_sample: bool = True,
+    min_length: int = None,
+    use_cache: bool = True,
+    top_p: float = 1.0,
+    temperature: float = 1.0,
+    top_k: int = 1,
+    repetition_penalty: float = 1.0,
+    length_penalty: int = 1,
+):
+    """
+    Function to perform text generation with user-defined parameters
+    """
+
+    torch.cuda.manual_seed(seed)
+    torch.manual_seed(seed)
+
+    batch = tokenizer(prompt, return_tensors="pt")
+    batch = {k: v.to("cuda") for k, v in batch.items()}
+
+    with torch.no_grad():
+        outputs = model.generate(
+            **batch,
+            max_new_tokens=max_new_tokens,
+            do_sample=do_sample,
+            top_p=top_p,
+            temperature=temperature,
+            min_length=min_length,
+            use_cache=use_cache,
+            top_k=top_k,
+            repetition_penalty=repetition_penalty,
+            length_penalty=length_penalty,
+        )
+
+    output_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return output_text
+
+
+interface = gr.Interface(
+    fn=generate_text,
+    inputs=[gr.Textbox(label="Prompt")],
+    outputs="text"
+)
+
+interface.launch(debug=True)

requirements.txt

@@ -0,0 +1,5 @@
+peft==0.10.0
+fire==0.6.0
+accelerate==0.29.3
+bitsandbytes==0.43.1
+gradio==4.27.0