Update app.py

app.py

@@ -1,59 +1,43 @@
 import gradio as gr
-from transformers import pipeline, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
 
-def load_model(model_name):
-    return pipeline("text-generation", model=model_name, device="cpu")
+# Laden des Modells und Tokenizers
+model_name = "dumb-dev/TinyLlama-1.1B-Chat-rust-cpp-encodings"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name)
 
-def generate(
-    model_name,
-    template_name,
-    user_input,
-    temperature=0.4,
-    top_p=0.95,
-    top_k=50,
-    max_new_tokens=256,
-):
-    pipe = load_model(model_name)
-    # Need to add additional options later.
-    if template_name == "Falcon 1B Template":
-        message_template = [
-            {"role": "user", "content": "Hello!"},
-            {"role": "assistant", "content": "Hello! How can I assist you today?"},
-            {"role": "user", "content": user_input},
-        ]
-    else:  # Default to "TinyLlama Template"
-        message_template = [
-            {
-                "role": "system",
-                "content": "You are a highly knowledgeable and friendly chatbot equipped with extensive information across various domains. Your goal is to understand and respond to user inquiries with accuracy and clarity. You're adept at providing detailed explanations, concise summaries, and insightful responses. Your interactions are always respectful, helpful, and focused on delivering the most relevant information to the user.",
-            },
-            {"role": "user", "content": user_input},
-        ]
+def generate_text(prompt, top_p, top_k, max_tokens, temperature):
+    inputs = tokenizer(prompt, return_tensors="pt")
+    input_ids = inputs["input_ids"]
 
-    # Set tokenize correctly. Otherwise ticking the box breaks it.
-    prompt = pipe.tokenizer.apply_chat_template(message_template, tokenize=False, add_generation_prompt=True)
-    outputs = pipe(prompt, max_new_tokens=max_new_tokens, do_sample=True,
-                   temperature=temperature, top_k=top_k, top_p=top_p, repetition_penalty=1.10)
-    return outputs[0]["generated_text"]
+    with torch.no_grad():
+        outputs = model.generate(
+            input_ids,
+            do_sample=True,
+            max_length=max_tokens,
+            top_p=top_p,
+            top_k=top_k,
+            temperature=temperature,
+        )
+    
+    generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return generated_text
 
-model_choices = ["dumb-dev/TinyLlama-1.1B-Chat-rust-cpp-encodings"]
-template_choices = ["TinyLlama Template"]
-# What at the best options? 
-g = gr.Interface(
-    fn=generate,
+# Gradio Interface
+interface = gr.Interface(
+    fn=generate_text,
     inputs=[
-        gr.components.Dropdown(choices=model_choices, label="Model", value=model_choices[0], interactive=True),
-        gr.components.Dropdown(choices=template_choices, label="Template", value=template_choices[0], interactive=True),
-        gr.components.Textbox(lines=2, label="Prompt", value="How many planets are in our solar system?"),
-        gr.components.Slider(minimum=0, maximum=1, value=0.4, label="Temperature"),
-        gr.components.Slider(minimum=0, maximum=1, value=0.95, label="Top p"),
-        gr.components.Slider(minimum=0, maximum=100, step=1, value=50, label="Top k"),
-        gr.components.Slider(minimum=1, maximum=1024, step=1, value=256, label="Max tokens"),  
+        gr.Textbox(lines=2, placeholder="Enter your prompt here..."),
+        gr.Slider(0, 1, step=0.01, value=0.9, label="Top-p"),
+        gr.Slider(0, 100, step=1, value=50, label="Top-k"),
+        gr.Slider(1, 512, step=1, value=100, label="Max tokens"),
+        gr.Slider(0.1, 2, step=0.1, value=1, label="Temperature")
     ],
-    outputs=[gr.Textbox(lines=10, label="Output")],
-    title="Hugging Face Transformers Model",
-    description="A simple interface for using my Model",
-    concurrency_limit=1
+    outputs="text",
+    title="TinyLlama-1.1B Chat",
+    description="Generate text using TinyLlama-1.1B-Chat model with adjustable parameters."
 )
 
-g.launch(max_threads=2)
+if __name__ == "__main__":
+    interface.launch()