Upload 2 files

app.py

@@ -1,15 +1,16 @@
 import requests as rq
 import streamlit as st
-import os
+import transformers
+import torch
 
-API_URL = "https://api-inference.huggingface.co/models/meta-llama/Meta-Llama-3-8B"
-HF_LLAMA3_TOKEN = os.getenv("HF_LLAMA3_TOKEN")
-headers = {"Authorization": "Bearer " + HF_LLAMA3_TOKEN}
+model_id = "meta-llama/Meta-Llama-3-8B"
+pipeline = transformers.pipeline(
+    "text-generation", model=model_id, model_kwargs={"torch_dtype": torch.bfloat16}, device_map="auto"
+)
 
 
-def query(payload):
-    response = rq.post(API_URL, headers=headers, json=payload)
-    return response.json()
+def query(question):
+    return pipeline(question)
 
 
 st.title("LLama3 8B T")
@@ -17,7 +18,5 @@ st.subheader("This is a demo of the LLama3 8B T model.")
 
 user_input = st.text_area("Enter your text here:")
 if st.button("Submit"):
-    output = query({
-        "inputs": user_input,
-    })
-    st.write(output)
+    answer = query(user_input)
+    st.write(answer)

requirements.txt

@@ -1,2 +1,4 @@
 requests
-streamlit
+streamlit
+transformers
+torch