trying new things out again

app.py

@@ -8,7 +8,7 @@ import requests
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
+client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct")
 
 
 
@@ -34,19 +34,19 @@ def respond(
 
     messages.append({"role": "user", "content": message})
 
-    #response = ""
+    response = ""
 
-    # for message in client.chat_completion(
-    #     messages,
-    #     max_tokens=max_tokens,
-    #     stream=True,
-    #     temperature=temperature,
-    #     top_p=top_p,
-    # ):
-    #     token = message.choices[0].delta.content
+    for message in client.chat_completion(
+        messages,
+        max_tokens=max_tokens,
+        stream=True,
+        temperature=temperature,
+        top_p=top_p,
+    ):
+        token = message.choices[0].delta.content
 
-    #     response += token
-    #     yield response
+        response += token
+        yield response
 
     ### doesn't work
     # input_ids = tokenizer.encode(message, return_tensors = 'pt')
@@ -54,14 +54,14 @@ def respond(
     #     output_text = tokenizer.decode(output, skip_special_tokens=True)
     #     yield output_text
     
-    API_URL = "https://api-inference.huggingface.co/models/meta-llama/Meta-Llama-3-8B-Instruct"
-    headers = {"Authorization": "Bearer "+os.environ['hf_token']}
-    response = requests.post(API_URL, headers=headers, json={"inputs":message})
-    data  = response.json()
-    returnval = ""
-    for item in data:
-        returnval = returnval + item['generated_text']
-    return returnval
+    # API_URL = "https://api-inference.huggingface.co/models/meta-llama/Meta-Llama-3-8B-Instruct"
+    # headers = {"Authorization": "Bearer "+os.environ['hf_token']}
+    # response = requests.post(API_URL, headers=headers, json={"inputs":message})
+    # data  = response.json()
+    # returnval = ""
+    # for item in data:
+    #     returnval = returnval + item['generated_text']
+    #     yield returnval