Mikhil-jivus commited on
Commit
721cdc9
1 Parent(s): 559acc9

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +22 -15
app.py CHANGED
@@ -1,11 +1,14 @@
1
  import gradio as gr
2
- from huggingface_hub import InferenceClient
 
3
 
4
- """
5
- For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
6
- """
7
- client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
8
 
 
 
 
9
 
10
  def respond(
11
  message,
@@ -25,19 +28,24 @@ def respond(
25
 
26
  messages.append({"role": "user", "content": message})
27
 
28
- response = ""
 
 
29
 
30
- for message in client.chat_completion(
31
- messages,
32
- max_tokens=max_tokens,
33
- stream=True,
34
  temperature=temperature,
35
  top_p=top_p,
36
- ):
37
- token = message.choices[0].delta.content
 
38
 
39
- response += token
40
- yield response
 
 
41
 
42
  """
43
  For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
@@ -58,6 +66,5 @@ demo = gr.ChatInterface(
58
  ],
59
  )
60
 
61
-
62
  if __name__ == "__main__":
63
  demo.launch()
 
1
  import gradio as gr
2
+ from transformers import AutoTokenizer, AutoModelForCausalLM
3
+ import os
4
 
5
+ # Define the repository ID and access token
6
+ repo_id = "Mikhil-jivus/Llama-32-3B-FineTuned"
7
+ access_token = os.getenv('HF_TOKEN')
 
8
 
9
+ # Load the tokenizer and model from the Hugging Face repository
10
+ tokenizer = AutoTokenizer.from_pretrained(repo_id, use_auth_token=access_token)
11
+ model = AutoModelForCausalLM.from_pretrained(repo_id, use_auth_token=access_token)
12
 
13
  def respond(
14
  message,
 
28
 
29
  messages.append({"role": "user", "content": message})
30
 
31
+ # Tokenize the input messages
32
+ input_text = system_message + " ".join([f"{msg['role']}: {msg['content']}" for msg in messages])
33
+ input_ids = tokenizer.encode(input_text, return_tensors="pt")
34
 
35
+ # Generate a response
36
+ chat_history_ids = model.generate(
37
+ input_ids,
38
+ max_length=max_tokens,
39
  temperature=temperature,
40
  top_p=top_p,
41
+ pad_token_id=tokenizer.eos_token_id,
42
+ do_sample=True,
43
+ )
44
 
45
+ # Decode the response
46
+ response = tokenizer.decode(chat_history_ids[:, input_ids.shape[-1]:][0], skip_special_tokens=True)
47
+
48
+ yield response
49
 
50
  """
51
  For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
 
66
  ],
67
  )
68
 
 
69
  if __name__ == "__main__":
70
  demo.launch()