ManishThota commited on
Commit
ea0653d
1 Parent(s): 3e161a0

Create app_main.py

Browse files
Files changed (1) hide show
  1. app_main.py +63 -0
app_main.py ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ from PIL import Image
3
+ import torch
4
+ from transformers import AutoModelForCausalLM, AutoTokenizer
5
+
6
+
7
+ # # Ensure GPU usage if available
8
+ device = "cuda" if torch.cuda.is_available() else "cpu"
9
+
10
+
11
+
12
+ # Initialize the model and tokenizer
13
+ model = AutoModelForCausalLM.from_pretrained("ManishThota/SparrowVQE",
14
+ torch_dtype=torch.float16,
15
+ device_map="auto",
16
+ trust_remote_code=True)
17
+ tokenizer = AutoTokenizer.from_pretrained("ManishThota/SparrowVQE", trust_remote_code=True)
18
+
19
+ def predict_answer(image, question, max_tokens=100):
20
+ #Set inputs
21
+ text = f"A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: <image>\n{question}? ASSISTANT:"
22
+ image = image.convert("RGB")
23
+
24
+ input_ids = tokenizer(text, return_tensors='pt').input_ids.to(device)
25
+ image_tensor = model.image_preprocess(image)
26
+
27
+ #Generate the answer
28
+ output_ids = model.generate(
29
+ input_ids,
30
+ max_new_tokens=max_tokens,
31
+ images=image_tensor,
32
+ use_cache=True)[0]
33
+
34
+ return tokenizer.decode(output_ids[input_ids.shape[1]:], skip_special_tokens=True).strip()
35
+
36
+ def gradio_predict(image, question, max_tokens):
37
+ answer = predict_answer(image, question, max_tokens)
38
+ return answer
39
+
40
+
41
+ # examples = [["data/week_01_page_024.png", 'Can you explain the slide?',100],
42
+ # ["data/week_03_page_091.png", 'Can you explain the slide?',100],
43
+ # ["data/week_01_page_062.png", 'Are the training images labeled?',100],
44
+ # ["data/week_05_page_027.png", 'What is meant by eigenvalue multiplicity?',100],
45
+ # ["data/week_05_page_030.png", 'What does K represent?',100],
46
+ # ["data/week_15_page_046.png", 'How are individual heterogeneous models trained?',100],
47
+ # ["data/week_15_page_021.png", 'How does Bagging affect error?',100],
48
+ # ["data/week_15_page_037.png", "What does the '+' and '-' represent?",100]]
49
+
50
+ # Define the Gradio interface
51
+ iface = gr.Interface(
52
+ fn=gradio_predict,
53
+ inputs=[gr.Image(type="pil", label="Upload or Drag an Image"),
54
+ gr.Textbox(label="Question", placeholder="e.g. Can you explain the slide?", scale=4),
55
+ gr.Slider(2, 500, value=100, label="Token Count", info="Choose between 2 and 500")],
56
+ outputs=gr.TextArea(label="Answer"),
57
+ # examples=examples,
58
+ title="Sparrow - Tiny 3B | Visual Question Answering",
59
+ description="An interactive chat model that can answer questions about images in an Academic context. \n We can input images, and the system will analyze them to provide information about their contents. I've utilized this capability by feeding slides from PowerPoint presentations used in classes and the lecture content passed as text. Consequently, the model now mimics the behavior and responses of my professors. So, if I present any PowerPoint slide, it explains it just like my professor would, further it can be personalized.",
60
+ )
61
+
62
+ # Launch the app
63
+ iface.queue().launch(debug=True)