arad1367 commited on
Commit
2f43f91
β€’
1 Parent(s): f42f33d

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +62 -16
app.py CHANGED
@@ -33,14 +33,12 @@ processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-2B-Instruct", trust_rem
33
 
34
  @spaces.GPU()
35
  def process_pdf_and_query(pdf_file, user_query):
36
- # Convert the PDF to images
37
- images = convert_from_path(pdf_file.name) # pdf_file.name gives the file path
38
  num_images = len(images)
39
 
40
- # Indexing the PDF in RAG
41
  RAG.index(
42
  input_path=pdf_file.name,
43
- index_name="image_index", # index will be saved at index_root/index_name/
44
  store_collection_with_index=False,
45
  overwrite=True
46
  )
@@ -65,7 +63,7 @@ def process_pdf_and_query(pdf_file, user_query):
65
  }
66
  ]
67
 
68
- # Generate text with the Qwen model
69
  text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
70
  image_inputs, video_inputs = process_vision_info(messages)
71
  inputs = processor(
@@ -77,7 +75,6 @@ def process_pdf_and_query(pdf_file, user_query):
77
  )
78
  inputs = inputs.to("cuda")
79
 
80
- # Generate the output response
81
  generated_ids = model.generate(**inputs, max_new_tokens=50)
82
  generated_ids_trimmed = [
83
  out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
@@ -88,18 +85,67 @@ def process_pdf_and_query(pdf_file, user_query):
88
 
89
  return output_text[0], num_images
90
 
91
- # Define the Gradio Interface
92
- pdf_input = gr.File(label="Upload PDF") # Single PDF file input
93
- query_input = gr.Textbox(label="Enter your query", placeholder="Ask a question about the PDF") # User query input
94
- output_text = gr.Textbox(label="Model Answer") # Output for the model's answer
95
- output_images = gr.Textbox(label="Number of Images in PDF") # Output for number of images
96
 
97
- # Launch the Gradio app
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
98
  demo = gr.Interface(
99
  fn=process_pdf_and_query,
100
- inputs=[pdf_input, query_input], # List of inputs
101
- outputs=[output_text, output_images], # List of outputs
102
- title="Multimodal RAG with Image Query - By Pejman Ebrahimi"
 
 
 
 
103
  )
104
 
105
- demo.launch(debug=True) # Start the interface
 
 
 
 
 
33
 
34
  @spaces.GPU()
35
  def process_pdf_and_query(pdf_file, user_query):
36
+ images = convert_from_path(pdf_file.name)
 
37
  num_images = len(images)
38
 
 
39
  RAG.index(
40
  input_path=pdf_file.name,
41
+ index_name="image_index",
42
  store_collection_with_index=False,
43
  overwrite=True
44
  )
 
63
  }
64
  ]
65
 
66
+
67
  text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
68
  image_inputs, video_inputs = process_vision_info(messages)
69
  inputs = processor(
 
75
  )
76
  inputs = inputs.to("cuda")
77
 
 
78
  generated_ids = model.generate(**inputs, max_new_tokens=50)
79
  generated_ids_trimmed = [
80
  out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
 
85
 
86
  return output_text[0], num_images
87
 
 
 
 
 
 
88
 
89
+ css = """
90
+ .duplicate-button {
91
+ background-color: #6272a4;
92
+ color: white;
93
+ font-weight: bold;
94
+ border-radius: 5px;
95
+ margin-top: 20px;
96
+ padding: 10px;
97
+ text-align: center;
98
+ }
99
+ .gradio-container {
100
+ background-color: #282a36;
101
+ color: #f8f8f2;
102
+ font-family: 'Courier New', Courier, monospace;
103
+ padding: 20px;
104
+ border-radius: 10px;
105
+ box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
106
+ }
107
+ """
108
+
109
+ explanation = """
110
+ ### Multimodal RAG with Image Query
111
+ This demo showcases the **Multimodal RAG (Retriever-Augmented Generation)** model. The RAG system integrates retrieval and generation, allowing it to retrieve relevant information from a multimodal database (like PDFs with text and images) and then generate detailed responses.
112
+
113
+ We use **ColPali**, a state-of-the-art multimodal retriever, combined with the **Byaldi** library from **answer.ai**, which simplifies using ColPali. The language model used for generating answers is **Qwen/Qwen2-VL-2B-Instruct**, a powerful vision-language model capable of understanding both text and images.
114
+ """
115
+
116
+ footer = """
117
+ <div style="text-align: center; margin-top: 20px;">
118
+ <a href="https://www.linkedin.com/in/pejman-ebrahimi-4a60151a7/" target="_blank">LinkedIn</a> |
119
+ <a href="https://github.com/arad1367" target="_blank">GitHub</a> |
120
+ <a href="https://arad1367.pythonanywhere.com/" target="_blank">Live demo of my PhD defense</a> |
121
+ <a href="https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct" target="_blank">Qwen/Qwen2-VL-2B-Instruct</a> |
122
+ <a href="https://github.com/AnswerDotAI/byaldi" target="_blank">Byaldi</a> |
123
+ <a href="https://github.com/illuin-tech/colpali" target="_blank">ColPali</a>
124
+ <br>
125
+ Made with πŸ’– by Pejman Ebrahimi
126
+ </div>
127
+ """
128
+
129
+ pdf_input = gr.File(label="Upload PDF")
130
+ query_input = gr.Textbox(label="Enter your query", placeholder="Ask a question about the PDF")
131
+ output_text = gr.Textbox(label="Model Answer")
132
+ output_images = gr.Textbox(label="Number of Images in PDF")
133
+ duplicate_button = gr.DuplicateButton(value="Duplicate Space for private use", elem_classes="duplicate-button")
134
+
135
+ # Launch the Gradio app
136
  demo = gr.Interface(
137
  fn=process_pdf_and_query,
138
+ inputs=[pdf_input, query_input],
139
+ outputs=[output_text, output_images],
140
+ title="Multimodal RAG with Image Query - By Pejman Ebrahimi - Please like the space if it is useful",
141
+ theme='freddyaboulton/dracula_revamped',
142
+ css=css,
143
+ description=explanation,
144
+ allow_flagging="auto"
145
  )
146
 
147
+ with demo:
148
+ gr.HTML(footer)
149
+ duplicate_button
150
+
151
+ demo.launch(debug=True)