Multimodal-RAG / app.py
Tile's picture
first commit
74afc84
raw
history blame
14.7 kB
# import os
# os.system("pip uninstall -y gradio")
# os.system("pip install gradio==3.43.2")
import argparse
import sys
import os
# import cv2
import glob
import gradio as gr
import numpy as np
import json
from PIL import Image
from tqdm import tqdm
from pathlib import Path
import uvicorn
from fastapi.staticfiles import StaticFiles
import random
import time
import requests
from fastapi import FastAPI
from conversation import SeparatorStyle, conv_templates, default_conversation
from utils import (
build_logger,
moderation_msg,
server_error_msg,
)
from config import cur_conv
logger = build_logger("gradio_web_server", "gradio_web_server.log")
headers = {"Content-Type": "application/json"}
# create a FastAPI app
app = FastAPI()
# # create a static directory to store the static files
# static_dir = Path('/data/Multimodal-RAG/GenerativeAIExamples/ChatQnA/langchain/redis/chips-making-deals/')
static_dir = Path('/')
# mount FastAPI StaticFiles server
app.mount("/static", StaticFiles(directory=static_dir), name="static")
theme = gr.themes.Base(
primary_hue=gr.themes.Color(
c100="#dbeafe", c200="#bfdbfe", c300="#93c5fd", c400="#60a5fa", c50="#eff6ff", c500="#0054ae", c600="#00377c", c700="#00377c", c800="#1e40af", c900="#1e3a8a", c950="#0a0c2b"),
secondary_hue=gr.themes.Color(
c100="#dbeafe", c200="#bfdbfe", c300="#93c5fd", c400="#60a5fa", c50="#eff6ff", c500="#0054ae", c600="#0054ae", c700="#0054ae", c800="#1e40af", c900="#1e3a8a", c950="#1d3660"),
).set(
body_background_fill_dark='*primary_950',
body_text_color_dark='*neutral_300',
border_color_accent='*primary_700',
border_color_accent_dark='*neutral_800',
block_background_fill_dark='*primary_950',
block_border_width='2px',
block_border_width_dark='2px',
button_primary_background_fill_dark='*primary_500',
button_primary_border_color_dark='*primary_500',
)
css='''
@font-face {
font-family: IntelOne;
src: url("file/assets/intelone-bodytext-font-family-regular.ttf");
}
'''
## <td style="border-bottom:0"><img src="file/assets/DCAI_logo.png" height="300" width="300"></td>
html_title = '''
<table>
<tr style="height:150px">
<td style="border-bottom:0"><img src="file/assets/intel-labs.png" height="100" width="100"></td>
<td style="border-bottom:0; vertical-align:bottom">
<p style="font-size:xx-large;font-family:IntelOne, Georgia, sans-serif;color: white;">
Cognitive AI:
<br>
Multimodal RAG on Videos
</p>
</td>
<td style="border-bottom:0;"><img src="file/assets/gaudi.png" width="100" height="100"></td>
<td style="border-bottom:0;"><img src="file/assets/xeon.png" width="100" height="100"></td>
<td style="border-bottom:0;"><img src="file/assets/IDC7.png" width="400" height="350"></td>
</tr>
</table>
'''
# force dark theme
js_func = """
function refresh() {
const url = new URL(window.location);
if (url.searchParams.get('__theme') !== 'dark') {
url.searchParams.set('__theme', 'dark');
window.location.href = url.href;
}
}
"""
debug = False
def print_debug(t):
if debug:
print(t)
# https://stackoverflow.com/a/57781047
# Resizes a image and maintains aspect ratio
# def maintain_aspect_ratio_resize(image, width=None, height=None, inter=cv2.INTER_AREA):
# # Grab the image size and initialize dimensions
# dim = None
# (h, w) = image.shape[:2]
# # Return original image if no need to resize
# if width is None and height is None:
# return image
# # We are resizing height if width is none
# if width is None:
# # Calculate the ratio of the height and construct the dimensions
# r = height / float(h)
# dim = (int(w * r), height)
# # We are resizing width if height is none
# else:
# # Calculate the ratio of the width and construct the dimensions
# r = width / float(w)
# dim = (width, int(h * r))
# # Return the resized image
# return cv2.resize(image, dim, interpolation=inter)
def time_to_frame(time, fps):
'''
convert time in seconds into frame number
'''
return int(time * fps - 1)
def str2time(strtime):
strtime = strtime.strip('"')
hrs, mins, seconds = [float(c) for c in strtime.split(':')]
total_seconds = hrs * 60**2 + mins * 60 + seconds
return total_seconds
def get_iframe(video_path: str, start: int = -1, end: int = -1):
return f"""<video controls="controls" preload="metadata" src="{video_path}" width="540" height="310"></video>"""
#TODO
# def place(galleries, evt: gr.SelectData):
# print(evt.value)
# start_time = evt.value.split('||')[0].strip()
# print(start_time)
# # sub_video_id = evt.value.split('|')[-1]
# if start_time in start_time_index_map.keys():
# sub_video_id = start_time_index_map[start_time]
# else:
# sub_video_id = 0
# path_to_sub_video = f"/static/video_embeddings/mp4.keynotes23/sub-videos/keynotes23_split{sub_video_id}.mp4"
# # return evt.value
# return get_iframe(path_to_sub_video)
# def process(text_query):
# tmp_dir = os.environ.get('VID_CACHE_DIR', os.environ.get('TMPDIR', './video_embeddings'))
# frames, transcripts = run_query(text_query, path=tmp_dir)
# # return video_file_path, [(image, caption) for image, caption in zip(frame_paths, transcripts)]
# return [(frame, caption) for frame, caption in zip(frames, transcripts)], ""
description = "This Space lets you engage with multimodal RAG on a video through a chat box."
no_change_btn = gr.Button.update()
enable_btn = gr.Button.update(interactive=True)
disable_btn = gr.Button.update(interactive=False)
# textbox = gr.Textbox(
# show_label=False, placeholder="Enter text and press ENTER", container=False
# )
def clear_history(request: gr.Request):
logger.info(f"clear_history. ip: {request.client.host}")
state = cur_conv.copy()
return (state, state.to_gradio_chatbot(), "", None) + (disable_btn,) * 1
def add_text(state, text, request: gr.Request):
logger.info(f"add_text. ip: {request.client.host}. len: {len(text)}")
if len(text) <= 0 :
state.skip_next = True
return (state, state.to_gradio_chatbot(), "", None) + (no_change_btn,) * 1
text = text[:1536] # Hard cut-off
state.append_message(state.roles[0], text)
state.append_message(state.roles[1], None)
state.skip_next = False
return (state, state.to_gradio_chatbot(), "") + (disable_btn,) * 1
def http_bot(
state, request: gr.Request
):
logger.info(f"http_bot. ip: {request.client.host}")
start_tstamp = time.time()
if state.skip_next:
# This generate call is skipped due to invalid inputs
path_to_sub_videos = state.get_path_to_subvideos()
yield (state, state.to_gradio_chatbot(), path_to_sub_videos) + (no_change_btn,) * 1
return
if len(state.messages) == state.offset + 2:
# First round of conversation
new_state = cur_conv.copy()
new_state.append_message(new_state.roles[0], state.messages[-2][1])
new_state.append_message(new_state.roles[1], None)
state = new_state
# Construct prompt
prompt = state.get_prompt()
all_images = state.get_images(return_pil=False)
# Make requests
is_very_first_query = True
if len(all_images) == 0:
# first query need to do RAG
pload = {
"query": prompt,
}
else:
# subsequence queries, no need to do Retrieval
is_very_first_query = False
pload = {
"prompt": prompt,
"path-to-image": all_images[0],
}
if is_very_first_query:
url = worker_addr + "/v1/rag/chat"
else:
url = worker_addr + "/v1/rag/multi_turn_chat"
logger.info(f"==== request ====\n{pload}")
logger.info(f"==== url request ====\n{url}")
#uncomment this for testing UI only
# state.messages[-1][-1] = f"response {len(state.messages)}"
# yield (state, state.to_gradio_chatbot()) + (enable_btn,) * 1
# return
state.messages[-1][-1] = "▌"
path_to_sub_videos = state.get_path_to_subvideos()
yield (state, state.to_gradio_chatbot(), path_to_sub_videos) + (disable_btn,) * 1
try:
# Stream output
response = requests.post(url, headers=headers, json=pload, timeout=100, stream=True)
for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"):
if chunk:
res = json.loads(chunk.decode())
## old_method
# if response.status_code == 200:
# cur_json = ""
# for chunk in response:
# # print('chunk is ---> ', chunk.decode('utf-8'))
# cur_json += chunk.decode('utf-8')
# try:
# res = json.loads(cur_json)
# except:
# # a whole json does not include in this chunk, need to concatenate with next chunk
# continue
# # successfully load json into res
# cur_json = ""
if state.path_to_img is None and 'path-to-image' in res:
state.path_to_img = res['path-to-image']
if state.video_title is None and 'title' in res:
state.video_title = res['title']
if 'answer' in res:
# print(f"answer is {res['answer']}")
output = res["answer"]
# print(f"state.messages is {state.messages[-1][-1]}")
state.messages[-1][-1] = state.messages[-1][-1][:-1] + output + "▌"
path_to_sub_videos = state.get_path_to_subvideos()
yield (state, state.to_gradio_chatbot(), path_to_sub_videos) + (disable_btn,) * 1
time.sleep(0.03)
# else:
# raise requests.exceptions.RequestException()
except requests.exceptions.RequestException as e:
state.messages[-1][-1] = server_error_msg
yield (state, state.to_gradio_chatbot(), None) + (
enable_btn,
)
return
state.messages[-1][-1] = state.messages[-1][-1][:-1]
path_to_sub_videos = state.get_path_to_subvideos()
logger.info(path_to_sub_videos)
yield (state, state.to_gradio_chatbot(), path_to_sub_videos) + (enable_btn,) * 1
finish_tstamp = time.time()
logger.info(f"{state.messages[-1][-1]}")
# with open(get_conv_log_filename(), "a") as fout:
# data = {
# "tstamp": round(finish_tstamp, 4),
# "url": url,
# "start": round(start_tstamp, 4),
# "finish": round(start_tstamp, 4),
# "state": state.dict(),
# }
# fout.write(json.dumps(data) + "\n")
return
dropdown_list = [
"What did Intel present at Nasdaq?",
"From Chips Act Funding Announcement, by which year is Intel committed to Net Zero gas emissions?",
"What percentage of renewable energy is Intel planning to use?",
"a band playing music",
"Which US state is Silicon Desert referred to?",
"and which US state is Silicon Forest referred to?",
"How do trigate fins work?",
"What is the advantage of trigate over planar transistors?",
"What are key objectives of transistor design?",
"How fast can transistors switch?",
]
with gr.Blocks(theme=theme, css=css, js=js_func) as demo:
# gr.Markdown(description)
state = gr.State(default_conversation.copy())
gr.HTML(value=html_title)
with gr.Row():
with gr.Column(scale=4):
video = gr.Video(height=512, width=512, elem_id="video" )
with gr.Column(scale=7):
chatbot = gr.Chatbot(
elem_id="chatbot", label="Multimodal RAG Chatbot", height=450
)
with gr.Row():
with gr.Column(scale=8):
# textbox.render()
textbox = gr.Dropdown(
dropdown_list,
allow_custom_value=True,
# show_label=False,
# container=False,
label="Query",
info="Enter your query here or choose a sample from the dropdown list!"
)
with gr.Column(scale=1, min_width=50):
submit_btn = gr.Button(
value="Send", variant="primary", interactive=True
)
with gr.Row(elem_id="buttons") as button_row:
clear_btn = gr.Button(value="🗑️ Clear history", interactive=False)
# Register listeners
btn_list = [clear_btn]
clear_btn.click(
clear_history, None, [state, chatbot, textbox, video] + btn_list
)
# textbox.submit(
# add_text,
# [state, textbox],
# [state, chatbot, textbox,] + btn_list,
# ).then(
# http_bot,
# [state, ],
# [state, chatbot, video] + btn_list,
# )
submit_btn.click(
add_text,
[state, textbox],
[state, chatbot, textbox,] + btn_list,
).then(
http_bot,
[state, ],
[state, chatbot, video] + btn_list,
)
print_debug('Beginning')
# btn.click(fn=process,
# inputs=[text_query],
# # outputs=[video_player, gallery],
# outputs=[gallery, html],
# )
# gallery.select(place, [gallery], [html])
demo.queue()
app = gr.mount_gradio_app(app, demo, path='/')
share = False
enable_queue = True
# try:
# demo.queue(concurrency_count=3)#, enable_queue=False)
# demo.launch(enable_queue=enable_queue, share=share, server_port=17808, server_name='0.0.0.0')
# #BATCH -w isl-gpu48
# except:
# demo.launch(enable_queue=False, share=share, server_port=17808, server_name='0.0.0.0')
# serve the app
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--host", type=str, default="0.0.0.0")
parser.add_argument("--port", type=int, default=7860)
parser.add_argument("--concurrency-count", type=int, default=20)
parser.add_argument("--share", action="store_true")
parser.add_argument("--worker-address", type=str, default="198.175.88.247")
parser.add_argument("--worker-port", type=int, default=7899)
args = parser.parse_args()
logger.info(f"args: {args}")
global worker_addr
worker_addr = f"http://{args.worker_address}:{args.worker_port}"
uvicorn.run(app, host=args.host, port=args.port)
# for i in examples:
# print(f'Processing {i[0]}')
# results = process(*i)
# print(f'{len(results[0])} results returned')