import os, time, threading import gradio as gr from display_model import * # default_llm = 'gpt-3.5-turbo-0125' default_llm = 'gpt-4-0613' # scan_id = "scene0132_00" scan_id_mapper={ 'Living Room':'scene0024_00', 'Bedroom':'scene0051_00', #144 'Kitchen':'scene0335_01', #164 197 'Bathroom':'scene0014_00', #14 26 'Office':'scene0114_00' # gym 428 } defualt_scene_type = 'Living Room' scan_id = scan_id_mapper[defualt_scene_type] def get_path(scan_id): ply_file = os.path.join("scenes", f"{scan_id}_vh_clean_2_aligned.ply") glb_file = os.path.join("scenes", f"{scan_id}_vh_clean_2_aligned.glb") new_ply_file = os.path.join("scenes", f"{scan_id}_vh_clean_2_aligned_AddBox.ply") new_glb_file = os.path.join("scenes", f"{scan_id}_vh_clean_2_aligned_AddBox.glb") objects_info_file = os.path.join("objects_info", f"objects_info_{scan_id}.npy") return {"ply_file":ply_file, "glb_file":glb_file, "new_ply_file":new_ply_file, "new_glb_file":new_glb_file, "objects_info_file":objects_info_file} def insert_user_none_between_assistant(messages): # 初始化结果列表 result = [] # 初始状态设置为"user"，以确保列表第一个条目为"assistant"时能正确插入 last_role = "user" for msg in messages: # 检查当前信息的角色 current_role = msg["role"] # 如果上一个和当前信息均为"assistant"，插入content为None的"user"信息 if last_role == "assistant" and current_role == "assistant": result.append({"role": "user", "content": None}) # 将当前信息添加到结果列表 result.append(msg) # 更新上一条信息的角色 last_role = current_role return result def timer_check_update(code_interpreter, update_interval, stop_event): """ 定时检查 code_interpreter.has_update 是否为True， 如果为True，则触发界面更新逻辑并重置状态。 参数： - code_interpreter: CodeInterpreter的实例，预期包含has_update属性。 - update_interval: 定时器检查间隔，以秒为单位。 - stop_event: 一个threading.Event()实例，用于停止定时器线程。 """ while not stop_event.is_set(): if code_interpreter.has_update: # 实现更新界面显示的逻辑 print("Detected update, trigger UI refreshing...") # 在这里添加更新界面显示的代码 # ... # 重置has_update状态 code_interpreter.has_update = False # 等待下次检查 time.sleep(update_interval) def process_instruction_callback(inp_api_key, instruction, llm_name): if not inp_api_key: print("Please input OpenAI API Key.") return else: os.environ["OPENAI_API_KEY"] = inp_api_key from transcrib3d_main import gen_prompt, get_gpt_response, get_openai_config from code_interpreter import CodeInterpreter print("llm_name:",llm_name) # generate prompt from user instruction # scan_id = "scene0132_00" prompt = gen_prompt(instruction, scan_id) # get oepnai config openai_config = get_openai_config(llm_name) openai_config['api_key'] = inp_api_key # get LLM response code_interpreter = CodeInterpreter(**openai_config) get_gpt_response(prompt, code_interpreter) messages = code_interpreter.pretext # draw the answer bounding box to the scene generate_answer_glb(messages[-1]['content']) # model3d.update(value=new_glb_file) # form gradio chat history messages = insert_user_none_between_assistant(messages[1:]) gradio_messages = [] for idx in range(int(len(messages)/2)): gradio_message = [messages[idx*2]['content'], messages[idx*2+1]['content']] gradio_messages.append(gradio_message) # return gradio_messages return get_path(scan_id)['new_glb_file'], gradio_messages def generate_answer_glb(answer_content): ply_file, glb_file, new_ply_file, new_glb_file, objects_info_file = get_path(scan_id).values() from transcrib3d_main import extract_answer_id_from_last_line last_line = answer_content.splitlines()[-1] if len(answer_content) > 0 else '' answer_id, _ = extract_answer_id_from_last_line(last_line) print("extracted answer id:", answer_id) # get the bounding box of the answer object box = np.load(objects_info_file, allow_pickle=True)[answer_id]['extension'] print("box extension:",box) # add the box to ply add_1box_to_ply(box, ply_file, new_ply_file) ply_to_glb(new_ply_file, new_glb_file) def llm_dropdown_callback(llm_name): print("type in callback:",type(llm_name)) llm_name = str(llm_name) print("llm_name in callback:",llm_name) return llm_name def scene_type_dropdown_callback(scene_type): # update scan_id as a global vairable global scan_id scan_id = scan_id_mapper[scene_type] print("scan_id inside scene_type_dropdown_callback:",scan_id) return get_path(scan_id)['glb_file'] with gr.Blocks() as demo: gr.Markdown("## Transcrib3D-Demo") with gr.Row(): with gr.Column(): # Left-1: Scene Type Selection scene_type_dropdown = gr.Dropdown( choices=['Living Room', 'Bedroom', 'Kitchen', 'Bathroom', 'Office'], label='Scene Type', type='value', value=defualt_scene_type ) scan_id_text = gr.Text(visible=False) # Left-2: 3D Scene model3d = gr.Model3D( value=f"scenes/{scan_id}_vh_clean_2_aligned.glb", # label=f"ScanNet {scan_id}", #how to update this label???? # label="3D Scene from ScanNet Dataset", label="Note: It takes ~10 seconds to load the 3D scene online. Please be patient.", camera_position=(90,120,8), zoom_speed=0.25, # height=635, height=725 ) # Left-3: Tips # gr.Markdown("🖱️:arrow_up::arrow_down:: SCROLL to zoom in/out.\t🖱️🔁 DRAG to rotate.\tCTRL+🖱️🔁 Press CTRL and DRAG to pan.") html_content = """
🖱️🔼🔽: SCROLL to zoom in/out.   🖱️🔁: DRAG to rotate.   [CTRL]+🖱️🔁: Press CTRL and DRAG to pan.
""" gr.HTML(value=html_content) with gr.Column(): # Right-1: Openai Api Key Input inp_api_key = gr.Textbox( label='OpenAI API Key (This will not stored anywhere and you could revoke it after use. Generate new key at [https://platform.openai.com/api-keys].)', placeholder='sk-xxxxxxxxx', lines=1) # Right-2: LLM Selection llm_dropdown = gr.Dropdown( # choices=['gpt-4-turbo','gpt-4','gpt-3.5-turbo'], choices=['gpt-4-0613', 'gpt-4-0125-preview', 'gpt-4-1106-preview', 'gpt-3.5-turbo-0125'], label="LLM Selection", type='value', value=default_llm ) llm_name_text = gr.Text(value=default_llm, visible=False) # Right-3: User Instruction Input user_instruction_textbox = gr.Textbox( label="Instruction", placeholder="Describe an object in the scene with its attributes and its relation with other objects, e.g. 'The plant between the armchair and the piano.'", # scale=4 ) # Right-4: Submit Button bt = gr.Button( value="Submit", # scale=1 ) # Right-5: Dialogue dialogue = gr.Chatbot( height=470 # value = [["1","2"], [None, '3']] ) # print("Type2:",type(model3d)) # 直接在 inputs列表里写model3d，会导致实际传给callback函数的是str # bt.click(fn=process_instruction_callback, inputs=user_instruction_textbox, outputs=dialogue) bt.click(fn=process_instruction_callback, inputs=[inp_api_key, user_instruction_textbox,llm_name_text], outputs=[model3d,dialogue]) user_instruction_textbox.submit(fn=process_instruction_callback, inputs=[inp_api_key, user_instruction_textbox, llm_name_text], outputs=[model3d,dialogue]) scene_type_dropdown.select(fn=scene_type_dropdown_callback, inputs=scene_type_dropdown, outputs=model3d) llm_dropdown.select(fn=llm_dropdown_callback, inputs=llm_dropdown, outputs=llm_name_text) # 直接用lambda函数定义一个映射 # type(user_instruction_textbox.value) # user_instruction_textbox. # user_instruction_textbox.submit(fn=lambda: process_instruction_callback(user_instruction_textbox, model3d), inputs=[], outputs=dialogue) # user_instruction_textbox. # bt.click(fn=lambda: process_instruction_callback(user_instruction_textbox, model3d), inputs=[], outputs=dialogue) # os.system('uname -a') # 显示所有系统信息 # demo.launch() if __name__ == "__main__": # demo.launch(share=True, server_name="0.0.0.0", server_port=7860) demo.launch(server_name="0.0.0.0", server_port=7860)