Add concurrency

app.py

@@ -73,14 +73,14 @@ def generate_for_prompt(input_text, state, ret_scale_factor, max_nm_rets, num_wo
         elif type(output) == list:
             for image in output:
                 filename = save_image_to_local(image)
-                response += f'<img src="/file={filename}">'
+                response += f'<br/><img src="/file={filename}">'
         elif type(output) == Image.Image:
-                filename = save_image_to_local(output)
-                response += f'<img src="/file={filename}">'
+            filename = save_image_to_local(output)
+            response += f'<br/><img src="/file={filename}">'
 
     # TODO(jykoh): Persist image inputs.
     chat_history = model_inputs + [' '.join([s for s in model_outputs if type(s) == str]) + '\n']
-    conversation.append((input_text, response))
+    conversation.append((input_text, response.replace('[RET]', '')))  # Remove [RET] from outputs.
 
     # Set input image to None.
     print('state', state, flush=True)
@@ -98,7 +98,7 @@ with gr.Blocks() as demo:
 
     with gr.Row():
         with gr.Column(scale=0.3, min_width=0):
-            ret_scale_factor = gr.Slider(minimum=0.0, maximum=3.0, value=1.0, step=0.1, interactive=True, label="Multiplier for returning images (higher means more frequent)")
+            ret_scale_factor = gr.Slider(minimum=0.0, maximum=3.0, value=1.3, step=0.1, interactive=True, label="Multiplier for returning images (higher means more frequent)")
             max_ret_images = gr.Number(minimum=0, maximum=3, value=1, precision=1, interactive=True, label="Max images to return")
             gr_max_len = gr.Number(value=32, precision=1, label="Max # of words returned", interactive=True)
             gr_temperature = gr.Number(value=0.0, label="Temperature", interactive=True)
@@ -113,4 +113,6 @@ with gr.Blocks() as demo:
     image_btn.upload(upload_image, [gr_state, image_btn], [gr_state, chatbot])
     clear_btn.click(reset, [], [gr_state, chatbot])
 
-demo.launch(share=False, debug=True, server_name="0.0.0.0")
+# demo.launch(share=False, debug=True, server_name="0.0.0.0")
+demo.queue(concurrency_count=5)
+demo.launch(debug=True)

fromage/models.py

@@ -634,8 +634,6 @@ def load_fromage(embeddings_dir: str, model_args_path: str, model_ckpt_path: str
   ret_token_idx = tokenizer('[RET]', add_special_tokens=False).input_ids
   assert len(ret_token_idx) == 1, ret_token_idx
   model_kwargs['retrieval_token_idx'] = ret_token_idx[0]
-  # model_kwargs['opt_version'] = 'facebook/opt-125m'
-  # model_kwargs['visual_encoder'] = 'openai/clip-vit-base-patch32'
   args = namedtuple('args', model_kwargs)(**model_kwargs)
 
   # Initialize model for inference.