# refer to repo https://github.com/gradio-app/gradio/blob/main/demo/chatbot_multimodal/run.ipynb for enhancement import PIL.Image import gradio as gr import base64 import time import os import google.generativeai as genai import requests import pathlib txt_model = genai.GenerativeModel('gemini-pro') vis_model = genai.GenerativeModel('gemini-pro-vision') txt_prompt_1 = """ 请列出图片中的所有食物项目,每个项目应在 Markdown 格式的食物-营养成分表中占据一行,表格包括四列:序号、食物、注意营养成分和推荐摄入量。其中,“主要营养成分”指的是食物中的主要成分,而“推荐摄入量”则是该成分的建议每日或每周摄入量。 ** 示例食物-营养成分表 | 序号 | 食物 | 主要营养成分 | 推荐摄入量 | |-----|--------------|------------------------------------------------------------------------------------------|--------------------------------------| """ txt_display_1 = 'content of the letter: ' txt_prompt_selfie = """ 以照片中的主体人物与背景做为对象,分别对人物与背景分别做出评语,对人物做出3个评语,对背景也做出3个评语。这些评语必须是轻松诙谐的用词与内容。 ** 人物评语 | 序号 | 特征 | 评语 |-----|--------------|----------------------------------------------------------------------------- ** 背景评语 | 序号 | 特征 | 评语 |-----|--------------|----------------------------------------------------------------------------- """ txt_display_selfie = '--- ' import os GOOGLE_API_KEY=os.getenv('GOOGLE_API_KEY') genai.configure(api_key=GOOGLE_API_KEY) SMS_URL =os.getenv('SMS_URL') SMS_TOK =os.getenv('SMS_TOK') sms_text ="..." # Image to Base 64 Converter def image_to_base64(image_path): with open(image_path, 'rb') as img: encoded_string = base64.b64encode(img.read()) return encoded_string.decode('utf-8') # Function that takes User Inputs, generates Response and displays on Chat UI def selfie_response(img): if not img: response = txt_model.generate_content(txt_prompt_selfie) return response else: img = PIL.Image.open(img) response = vis_model.generate_content([txt_prompt_selfie,img]) return response.text # Function that takes User Inputs, generates Response and displays on Chat UI def app1_response(img): if not img: response = txt_model.generate_content(txt_prompt_1) return response else: img = PIL.Image.open(img) response = vis_model.generate_content([txt_prompt_1,img]) return response.text # SMS service ends in March 2024, to restore service @Sinch Simple Text def send_SMS(resp_text): url = SMS_URL headers = { "Authorization": SMS_TOK, "Content-Type": "application/json" } data = { "from": "12085686834", "to": ["18587331029"], "body": resp_text } response = requests.post(url, json=data, headers=headers) return response.text return "....." # gradio block with gr.Blocks() as app1: with gr.Column(): gr.Markdown("## 🥷 点评餐盘 ##") gr.Markdown("```for xxxxxx ✉ and/or xxx ⌦, include photo here...```") image_box = gr.Image(label="✂ 对象:餐盘", type="filepath") btn1 = gr.Button("点评这盘菜 ☑") out1 = gr.Textbox(label="here are the actionables...") btn2 = gr.Button("(disabled)发短信 ↗️") out2 = gr.Textbox(label="(disabled) no message been sent, this is just a mockup confirmation...") btn1.click(fn=app1_response, inputs=[image_box], outputs=out1) btn2.click(fn=send_SMS, inputs=out1, outputs=out2) gr.Markdown(""" # 🥷 Summerize eMail & Make a Plan # [demo video](https://youtu.be/lJ4jIAEVRNY) """) examples=[ os.path.join(os.path.dirname(__file__), "images/missionemail.png"), ], with gr.Blocks() as app_selfie: with gr.Column(): gr.Markdown("## ↗️ 点评自拍(请站在海报墙前面拍) ##") gr.Markdown("```xxxx ⌦, paste screenshot here...```") image_box = gr.Image(label="✂ 乐意保墙,与LOGO合拍", type="filepath") btn_selfie1 = gr.Button("生成趣味点评 ☑") out_selfife1 = gr.Textbox(label="here are the actionables...") btn_selfie2 = gr.Button("(disabled)发短信 ↗️") out_selfife2 = gr.Textbox(label="(disabled) no message been sent, this is just a mockup confirmation...") btn_selfie1.click(fn=selfie_response, inputs=[image_box], outputs=out_selfife1) btn_selfie2.click(fn=send_SMS, inputs=out_selfife1, outputs=out_selfife2) gr.Markdown(""" # ↗️ 与logo合拍 # - screen capture (Win + shift + S) - use MD editor below [markdown editor](https://stackedit.io/app#) """) examples=[ os.path.join(os.path.dirname(__file__), "images/missionemail.png"), ], with gr.Blocks() as demo: gr.Markdown("## 自拍 ↗️ + 餐盘 🥷 ##") gr.TabbedInterface([app_selfie, app1], ["➀ 点评我的自拍", "➁ 点评我的餐盘"]) demo.queue() demo.launch()