Upload folder using huggingface_hub

.gitignore

@@ -0,0 +1,2 @@
+flagged
+*.log

.vscode/launch.json

@@ -0,0 +1,17 @@
+{
+    // 使用 IntelliSense 了解相关属性。 
+    // 悬停以查看现有属性的描述。
+    // 欲了解更多信息，请访问: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: 当前文件",
+            "type": "python",
+            "request": "launch",
+            "program": "test_hd.py",
+            "console": "integratedTerminal",
+            "python" :"/home/hadoop-automl/tianrunhe/anaconda3/envs/gradio_env_3.9/bin/python"
+            // "python":"/home/hadoop-automl/tianrunhe/anaconda3/envs/gradio/bin/python"
+        }
+    ]
+}

README.md

@@ -1,12 +1,6 @@
 ---
-title: Gradio Demo
-emoji: ⚡
-colorFrom: indigo
-colorTo: indigo
+title: gradio_demo
+app_file: try_grpc.py
 sdk: gradio
-sdk_version: 4.31.5
-app_file: app.py
-pinned: false
+sdk_version: 4.31.4
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

gradio_cached_examples/15/log.csv

@@ -0,0 +1,2 @@
+output,flag,username,timestamp
+,,,2024-05-21 21:35:52.230145

main.py

@@ -0,0 +1,83 @@
+import gradio as gr
+from TritonServerClient import TritonServerClient, InferInput, InferRequestedOutput
+from TritonServerClient.utils import np_to_triton_dtype
+from functools import wraps
+import numpy as np
+
+def prepare_params(query):
+   bs=1
+   # query = "#角色：直播间文案写作专家。#任务：根据用户输入的商品信息写出用于推广营销的直播文案。#目标：引导用户购买直播间商品，提高销量。#要求：- 如果没有输入商品信息，则邀请输入。- 文案写作 - 按照“引出话题”、“产品讲解”、“购买方式”三个方面写作文案。 - 在“产品讲解”部分详细讲解产品细节： - 【重要】充分扩充内容，吸引用户购买。 - 详细描述商品的优惠活动信息（如用户输入没有，不要询问，直接补充）。 - 在“购买方式”部分： - 仅包括引导用户点击链接购买的内容。 - 不要包括快递物流承诺。 - 【重要】每个部分多于5句话。 - 使用口语化的文字。 - 表达方式有亲和力和感染力。 - 对用户们使用“宝子们”、“宝宝们”、“家人们”等有亲和力的称呼。 - 多使用自问自答的语句。- 当你准备好了回复“请输入商品描述和优惠信息”我们就开始。#输出：- 包含在""里。- 写出每一部分内容的标题，包括【引出话题】、【产品讲解】、【购买方式】。- 示例：""【引出话题】【产品讲解】【购买方式】""- 输入商品信息:%s" % prompt
+   # query = '请你扮演直播间直播卖货的主播角色，根据用户提出的问题，检索知识库中的信息，如果查找到相关的信息，使用口语化的文字，用非常简短的一句话回答用户的问题；如果查找不到相关的信息，直接回答没有，并引导用户关注本场直播商品。你的目的是激发用户的购买欲、提高直播商品销量。回答对用户们使用“宝子”、“宝宝”、“亲”等有亲和力的称呼。\n\n# 用户提出的问题：\n这里面有可乐吗\n\n# 知识库：\n商品名称：外送欢乐桶(券)兑换券\n直播链接序号：64号链接号链接\n商品现价：66元\n商品原价：142元\n折扣：5\n品牌名称：麦当劳麦乐送\n券包张数：1\n套餐详情：["麦辣鸡腿汉堡1个","板烧鸡腿堡1个","中可乐3杯","四拼小食桶A"]\n补充：四拼小食桶A(麦麦脆汁鸡(鸡腿)1块＋中薯条1份＋麦辣鸡翅2块＋麦乐鸡5块)＋麦辣鸡腿堡1个＋板烧鸡腿堡1个＋中可乐3杯 原材料：面包\n\n\n# 输出格式：\n答：\n\n# 要求\n对于用户问到了知识库中未提及的信息不要编造，直接不回答'
+
+   title_text =  np.array([query.encode('utf-8')], dtype=np.string_)
+   title_text = np.tile(title_text, (bs, 1))
+
+   data_batch = {}
+   data_batch['query'] = title_text
+
+   inputs = [
+            InferInput("query", data_batch['query'].shape,
+                                np_to_triton_dtype(data_batch['query'].dtype)),
+   ]
+
+   inputs[0].set_data_from_numpy(data_batch['query'])
+
+   return inputs
+
+
+def make_a_try(inputs, outputs='response', model_name='qwen', model_version='1'):
+    outputs_list = []
+    ori_outputs_list = outputs.strip().split(",")
+    for out_ele in ori_outputs_list:
+        outputs_list.append(out_ele.strip())
+    outputs = [InferRequestedOutput(x) for x in outputs_list]
+
+    response = my_client.predict(model_name=model_name, inputs=inputs, model_version=model_version, outputs=outputs)
+
+    rsp_info = {}
+    if outputs_list == []:
+        for out_name_ele in response._result.outputs:
+            outputs_list.append(out_name_ele.name)
+    for output_name in outputs_list:
+        res = response.as_numpy(output_name)
+        response = np.expand_dims(res, axis=0)
+        response = response[0].decode('utf-8')
+        rsp_info[output_name] = response
+   
+    return rsp_info['response']
+
+
+
+def greet(prompt):
+    """Greet someone."""
+    # print(prompt)
+    inputs = prepare_params(prompt)
+    result = make_a_try(inputs)
+
+    return result
+
+
+if __name__ == "__main__":
+    param_info = {}
+    param_info['appkey'] = "com.sankuai.automl.serving"
+    # param_info['appkey'] = "com.sankuai.automl.streamvlm"
+
+    param_info['remote_appkey'] = "com.sankuai.automl.chat3"
+    param_info['model_name'] = "qwen"
+    param_info['model_version'] = "1"
+    param_info['time_out'] = 60000
+    param_info['server_targets'] = []
+    param_info['outputs'] = 'response'
+
+    appkey, remote_appkey, model_name, model_version, time_out, server_targets =  param_info['appkey'], param_info['remote_appkey'], param_info['model_name'], param_info['model_version'], param_info['time_out'], param_info['server_targets']
+
+    # my_client = TritonServerClient(appkey=appkey, remote_appkey=remote_appkey, time_out=time_out, server_targets=server_targets)
+
+   # 以上部分，用户可以修改代码
+    demo = gr.Interface(
+        fn=greet,
+        inputs=["textbox"],
+        outputs=["textbox"],
+    )
+
+    demo.launch(server_name="0.0.0.0", server_port=8088, debug=True, share=True)

multi_main.py

@@ -0,0 +1,105 @@
+import gradio as gr
+from TritonServerClient import TritonServerClient, InferInput, InferRequestedOutput
+from TritonServerClient.utils import np_to_triton_dtype
+from functools import wraps
+import numpy as np
+
+
+def prepare_params(query):
+   bs=1
+   # query = "#角色：直播间文案写作专家。#任务：根据用户输入的商品信息写出用于推广营销的直播文案。#目标：引导用户购买直播间商品，提高销量。#要求：- 如果没有输入商品信息，则邀请输入。- 文案写作 - 按照“引出话题”、“产品讲解”、“购买方式”三个方面写作文案。 - 在“产品讲解”部分详细讲解产品细节： - 【重要】充分扩充内容，吸引用户购买。 - 详细描述商品的优惠活动信息（如用户输入没有，不要询问，直接补充）。 - 在“购买方式”部分： - 仅包括引导用户点击链接购买的内容。 - 不要包括快递物流承诺。 - 【重要】每个部分多于5句话。 - 使用口语化的文字。 - 表达方式有亲和力和感染力。 - 对用户们使用“宝子们”、“宝宝们”、“家人们”等有亲和力的称呼。 - 多使用自问自答的语句。- 当你准备好了回复“请输入商品描述和优惠信息”我们就开始。#输出：- 包含在""里。- 写出每一部分内容的标题，包括【引出话题】、【产品讲解】、【购买方式】。- 示例：""【引出话题】【产品讲解】【购买方式】""- 输入商品信息:%s" % prompt
+   # query = '请你扮演直播间直播卖货的主播角色，根据用户提出的问题，检索知识库中的信息，如果查找到相关的信息，使用口语化的文字，用非常简短的一句话回答用户的问题；如果查找不到相关的信息，直接回答没有，并引导用户关注本场直播商品。你的目的是激发用户的购买欲、提高直播商品销量。回答对用户们使用“宝子”、“宝宝”、“亲”等有亲和力的称呼。\n\n# 用户提出的问题：\n这里面有可乐吗\n\n# 知识库：\n商品名称：外送欢乐桶(券)兑换券\n直播链接序号：64号链接号链接\n商品现价：66元\n商品原价：142元\n折扣：5\n品牌名称：麦当劳麦乐送\n券包张数：1\n套餐详情：["麦辣鸡腿汉堡1个","板烧鸡腿堡1个","中可乐3杯","四拼小食桶A"]\n补充：四拼小食桶A(麦麦脆汁鸡(鸡腿)1块＋中薯条1份＋麦辣鸡翅2块＋麦乐鸡5块)＋麦辣鸡腿堡1个＋板烧鸡腿堡1个＋中可乐3杯 原材料：面包\n\n\n# 输出格式：\n答：\n\n# 要求\n对于用户问到了知识库中未提及的信息不要编造，直接不回答'
+
+   title_text =  np.array([query.encode('utf-8')], dtype=np.string_)
+   title_text = np.tile(title_text, (bs, 1))
+
+   data_batch = {}
+   data_batch['query'] = title_text
+
+   inputs = [
+            InferInput("query", data_batch['query'].shape,
+                                np_to_triton_dtype(data_batch['query'].dtype)),
+   ]
+
+   inputs[0].set_data_from_numpy(data_batch['query'])
+
+   return inputs
+
+
+def make_a_try(inputs, outputs='response', model_name='qwen', model_version='1'):
+    outputs_list = []
+    ori_outputs_list = outputs.strip().split(",")
+    for out_ele in ori_outputs_list:
+        outputs_list.append(out_ele.strip())
+    outputs = [InferRequestedOutput(x) for x in outputs_list]
+
+    response = my_client.predict(model_name=model_name, inputs=inputs, model_version=model_version, outputs=outputs)
+
+    rsp_info = {}
+    if outputs_list == []:
+        for out_name_ele in response._result.outputs:
+            outputs_list.append(out_name_ele.name)
+    for output_name in outputs_list:
+        res = response.as_numpy(output_name)
+        response = np.expand_dims(res, axis=0)
+        response = response[0].decode('utf-8')
+        rsp_info[output_name] = response
+    print("response:",rsp_info)
+    return rsp_info['response']
+
+
+
+def greet(prompt):
+    """Greet someone."""
+    # print(prompt)
+    print("prompt:",prompt)
+    inputs = prepare_params(prompt)
+    print(inputs)
+    result = make_a_try(inputs)
+
+    return result
+
+def clear_input():
+
+    return ""
+
+if __name__ == "__main__":
+    param_info = {}
+    # param_info['appkey'] = "com.sankuai.automl.serving"
+    param_info['appkey'] = "com.sankuai.automl.streamvlm"
+
+    param_info['remote_appkey'] = "com.sankuai.automl.chat3"
+    param_info['model_name'] = "qwen"
+    param_info['model_version'] = "1"
+    param_info['time_out'] = 60000
+    param_info['server_targets'] = []
+    param_info['outputs'] = 'response'
+
+    appkey, remote_appkey, model_name, model_version, time_out, server_targets =  param_info['appkey'], param_info['remote_appkey'], param_info['model_name'], param_info['model_version'], param_info['time_out'], param_info['server_targets']
+
+    my_client = TritonServerClient(appkey=appkey, remote_appkey=remote_appkey, time_out=time_out, server_targets=server_targets)
+
+#    # 以上部分，用户可以��改代码
+#     demo = gr.Interface(
+#         fn=greet,
+#         inputs=["textbox"],
+#         outputs=["textbox"],
+#     )
+    with gr.Blocks(title='demo') as  demo:
+        with gr.Row():
+            with gr.Column():
+                promptbox = gr.Textbox(label = "prompt")
+            
+            with gr.Column():
+                output = gr.Textbox(label = "output")
+        with gr.Row():
+            submit = gr.Button("submit")
+            clear = gr.Button("clear")
+
+        submit.click(fn=greet,inputs=[promptbox],outputs=[output])
+        clear.click(fn=clear_input, inputs=[], outputs=[output])
+
+    # demo.launch(server_name="0.0.0.0", server_port=8088, debug=True, share=True)
+    # demo.launch(server_name="0.0.0.0", server_port=8080, debug=True, share=True)
+
+#http://10.99.5.48:8080/ 

requirements.txt

@@ -0,0 +1,78 @@
+aiofiles==23.2.1
+aiohttp==3.8.6
+aiosignal==1.3.1
+altair==5.0.1
+annotated-types==0.5.0
+anyio==3.7.1
+async-timeout==4.0.3
+asynctest==0.13.0
+attrs==23.2.0
+certifi @ file:///croot/certifi_1671487769961/work/certifi
+charset-normalizer==3.3.2
+click==8.1.7
+cycler==0.11.0
+exceptiongroup==1.2.1
+fastapi==0.99.1
+ffmpy==0.3.2
+filelock==3.12.2
+fonttools==4.38.0
+frozenlist==1.3.3
+fsspec==2023.1.0
+gradio==3.34.0
+gradio_client==0.2.6
+grpcio==1.62.2
+h11==0.14.0
+httpcore==0.17.3
+httpx==0.24.1
+huggingface-hub==0.16.4
+idna==3.7
+importlib-metadata==6.7.0
+importlib-resources==5.12.0
+Jinja2==3.1.3
+jsonschema==4.17.3
+kiwisolver==1.4.5
+linkify-it-py==2.0.3
+markdown-it-py==2.2.0
+MarkupSafe==2.1.5
+matplotlib==3.5.3
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+multidict==6.0.5
+numpy==1.21.6
+octo-rpc==0.4.7
+orjson==3.9.7
+packaging==24.0
+pandas==1.3.5
+Pillow==9.5.0
+pkgutil_resolve_name==1.3.10
+ply==3.11
+protobuf==3.20.1
+psutil==5.9.8
+pydantic==1.10.11
+pydantic_core==2.14.6
+pydub==0.25.1
+Pygments==2.17.2
+pyparsing==3.1.2
+pyrsistent==0.19.3
+python-cat==0.0.11
+python-dateutil==2.9.0.post0
+python-multipart==0.0.8
+pytz==2024.1
+PyYAML==6.0.1
+requests==2.31.0
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.1
+starlette==0.27.0
+thrift==0.20.0
+thriftpy2==0.4.20
+toolz==0.12.1
+tqdm==4.66.2
+TritonServerClient==0.0.7
+typing_extensions==4.7.1
+uc-micro-py==1.0.3
+urllib3==2.0.7
+uvicorn==0.22.0
+websockets==11.0.3
+yarl==1.9.4
+zipp==3.15.0

run.sh

@@ -0,0 +1,9 @@
+source /workdir/yanghandi/gradio_demo/setup.sh
+# source setup.sh
+python main.py &> gradio.log &
+
+python try_hd_v2.py &> multi_gradio.log &
+
+python try_hd_v2.py &> try_hd_v2.log &
+
+python test_hd.py &> test_hd.log &

setup.sh

@@ -0,0 +1,2 @@
+conda deactivate
+conda activate gradio

test_hd.py

@@ -0,0 +1,187 @@
+import argparse
+import queue
+import sys
+import uuid
+from functools import partial
+
+import numpy as np
+import tritonclient.grpc as grpcclient
+from tritonclient.utils import InferenceServerException
+import gradio as gr
+from functools import wraps
+
+####
+from PIL import Image
+import base64
+import io
+#####
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+import socket
+####
+import os
+import uuid
+####
+
+class UserData:
+    def __init__(self):
+        self._completed_requests = queue.Queue()
+
+def callback(user_data, result, error):
+    if error:
+        user_data._completed_requests.put(error)
+    else:
+        user_data._completed_requests.put(result)
+
+def make_a_try(img_url,text):
+    model_name = 'ensemble_mllm'
+    user_data = UserData()
+    sequence_id = 100
+    int_sequence_id0 = sequence_id
+    result_list=[]
+    try:
+        triton_client = grpcclient.InferenceServerClient(
+            url="10.95.163.43:8001",
+            # verbose=FLAGS.verbose,
+            verbose = True, #False
+            ssl=False,
+            root_certificates=None,
+            private_key=None,
+            certificate_chain=None,
+        )
+    except Exception as e:
+        print("channel creation failed: " + str(e))
+        return ""
+    # Infer
+    inputs = []
+    img_url_bytes = img_url.encode("utf-8")
+    img_url_bytes = np.array(img_url_bytes, dtype=bytes)
+    img_url_bytes = img_url_bytes.reshape([1, -1])
+
+    inputs.append(grpcclient.InferInput('IMAGE_URL', img_url_bytes.shape, "BYTES"))
+    inputs[0].set_data_from_numpy(img_url_bytes)
+
+    text_bytes = text.encode("utf-8")
+    text_bytes = np.array(text_bytes, dtype=bytes)
+    text_bytes = text_bytes.reshape([1, -1])
+    # text_input = np.expand_dims(text_bytes, axis=0)
+    text_input = text_bytes
+
+    inputs.append(grpcclient.InferInput('TEXT', text_input.shape, "BYTES"))
+    inputs[1].set_data_from_numpy(text_input)
+
+    outputs = []
+    outputs.append(grpcclient.InferRequestedOutput("OUTPUT"))
+    # Test with outputs
+    results = triton_client.infer(
+        model_name=model_name,
+        inputs=inputs,
+        outputs=outputs,
+        client_timeout=None, #FLAGS.client_timeout,
+        # headers={"test": "1"},
+        compression_algorithm=None, #FLAGS.grpc_compression_algorithm,
+    )
+
+    statistics = triton_client.get_inference_statistics(model_name=model_name)
+    print(statistics)
+    if len(statistics.model_stats) != 1:
+        print("FAILED: Inference Statistics")
+        return ""
+
+    # Get the output arrays from the results
+    output_data = results.as_numpy("OUTPUT")
+    result_str = output_data[0][0].decode('utf-8')
+
+    print("OUTPUT: "+ result_str)
+    return result_str
+
+def greet(image, text):
+    ###save img
+    static_path = f"/workdir/yanghandi/gradio_demo/static"
+    # 将图片转换为字节流
+    img_byte_arr = io.BytesIO()
+    try:
+        image.save(img_byte_arr, format='JPEG')
+    except Exception:
+        return ""
+    img_byte_arr = img_byte_arr.getvalue()
+
+    # 为图片生成一个唯一的文件名
+    # filename = "image_" + str(os.getpid()) + ".jpg" #uuid
+    unique_id = uuid.uuid4()
+    filename = f"image_{unique_id}.jpg"
+    filepath = os.path.join(static_path, filename)
+
+    # 将字节流写入文件
+    with open(filepath, 'wb') as f:
+        f.write(img_byte_arr)
+
+
+    img_url = f"http://10.99.5.48:8080/file=static/" + filename
+    # img_url = PIL_to_URL(img_url)
+    # img_url = "http://10.99.5.48:8080/file=static/0000.jpeg"
+    result = make_a_try(img_url,text)
+    # print(result)
+    return result
+
+
+def clear_output():
+
+    return ""
+
+def get_example():
+    return [
+        [f"/workdir/yanghandi/gradio_demo/static/0001.jpg", f"图中的人物是谁"]
+    ]
+if __name__ == "__main__":
+
+    param_info = {}
+    # param_info['appkey'] = "com.sankuai.automl.serving"
+    param_info['appkey'] = "10.199.14.151:8001"
+
+    # param_info['remote_appkey'] = "com.sankuai.automl.chat3"
+    param_info['remote_appkey'] = "10.199.14.151:8001"
+    param_info['model_name'] = 'ensemble_mllm'
+    param_info['model_version'] = "1"
+    param_info['time_out'] = 60000
+    param_info['server_targets'] = []
+    param_info['outputs'] = 'response'
+
+
+    gr.set_static_paths(paths=["static/"])
+    
+    with gr.Blocks(title='demo') as demo:
+        gr.Markdown("# 自研模型测试demo")
+        gr.Markdown("尝试使用该demo,上传图片并开始讨论它,或者尝试下面的例子")
+
+        with gr.Row():
+            with gr.Column():
+                # imagebox = gr.Image(value="static/0000.jpeg",type="pil")
+                imagebox = gr.Image(type="pil")
+                promptbox = gr.Textbox(label = "prompt")
+            
+            with gr.Column():
+                output = gr.Textbox(label = "output")
+        with gr.Row():
+            submit = gr.Button("submit")
+            clear = gr.Button("clear")
+
+        submit.click(fn=greet,inputs=[imagebox, promptbox],outputs=[output])
+        clear.click(fn=clear_output, inputs=[], outputs=[output])
+        
+        gr.Markdown("# example")
+        
+        gr.Examples(
+            examples = get_example(),
+            fn = greet,
+            inputs=[imagebox, promptbox],
+            outputs = [output],
+            cache_examples = True
+        )
+
+    demo.launch(server_name="0.0.0.0", server_port=8080, debug=True, share=True)   
+
+
+    # img_url = f"https://s3plus.sankuai.com/automl-pkgs/0000.jpeg"
+    # # img_url = f"http://10.99.5.48:8080/file=static/static/image_cff7077b-3506-4253-82b7-b6547f2f63c1.jpg"
+    # text = f"talk about this women"
+    # greet(img_url,text)

try_demo.py

@@ -0,0 +1,224 @@
+#!/usr/bin/env python
+# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import argparse
+import queue
+import sys
+import uuid
+from functools import partial
+
+import numpy as np
+import tritonclient.grpc as grpcclient
+from tritonclient.utils import InferenceServerException
+
+##
+import time
+import threading
+###
+
+FLAGS = None
+
+
+class UserData:
+    def __init__(self):
+        self._completed_requests = queue.Queue()
+
+
+# Define the callback function. Note the last two parameters should be
+# result and error. InferenceServerClient would povide the results of an
+# inference as grpcclient.InferResult in result. For successful
+# inference, error will be None, otherwise it will be an object of
+# tritonclientutils.InferenceServerException holding the error details
+def callback(user_data, result, error):
+    if error:
+        user_data._completed_requests.put(error)
+    else:
+        user_data._completed_requests.put(result)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        required=False,
+        default=False,
+        help="Enable verbose output",
+    )
+    # parser.add_argument(
+    #     "-u",
+    #     "--url",
+    #     type=str,
+    #     required=False,
+    #     default="localhost:8001",
+    #     help="Inference server URL and it gRPC port. Default is localhost:8001.",
+    # )
+    parser.add_argument(
+        "-u",
+        "--url",
+        type=str,
+        required=False,
+        default="10.199.14.151:8001",
+        help="Inference server URL and it gRPC port. Default is localhost:8001.",
+    )
+    parser.add_argument(
+        "-t",
+        "--stream-timeout",
+        type=float,
+        required=False,
+        default=None,
+        help="Stream timeout in seconds. Default is None.",
+    )
+    # parser.add_argument(
+    #     "-d",
+    #     "--dyna",
+    #     action="store_true",
+    #     required=False,
+    #     default=False,
+    #     help="Assume dynamic sequence model",
+    # )
+    # parser.add_argument(
+    #     "-o",
+    #     "--offset",
+    #     type=int,
+    #     required=False,
+    #     default=0,
+    #     help="Add offset to sequence ID used",
+    # )
+
+    FLAGS = parser.parse_args()
+
+    # # We use custom "sequence" models which take 1 input
+    # # value. The output is the accumulated value of the inputs. See
+    # # src/custom/sequence.
+    # int_sequence_model_name = (
+    #     "simple_dyna_sequence" if FLAGS.dyna else "simple_sequence"
+    # )
+    # string_sequence_model_name = (
+    #     "simple_string_dyna_sequence" if FLAGS.dyna else "simple_sequence"
+    # )
+    model_name = 'ensemble_mllm'
+    model_version = ""
+    batch_size = 1
+    # img_url = f"https://s3plus.sankuai.com/automl-pkgs/0000.jpeg"
+    img_url = "/workdir/yanghandi/gradio_demo/static/0000.jpeg"
+    # img_url = f"https://s3plus.sankuai.com/automl-pkgs/0003.jpeg"
+    text = f"详细描述一下这张图片"
+    sequence_id = 100
+    int_sequence_id0 = sequence_id
+
+    result_list = []
+    user_data = UserData()
+
+    # It is advisable to use client object within with..as clause
+    # when sending streaming requests. This ensures the client
+    # is closed when the block inside with exits.
+    with grpcclient.InferenceServerClient(
+        url=FLAGS.url, verbose=FLAGS.verbose
+    ) as triton_client:
+        try:
+            # Establish stream
+            triton_client.start_stream(
+                callback=partial(callback, user_data),
+                stream_timeout=FLAGS.stream_timeout,
+            )
+
+            # Create the tensor for INPUT
+            inputs = []
+            img_url_bytes = img_url.encode("utf-8")
+            img_url_bytes = np.array(img_url_bytes, dtype=bytes)
+            img_url_bytes = img_url_bytes.reshape([1, -1])
+
+            inputs.append(grpcclient.InferInput('IMAGE_URL', img_url_bytes.shape, "BYTES"))
+            inputs[0].set_data_from_numpy(img_url_bytes)
+
+            text_bytes = text.encode("utf-8")
+            text_bytes = np.array(text_bytes, dtype=bytes)
+            text_bytes = text_bytes.reshape([1, -1])
+            # text_input = np.expand_dims(text_bytes, axis=0)
+            text_input = text_bytes
+
+            inputs.append(grpcclient.InferInput('TEXT', text_input.shape, "BYTES"))
+            inputs[1].set_data_from_numpy(text_input)
+
+            outputs = []
+            outputs.append(grpcclient.InferRequestedOutput("OUTPUT"))
+            # Issue the asynchronous sequence inference.
+            triton_client.async_stream_infer(
+                model_name=model_name,
+                inputs=inputs,
+                outputs=outputs,
+                request_id="{}".format(sequence_id),
+                sequence_id=sequence_id,
+                sequence_start=True,
+                sequence_end=True,
+            )
+
+        except InferenceServerException as error:
+            print(error)
+            sys.exit(1)
+
+        # Retrieve results...
+        recv_count = 0
+
+        #####
+
+        ####
+
+        while True:
+             
+            # if len(result_list) == 80:
+            #     print("1")
+            data_item = user_data._completed_requests.get()
+            # try:
+            #     data_item = user_data._completed_requests.get(timeout=5)
+            # except Exception as e:
+            #     print("queue wrong")
+            #     break
+            if type(data_item) == InferenceServerException:
+                print('InferenceServerException: ', data_item)
+                sys.exit(1)
+            this_id = data_item.get_response().id.split("_")[0]
+            if int(this_id) != int_sequence_id0:
+                print("unexpected sequence id returned by the server: {}".format(this_id))
+                sys.exit(1)
+
+            result = data_item.as_numpy("OUTPUT")
+            if len(result[0][0])==0:
+                break
+
+            result_list.append(data_item.as_numpy("OUTPUT"))
+
+            recv_count = recv_count + 1
+            result_str = ''.join([item[0][0].decode('utf-8') for item in result_list])
+            print(f"{len(result_list)}: {result_str}")
+
+
+        print("hd",result_str)
+    print("PASS: Sequence")
+    print("hd",result_str)

try_demo_demo.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python
+# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import argparse
+import sys
+
+import numpy as np
+import tritonclient.grpc as grpcclient
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        required=False,
+        default=False,
+        help="Enable verbose output",
+    )
+    parser.add_argument(
+        "-u",
+        "--url",
+        type=str,
+        required=False,
+        default="10.95.163.43:8001",
+        help="Inference server URL. Default is localhost:8001.",
+    )
+    parser.add_argument(
+        "-s",
+        "--ssl",
+        action="store_true",
+        required=False,
+        default=False,
+        help="Enable SSL encrypted channel to the server",
+    )
+    parser.add_argument(
+        "-t",
+        "--client-timeout",
+        type=float,
+        required=False,
+        default=None,
+        help="Client timeout in seconds. Default is None.",
+    )
+    parser.add_argument(
+        "-r",
+        "--root-certificates",
+        type=str,
+        required=False,
+        default=None,
+        help="File holding PEM-encoded root certificates. Default is None.",
+    )
+    parser.add_argument(
+        "-p",
+        "--private-key",
+        type=str,
+        required=False,
+        default=None,
+        help="File holding PEM-encoded private key. Default is None.",
+    )
+    parser.add_argument(
+        "-x",
+        "--certificate-chain",
+        type=str,
+        required=False,
+        default=None,
+        help="File holding PEM-encoded certificate chain. Default is None.",
+    )
+    parser.add_argument(
+        "-C",
+        "--grpc-compression-algorithm",
+        type=str,
+        required=False,
+        default=None,
+        help="The compression algorithm to be used when sending request to server. Default is None.",
+    )
+
+    FLAGS = parser.parse_args()
+    try:
+        # triton_client = grpcclient.InferenceServerClient(
+        #     url=FLAGS.url,
+        #     verbose=FLAGS.verbose,
+        #     ssl=FLAGS.ssl,
+        #     root_certificates=FLAGS.root_certificates,
+        #     private_key=FLAGS.private_key,
+        #     certificate_chain=FLAGS.certificate_chain,
+        # )
+        triton_client = grpcclient.InferenceServerClient(
+            url=FLAGS.url,
+            # verbose=FLAGS.verbose,
+            verbose = True,
+            ssl=FLAGS.ssl,
+            root_certificates=None,
+            private_key=None,
+            certificate_chain=None,
+        )
+    except Exception as e:
+        print("channel creation failed: " + str(e))
+        sys.exit()
+
+    model_name = "ensemble_mllm"
+    img_url = f"https://s3plus.sankuai.com/automl-pkgs/0000.jpeg"
+    # img_url = f"https://s3plus.sankuai.com/automl-pkgs/0003.jpeg"
+    text = f"详细描述一下这张图片"
+
+    # Infer
+    inputs = []
+    img_url_bytes = img_url.encode("utf-8")
+    img_url_bytes = np.array(img_url_bytes, dtype=bytes)
+    img_url_bytes = img_url_bytes.reshape([1, -1])
+
+    inputs.append(grpcclient.InferInput('IMAGE_URL', img_url_bytes.shape, "BYTES"))
+    inputs[0].set_data_from_numpy(img_url_bytes)
+
+    text_bytes = text.encode("utf-8")
+    text_bytes = np.array(text_bytes, dtype=bytes)
+    text_bytes = text_bytes.reshape([1, -1])
+    # text_input = np.expand_dims(text_bytes, axis=0)
+    text_input = text_bytes
+
+    inputs.append(grpcclient.InferInput('TEXT', text_input.shape, "BYTES"))
+    inputs[1].set_data_from_numpy(text_input)
+
+    outputs = []
+    outputs.append(grpcclient.InferRequestedOutput("OUTPUT"))
+
+    # Test with outputs
+    results = triton_client.infer(
+        model_name=model_name,
+        inputs=inputs,
+        outputs=outputs,
+        client_timeout=None, #FLAGS.client_timeout,
+        # headers={"test": "1"},
+        compression_algorithm=None, #FLAGS.grpc_compression_algorithm,
+    )
+
+    statistics = triton_client.get_inference_statistics(model_name=model_name)
+    print(statistics)
+    if len(statistics.model_stats) != 1:
+        print("FAILED: Inference Statistics")
+        sys.exit(1)
+
+    # Get the output arrays from the results
+    output_data = results.as_numpy("OUTPUT")
+    result_str = output_data[0][0].decode('utf-8')
+
+    print("OUTPUT: "+ result_str)
+
+    # # Test with no outputs
+    # results = triton_client.infer(
+    #     model_name=model_name,
+    #     inputs=inputs,
+    #     outputs=None,
+    #     compression_algorithm=FLAGS.grpc_compression_algorithm,
+    # )
+
+    # # Get the output arrays from the results
+    # output0_data = results.as_numpy("OUTPUT0")
+    # output1_data = results.as_numpy("OUTPUT1")
+
+    # for i in range(16):
+    #     print(
+    #         str(input0_data[0][i])
+    #         + " + "
+    #         + str(input1_data[0][i])
+    #         + " = "
+    #         + str(output0_data[0][i])
+    #     )
+    #     print(
+    #         str(input0_data[0][i])
+    #         + " - "
+    #         + str(input1_data[0][i])
+    #         + " = "
+    #         + str(output1_data[0][i])
+    #     )
+    #     if (input0_data[0][i] + input1_data[0][i]) != output0_data[0][i]:
+    #         print("sync infer error: incorrect sum")
+    #         sys.exit(1)
+    #     if (input0_data[0][i] - input1_data[0][i]) != output1_data[0][i]:
+    #         print("sync infer error: incorrect difference")
+    #         sys.exit(1)
+
+    print("PASS: infer")

try_grpc.py

@@ -0,0 +1,139 @@
+import gradio as gr
+# from TritonServerClient import TritonServerClient, InferInput, InferRequestedOutput
+# from TritonServerClient.utils import np_to_triton_dtype
+from functools import wraps
+import numpy as np
+
+import tritonclient.grpc as grpcclient
+from tritonclient.utils import InferenceServerException
+
+# def prepare_params(query):
+#     bs=1
+#     # query = "#角色：直播间文案写作专家。#任务：根据用户输入的商品信息写出用于推广营销的直播文案。#目标：引导用户购买直播间商品，提高销量。#要求：- 如果没有输入商品信息，则邀请输入。- 文案写作 - 按照“引出话题”、“产品讲解”、“购买方式”三个方面写作文案。 - 在“产品讲解”部分详细讲解产品细节： - 【重要】充分扩充内容，吸引用户购买。 - 详细描述商品的优惠活动信息（如用户输入没有，不要询问，直接补充）。 - 在“购买方式”部分： - 仅包括引导用户点击链接购买的内容。 - 不要包括快递物流承诺。 - 【重要】每个部分多于5句话。 - 使用口语化的文字。 - 表达方式有亲和力和感染力。 - 对用户们使用“宝子们”、“宝宝们”、“家人们”等有亲和力的称呼。 - 多使用自问自答的语句。- 当你准备好了回复“请输入商品描述和优惠信息”我们就开始。#输出：- 包含在""里。- 写出每一部分内容的标题，包括【引出话题】、【产品讲解】、【购买方式】。- 示例：""【引出话题】【产品讲解】【购买方式】""- 输入商品信息:%s" % prompt
+#     # query = '请你扮演直播间直播卖货的主播角色，根据用户提出的问题，检索知识库中的信息，如果查找到相关的信息，使用口语化的文字，用非常简短的一句话回答用户的问题；如果查找不到相关的信息，直接回答没有，并引导用户关注本场直播商品。你的目的是激发用户的购买欲、提高直播商品销量。回答对用户们使用“宝子”、“宝宝”、“亲”等有亲和力的称呼。\n\n# 用户提出的问题：\n这里面有可乐吗\n\n# 知识库：\n商品名称：外送欢乐桶(券)兑换券\n直播链接序号：64号链接号链接\n商品现价：66元\n商品原价：142元\n折扣：5\n品牌名称：麦当劳麦乐送\n券包张数：1\n套餐详情：["麦辣鸡腿汉堡1个","板烧鸡腿堡1个","中可乐3杯","四拼小食桶A"]\n补充：四拼小食桶A(麦麦脆汁鸡(鸡腿)1块＋中薯条1份＋麦辣鸡翅2块＋麦乐鸡5块)＋麦辣鸡腿堡1个＋板烧鸡腿堡1个＋中可乐3杯 原材料：面包\n\n\n# 输出格式：\n答：\n\n# 要求\n对于用户问到了知识库中未提及的信息不要编造，直接不回答'
+#     ######
+#     #####
+#     title_text =  np.array([query.encode('utf-8')], dtype=np.string_)
+#     title_text = np.tile(title_text, (bs, 1))
+
+#     data_batch = {}
+#     data_batch['query'] = title_text
+
+#     inputs = [
+#             InferInput("query", data_batch['query'].shape,
+#                                 np_to_triton_dtype(data_batch['query'].dtype)),
+#     ]
+
+#     inputs[0].set_data_from_numpy(data_batch['query'])
+
+#     return inputs
+def prepare_params(query,img):
+    bs=1
+    # query = "#角色：直播间文案写作专家。#任务：根据用户输入的商品信息写出用于推广营销的直播文案。#目标：引导用户购买直播间商品，提高销量。#要求：- 如果没有输入商品信息，则邀请输入。- 文案写作 - 按照“引出话题”、“产品讲解”、“购买方式”三个方面写作文案。 - 在“产品讲解”部分详细讲解产品细节： - 【重要】充分扩充内容，吸引用户购买。 - 详细描述商品的优惠活动信息（如用户输入没有，不要询问，直接补充）。 - 在“购买方式”部分： - 仅包括引导用户点击链接购买的内容。 - 不要包括快递物流承诺。 - 【重要】每个部分多于5句话。 - 使用口语化的文字。 - 表达方式有亲和力和感染力。 - 对用户们使用“宝子们”、“宝宝们”、“家人们”等有亲和力的称呼。 - 多使用自问自答的语句。- 当你准备好了回复“请输入商品描述和优惠信息”我们就开始。#输出：- 包含在""里。- 写出每一部分内容的标题，包括【引出话题】、【产品讲解】、【购买方式】。- 示例：""【引出话题】【产品讲解】【购买方式】""- 输入商品信息:%s" % prompt
+    # query = '请你扮演直播间直播卖货的主播角色，根据用户提出的问题，检索知识库中的信息，如果查找到相关的信息，使用口语化的文字，用非常简短的一句话回答用户的问题；如果查找不到相关的信息，直接回答没有，并引导用户关注本场直播商品。你的目的是激发用户的购买欲、提高直播商品销量。回答对用户们使用“宝子”、“宝宝”、“亲��等有亲和力的称呼。\n\n# 用户提出的问题：\n这里面有可乐吗\n\n# 知识库：\n商品名称：外送欢乐桶(券)兑换券\n直播链接序号：64号链接号链接\n商品现价：66元\n商品原价：142元\n折扣：5\n品牌名称：麦当劳麦乐送\n券包张数：1\n套餐详情：["麦辣鸡腿汉堡1个","板烧鸡腿堡1个","中可乐3杯","四拼小食桶A"]\n补充：四拼小食桶A(麦麦脆汁鸡(鸡腿)1块＋中薯条1份＋麦辣鸡翅2块＋麦乐鸡5块)＋麦辣鸡腿堡1个＋板烧鸡腿堡1个＋中可乐3杯 原材料：面包\n\n\n# 输出格式：\n答：\n\n# 要求\n对于用户问到了知识库中未提及的信息不要编造，直接不回答'
+    inputs = []
+    ######
+    img_info = np.array([img.encode('utf-8')], dtype=np.string_)
+    img_info = np.tile(img_info, (bs,1))
+
+
+    inputs.append(InferInput("img", img_info.shape,
+                                np_to_triton_dtype(img_info.dtype)),)
+            
+    
+    inputs[0].set_data_from_numpy(img_info)
+    #####
+    title_text =  np.array([query.encode('utf-8')], dtype=np.string_)
+    title_text = np.tile(title_text, (bs, 1))
+
+    data_batch = {}
+    data_batch['query'] = title_text
+
+    inputs.append( InferInput("query", data_batch['query'].shape,
+                                np_to_triton_dtype(data_batch['query'].dtype))
+    )
+           
+
+    inputs[1].set_data_from_numpy(data_batch['query'])
+
+    return inputs
+
+def make_a_try(inputs, outputs='response', model_name='ensemble_mllm', model_version=''): # qwen 1
+    outputs_list = []
+    ori_outputs_list = outputs.strip().split(",")
+    for out_ele in ori_outputs_list:
+        outputs_list.append(out_ele.strip())
+    outputs = [InferRequestedOutput(x) for x in outputs_list]
+
+    response = my_client.predict(model_name=model_name, inputs=inputs, model_version=model_version, outputs=outputs)
+
+    rsp_info = {}
+    if outputs_list == []:
+        for out_name_ele in response._result.outputs:
+            outputs_list.append(out_name_ele.name)
+    for output_name in outputs_list:
+        res = response.as_numpy(output_name)
+        response = np.expand_dims(res, axis=0)
+        response = response[0].decode('utf-8')
+        rsp_info[output_name] = response
+    print("response:",rsp_info)
+    return rsp_info['response']
+
+
+
+# def greet(prompt):
+#     """Greet someone."""
+#     # print(prompt)
+#     print("prompt:",prompt)
+#     inputs = prepare_params(prompt)
+#     print(inputs)
+#     result = make_a_try(inputs)
+
+#     return result
+def greet(prompt,img):
+    """Greet someone."""
+    # print(prompt)
+    print("prompt:",prompt)
+    inputs = prepare_params(prompt,img)
+    print(inputs)
+    result = make_a_try(inputs)
+
+    return result
+
+def clear_input():
+
+    return ""
+
+
+if __name__ == "__main__":
+    param_info = {}
+    # param_info['appkey'] = "com.sankuai.automl.serving"
+    param_info['appkey'] = "com.sankuai.automl.streamvlm"
+
+    # param_info['remote_appkey'] = "com.sankuai.automl.chat3"
+    param_info['remote_appkey'] = "com.sankuai.automl.streamvlm"
+
+    param_info['model_name'] = "ensemble_mllm"
+    param_info['model_version'] = "1"
+    param_info['time_out'] = 60000
+    param_info['server_targets'] = []
+    param_info['outputs'] = 'response'
+
+    appkey, remote_appkey, model_name, model_version, time_out, server_targets =  param_info['appkey'], param_info['remote_appkey'], param_info['model_name'], param_info['model_version'], param_info['time_out'], param_info['server_targets']
+
+    my_client = TritonServerClient(appkey=appkey, remote_appkey=remote_appkey, time_out=time_out, server_targets=server_targets)
+    # triton_client.async_stream_infer(
+    #     model_name=model_name,
+    #     inputs=inputs,
+    #     outputs=outputs,
+    #     request_id="{}".format(sequence_id),
+    #     sequence_id=sequence_id,
+    #     sequence_start=True,
+    #     sequence_end=True,
+    # )
+
+    img_url = f"https://s3plus.sankuai.com/automl-pkgs/0000.jpeg"
+
+    greet("nihao",img_url)
+    # greet("nihao")
+    print("描述这张图片")

try_hd.py

@@ -0,0 +1,137 @@
+import gradio as gr
+from TritonServerClient import TritonServerClient, InferInput, InferRequestedOutput
+from TritonServerClient.utils import np_to_triton_dtype
+from functools import wraps
+import numpy as np
+
+
+# def prepare_params(query):
+#     bs=1
+#     # query = "#角色：直播间文案写作专家。#任务：根据用户输入的商品信息写出用于推广营销的直播文案。#目标：引导用户购买直播间商品，提高销量。#要求：- 如果没有输入商品信息，则邀请输入。- 文案写作 - 按照“引出话题”、“产品讲解”、“购买方式”三个方面写作文案。 - 在“产品讲解”部分详细讲解产品细节： - 【重要】充分扩充内容，吸引用户购买。 - 详细描述商品的优惠活动信息（如用户输入没有，不要询问，直接补充）。 - 在“购买方式”部分： - 仅包括引导用户点击链接购买的内容。 - 不要包括快递物流承诺。 - 【重要】每个部分多于5句话。 - 使用口语化的文字。 - 表达方式有亲和力和感染力。 - 对用户们使用“宝子们”、“宝宝们”、“家人们”等有亲和力的称呼。 - 多使用自问自答的语句。- 当你准备好了回复“请输入商品描述和优惠信息”我们就开始。#输出：- 包含在""里。- 写出每一部分内容的标题，包括【引出话题】、【产品讲解】、【购买方式】。- 示例：""【引出话题】【产品讲解】【购买方式】""- 输入商品信息:%s" % prompt
+#     # query = '请你扮演直播间直播卖货的主播角色，根据用户提出的问题，检索知识库中的信息，如果查找到相关的信息，使用口语化的文字，用非常简短的一句话回答用户的问题；如果查找不到相关的信息，直接回答没有，并引导用户关注本场直播商品。你的目的是激发用户的购买欲、提高直播商品销量。回答对用户们使用“宝子”、“宝宝”、“亲”等有亲和力的称呼。\n\n# 用户提出的问题：\n这里面有可乐吗\n\n# 知识库：\n商品名称：外送欢乐桶(券)兑换券\n直播链接序号：64号链接号链接\n商品现价：66元\n商品原价：142元\n折扣：5\n品牌名称：麦当劳麦乐送\n券包张数：1\n套餐详情：["麦辣鸡腿汉堡1个","板烧鸡腿堡1个","中可乐3杯","四拼小食桶A"]\n补充：四拼小食桶A(麦麦脆汁鸡(鸡腿)1块＋中薯条1份＋麦辣鸡翅2块＋麦乐鸡5块)＋麦辣鸡腿堡1个＋板烧鸡腿堡1个＋中可乐3杯 原材料：面包\n\n\n# 输出格式：\n答：\n\n# 要求\n对于用户问到了知识库中未提及的信息不要编造，直接不回答'
+#     ######
+#     #####
+#     title_text =  np.array([query.encode('utf-8')], dtype=np.string_)
+#     title_text = np.tile(title_text, (bs, 1))
+
+#     data_batch = {}
+#     data_batch['query'] = title_text
+
+#     inputs = [
+#             InferInput("query", data_batch['query'].shape,
+#                                 np_to_triton_dtype(data_batch['query'].dtype)),
+#     ]
+
+#     inputs[0].set_data_from_numpy(data_batch['query'])
+
+#     return inputs
+def prepare_params(query,img):
+    bs=1
+    # query = "#角色：直播间文案写作专家。#任务：根据用户输入的商品信息写出用于推广营销的直播文案。#目标：引导用户购买直播间商品，提高销量。#要求：- 如果没有输入商品信息，则邀请输入。- 文案写作 - 按照“引出话题”、“产品讲解”、“购买方式”三个方面写作文案。 - 在“产品讲解”部分详细讲解产品细节： - 【重要】充分扩充内容，吸引用户购买。 - 详细描述商品的优惠活动信息（如用户输入没有，不要询问，直接补充）。 - 在“购买方式”部分： - 仅包括引导用户点击链接购买的内容。 - 不要包括快递物流承诺。 - 【重要】每个部分多于5句话。 - 使用口语化的文字。 - 表达方式有亲和力和感染力。 - 对用户们使用“宝子们”、“宝宝们”、“家人们”等有亲和力的称呼。 - 多使用自问自答的语句。- 当你准备好了回复“请输入商品描述和优惠信息”我们就开始。#输出：- 包含在""里。- 写出每一部分内容的标题，包括【引出话题】、【产品讲解】、【购买方式】。- 示例：""【引出话题】【产品讲解】【购买方式】""- 输入商品信息:%s" % prompt
+    # query = '请你扮演直播间直播卖货的主播角色，根据用户提出的问题，检索知识库中的信息，如果查找到相关的信息，使用口语化的文字，用非常简短的一句话回答用户的问题；如果查找不到相关的信息，直接回答没有，并引导用户关注本场直播商品。你的目的是激发用户的购买欲、提高直播商品销量。回答对用户们使用“宝子”、“宝宝”、“亲”等有亲和力的称呼。\n\n# 用户提出的问题：\n这里面有可乐吗\n\n# 知识库：\n商品名称：外送欢乐桶(券)兑换券\n直播链接序号：64号链接号链接\n商品现价：66元\n商品原价：142元\n折扣：5\n品牌名称：麦当劳麦乐送\n券包张数：1\n套餐详情：["麦辣鸡腿汉堡1个","板烧鸡腿堡1个","中可乐3杯","四拼小食桶A"]\n补充：四拼小食桶A(麦麦脆汁鸡(鸡腿)1块＋中薯条1份＋麦辣鸡翅2块＋麦乐鸡5块)＋麦辣鸡腿堡1个＋板烧鸡腿堡1个＋中可乐3杯 原材料：面包\n\n\n# 输出格式：\n答：\n\n# 要求\n对于用户问到了知识库中未提及的信息不要编造，直接不回答'
+    inputs = []
+    ######
+    img_info = np.array([img.encode('utf-8')], dtype=np.string_)
+    img_info = np.tile(img_info, (bs,1))
+
+
+    inputs.append(InferInput("img", img_info.shape,
+                                np_to_triton_dtype(img_info.dtype)),)
+            
+    
+    inputs[0].set_data_from_numpy(img_info)
+    #####
+    title_text =  np.array([query.encode('utf-8')], dtype=np.string_)
+    title_text = np.tile(title_text, (bs, 1))
+
+    data_batch = {}
+    data_batch['query'] = title_text
+
+    inputs.append( InferInput("query", data_batch['query'].shape,
+                                np_to_triton_dtype(data_batch['query'].dtype))
+    )
+           
+
+    inputs[1].set_data_from_numpy(data_batch['query'])
+
+    return inputs
+
+def make_a_try(inputs, outputs='response', model_name='ensemble_mllm', model_version=''): # qwen 1
+    outputs_list = []
+    ori_outputs_list = outputs.strip().split(",")
+    for out_ele in ori_outputs_list:
+        outputs_list.append(out_ele.strip())
+    outputs = [InferRequestedOutput(x) for x in outputs_list]
+
+    response = my_client.predict(model_name=model_name, inputs=inputs, model_version=model_version, outputs=outputs)
+
+    rsp_info = {}
+    if outputs_list == []:
+        for out_name_ele in response._result.outputs:
+            outputs_list.append(out_name_ele.name)
+    for output_name in outputs_list:
+        res = response.as_numpy(output_name)
+        response = np.expand_dims(res, axis=0)
+        response = response[0].decode('utf-8')
+        rsp_info[output_name] = response
+    print("response:",rsp_info)
+    return rsp_info['response']
+
+
+
+# def greet(prompt):
+#     """Greet someone."""
+#     # print(prompt)
+#     print("prompt:",prompt)
+#     inputs = prepare_params(prompt)
+#     print(inputs)
+#     result = make_a_try(inputs)
+
+#     return result
+def greet(prompt,img):
+    """Greet someone."""
+    # print(prompt)
+    print("prompt:",prompt)
+    inputs = prepare_params(prompt,img)
+    print(inputs)
+    result = make_a_try(inputs)
+
+    return result
+
+def clear_input():
+
+    return ""
+
+
+if __name__ == "__main__":
+    param_info = {}
+    # param_info['appkey'] = "com.sankuai.automl.serving"
+    param_info['appkey'] = "com.sankuai.automl.streamvlm"
+
+    # param_info['remote_appkey'] = "com.sankuai.automl.chat3"
+    param_info['remote_appkey'] = "com.sankuai.automl.streamvlm"
+
+    param_info['model_name'] = "ensemble_mllm"
+    param_info['model_version'] = "1"
+    param_info['time_out'] = 60000
+    param_info['server_targets'] = []
+    param_info['outputs'] = 'response'
+
+    appkey, remote_appkey, model_name, model_version, time_out, server_targets =  param_info['appkey'], param_info['remote_appkey'], param_info['model_name'], param_info['model_version'], param_info['time_out'], param_info['server_targets']
+
+    my_client = TritonServerClient(appkey=appkey, remote_appkey=remote_appkey, time_out=time_out, server_targets=server_targets)
+    # triton_client.async_stream_infer(
+    #     model_name=model_name,
+    #     inputs=inputs,
+    #     outputs=outputs,
+    #     request_id="{}".format(sequence_id),
+    #     sequence_id=sequence_id,
+    #     sequence_start=True,
+    #     sequence_end=True,
+    # )
+
+    img_url = f"https://s3plus.sankuai.com/automl-pkgs/0000.jpeg"
+
+    greet("nihao",img_url)
+    # greet("nihao")
+    print("描述这张图片")

try_hd_v2.py

@@ -0,0 +1,218 @@
+import argparse
+import queue
+import sys
+import uuid
+from functools import partial
+
+import numpy as np
+import tritonclient.grpc as grpcclient
+from tritonclient.utils import InferenceServerException
+import gradio as gr
+from functools import wraps
+
+####
+from PIL import Image
+import base64
+import io
+#####
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+import socket
+####
+import os
+import uuid
+####
+
+class UserData:
+    def __init__(self):
+        self._completed_requests = queue.Queue()
+
+def callback(user_data, result, error):
+    if error:
+        user_data._completed_requests.put(error)
+    else:
+        user_data._completed_requests.put(result)
+
+def make_a_try(img_url,text):
+    model_name = 'ensemble_mllm'
+    user_data = UserData()
+    sequence_id = 100
+    int_sequence_id0 = sequence_id
+    result_list=[]
+    with grpcclient.InferenceServerClient(
+        url="10.199.14.151:8001", verbose = False
+    ) as triton_client:
+        try:
+            # Establish stream
+            triton_client.start_stream(
+                callback=partial(callback, user_data),
+                stream_timeout=None,
+            )
+            # Create the tensor for INPUT
+            inputs = []
+            img_url_bytes = img_url.encode("utf-8")
+            img_url_bytes = np.array(img_url_bytes, dtype=bytes)
+            img_url_bytes = img_url_bytes.reshape([1, -1])
+
+            inputs.append(grpcclient.InferInput('IMAGE_URL', img_url_bytes.shape, "BYTES"))
+            inputs[0].set_data_from_numpy(img_url_bytes)
+
+            text_bytes = text.encode("utf-8")
+            text_bytes = np.array(text_bytes, dtype=bytes)
+            text_bytes = text_bytes.reshape([1, -1])
+            # text_input = np.expand_dims(text_bytes, axis=0)
+            text_input = text_bytes
+
+            inputs.append(grpcclient.InferInput('TEXT', text_input.shape, "BYTES"))
+            inputs[1].set_data_from_numpy(text_input)
+
+            outputs = []
+            outputs.append(grpcclient.InferRequestedOutput("OUTPUT"))
+            # Issue the asynchronous sequence inference.
+            triton_client.async_stream_infer(
+                model_name=model_name,
+                inputs=inputs,
+                outputs=outputs,
+                request_id="{}".format(sequence_id),
+                sequence_id=sequence_id,
+                sequence_start=True,
+                sequence_end=True,
+            )
+        ######hd
+        except InferenceServerException as error:
+            print(error)
+            # sys.exit(1)
+            # continue
+            return ""
+
+        
+        # Retrieve results...
+        recv_count = 0
+
+        while True:
+            try:
+                data_item = user_data._completed_requests.get(timeout=5)
+            except Exception as e:
+                break
+            # data_item = user_data._completed_requests.get()
+            if type(data_item) == InferenceServerException:
+                print('InferenceServerException: ', data_item)
+                # sys.exit(1)
+                return ""
+            this_id = data_item.get_response().id.split("_")[0]
+            if int(this_id) != int_sequence_id0:
+                print("unexpected sequence id returned by the server: {}".format(this_id))
+                # sys.exit(1)
+                return ""
+            ####
+            result = data_item.as_numpy("OUTPUT")
+            if len(result[0][0])==0:
+                break
+            ####
+            result_list.append(data_item.as_numpy("OUTPUT"))
+
+            recv_count = recv_count + 1
+            result_str = ''.join([item[0][0].decode('utf-8') for item in result_list])
+    return result_str
+
+
+def greet(image, text):
+    ###save img
+    static_path = f"/workdir/yanghandi/gradio_demo/static"
+    # 将图片转换为字节流
+    img_byte_arr = io.BytesIO()
+    try:
+        image.save(img_byte_arr, format='JPEG')
+    except Exception:
+        return ""
+    img_byte_arr = img_byte_arr.getvalue()
+
+    # 为图片生成一个唯一的文件名
+    # filename = "image_" + str(os.getpid()) + ".jpg" #uuid
+    unique_id = uuid.uuid4()
+    filename = f"image_{unique_id}.jpg"
+    filepath = os.path.join(static_path, filename)
+
+    # 将字节流写入文件
+    with open(filepath, 'wb') as f:
+        f.write(img_byte_arr)
+
+
+    img_url = f"http://10.99.5.48:8080/file=static/" + filename
+    # img_url = PIL_to_URL(img_url)
+    # img_url = "http://10.99.5.48:8080/file=static/0000.jpeg"
+    result = make_a_try(img_url,text)
+    # print(result)
+    return result
+
+# def greet_example(image, text):
+#     ###save img
+#     # filename = image
+#     # static_path = "/workdir/yanghandi/gradio_demo/static"
+#     img_url = "http://10.99.5.48:8080/file=static/0000.jpeg" 
+
+#     # img_url = PIL_to_URL(img_url)
+#     # img_url = "http://10.99.5.48:8080/file=static/0000.jpeg"
+#     result = make_a_try(img_url,text)
+#     # print(result)
+#     return result
+
+def clear_output():
+
+    return ""
+
+def get_example():
+    return [
+        [f"/workdir/yanghandi/gradio_demo/static/0001.jpg", f"图中的人物是谁"]
+    ]
+if __name__ == "__main__":
+
+    param_info = {}
+    # param_info['appkey'] = "com.sankuai.automl.serving"
+    param_info['appkey'] = "10.199.14.151:8001"
+
+    # param_info['remote_appkey'] = "com.sankuai.automl.chat3"
+    param_info['remote_appkey'] = "10.199.14.151:8001"
+    param_info['model_name'] = 'ensemble_mllm'
+    param_info['model_version'] = "1"
+    param_info['time_out'] = 60000
+    param_info['server_targets'] = []
+    param_info['outputs'] = 'response'
+
+
+    gr.set_static_paths(paths=["static/"])
+    
+    with gr.Blocks(title='demo') as demo:
+        gr.Markdown("# 自研模型测试demo")
+        gr.Markdown("尝试使用该demo,上传图片并开始讨论它,或者尝试下面的例子")
+
+        with gr.Row():
+            with gr.Column():
+                # imagebox = gr.Image(value="static/0000.jpeg",type="pil")
+                imagebox = gr.Image(type="pil")
+                promptbox = gr.Textbox(label = "prompt")
+            
+            with gr.Column():
+                output = gr.Textbox(label = "output")
+        with gr.Row():
+            submit = gr.Button("submit")
+            clear = gr.Button("clear")
+
+        submit.click(fn=greet,inputs=[imagebox, promptbox],outputs=[output])
+        clear.click(fn=clear_output, inputs=[], outputs=[output])
+        
+        gr.Markdown("# example")
+        
+        gr.Examples(
+            examples = get_example(),
+            fn = greet,
+            inputs=[imagebox, promptbox],
+            outputs = [output],
+            cache_examples = True
+        )
+
+    demo.launch(server_name="0.0.0.0", server_port=8080, debug=True, share=True)   
+
+
+    # img_url = f"https://s3plus.sankuai.com/automl-pkgs/0000.jpeg"
+    # text = f"详细描述一下这张图片"
+    # greet(img_url,text)