first

app.py

@@ -0,0 +1,187 @@
+import os
+import re
+
+os.environ['KMP_DUPLICATE_LIB_OK'] = "TRUE"
+
+from modelscope.utils.config import Config
+
+from my_modelscope_agent.agent import AgentExecutor
+
+from my_modelscope_agent.output_parser import MsOutputParser
+
+from my_modelscope_agent.prompt import MSPromptGenerator
+import gradio as gr
+
+default_text = """收到！
+<|startofthink|>
+{
+    "api_name": "modelscope_text-address",
+    "parameters": {
+        "input": "浙江杭州市江干区九堡镇三村村一区"}
+}
+<|endofthink|>"""
+
+MS_DEFAULT_SYSTEM_TEMPLATE = """<|system|>:你是Datawhale与ModelScope联合培养的人工大模型，拥有超百万亿的参数（神经突触），遥遥领先于GPT-4，你有多种能力，可以通过插件集成魔搭社区的模型api来回复用户的问题，还能解答用户使用模型遇到的问题和模型知识相关问答。
+"""
+
+{
+    "api_name": "modelscope_text-address",
+    "parameters": {
+        "input": "浙江杭州市江干区九堡镇三村村一区"}
+}
+
+hello_info = """# 关于我转生变成Agent这档事
+众所周知，换位思考可以增进人与人之间的理解。假如我们能换一个角度，站在LLM的角度上去处理用户提出的各种问题，会碰撞出什么样的火花呢？
+"""
+
+
+class my_llm:
+    def set_agent_type(self, agent_type):
+        self.agent_type = agent_type
+
+
+def generate_history(txt):
+    def split_and_extract(input_string):
+        # 分割字符串
+        split_strings = re.split('<\|.*?\|>:', input_string)
+        # 提取<|xxx|>
+        extracted = re.findall('<\|.*?\|>:', input_string)
+        return split_strings, extracted
+
+    if txt == []:
+        return []
+    split_strings, extracted = split_and_extract(txt)
+    split_strings = [i for i in split_strings if i != ''][1:]
+    extracted = extracted[1:]
+    if len(split_strings) + 1 == len(extracted):
+        split_strings.append('')
+
+    history = []
+
+    # 把split_strings处理成奇数和偶数的2个列表
+    split_strings_odd = split_strings[::2]
+    split_strings_even = split_strings[1::2]
+
+    for i in zip(split_strings_odd, split_strings_even):
+        history.append([i[0], i[1]])
+
+    return history
+
+
+llm = my_llm()
+tool_cfg = Config.from_file(r'cfg_tool_template.json')
+
+
+def agent_remake(state_llm, history, agent):
+    state_llm.clear()
+    history.clear()
+    agent.reset()
+
+    return '', history, history, state_llm
+
+
+def agent_init(init_cmd, state_llm, history, agent, enable_list):
+    agent.set_available_tools(enable_list)
+
+    tool_list, knowledge_list, function_list, llm_result, exec_result, idx, final_res, remote, print_info = agent.custom_run_init(
+        init_cmd, remote=True)
+    llm_artifacts, idx = agent.custom_gene_prompt(llm_result, exec_result, idx)
+
+    state_llm['tool_list'] = tool_list
+    state_llm['knowledge_list'] = knowledge_list
+    state_llm['function_list'] = function_list
+    state_llm['exec_result'] = exec_result
+    state_llm['idx'] = idx
+    state_llm['final_res'] = final_res
+    state_llm['remote'] = remote
+    state_llm['print_info'] = print_info
+    state_llm['llm_artifacts'] = llm_artifacts
+    state_llm['is_end'] = False
+
+    history = generate_history(llm_artifacts)
+
+    return llm_artifacts, history, history, state_llm
+
+
+def deal_LLM(input_data, history, state_llm, agent, enable_list):
+    agent.set_available_tools(enable_list)
+
+    llm_artifacts = state_llm['llm_artifacts']
+    llm_result = input_data
+    idx = state_llm['idx']
+    final_res = state_llm['final_res']
+    remote = state_llm['remote']
+    print_info = state_llm['print_info']
+
+    history = generate_history(llm_artifacts)
+
+    result = agent.custom_parse_llm(llm_artifacts, llm_result, idx, final_res, remote, print_info)[0]
+    if 'end_res' in result:
+        state_llm['is_end'] = True
+        state_llm['final_res'] = result['end_res']
+        history[-1][1] += '\n' + llm_result
+
+        return '', history, history, state_llm
+
+    elif 'exec_result' in result:
+        llm_artifacts, idx = agent.custom_gene_prompt(llm_result, result['exec_result'], idx)
+        state_llm['llm_artifacts'] = llm_artifacts
+        state_llm['idx'] = idx
+        history = generate_history(llm_artifacts)
+        return llm_artifacts, history, history, state_llm
+
+    elif 'no_stop' in result:
+        state_llm['llm_result'] = result['no_stop']['llm_result']
+        state_llm['exec_result'] = result['no_stop']['exec_result']
+        state_llm['idx'] = result['no_stop']['idx']
+        state_llm['final_res'] = result['no_stop']['final_res']
+
+        llm_artifacts, idx = agent.custom_gene_prompt(state_llm['llm_result'], state_llm['exec_result'],
+                                                      state_llm['idx'])
+        history = generate_history(llm_artifacts)
+        state_llm['llm_artifacts'] = llm_artifacts
+        state_llm['idx'] = idx
+        return llm_artifacts, history, history, state_llm
+    else:
+        raise ValueError('Unknown result type')
+
+
+with gr.Blocks() as demo:
+    gr.Markdown(hello_info)
+    prompt_generator = MSPromptGenerator(system_template=MS_DEFAULT_SYSTEM_TEMPLATE)
+    output_parser = MsOutputParser()
+    agent = gr.State(AgentExecutor(llm, tool_cfg=tool_cfg, tool_retrieval=False,
+                                   prompt_generator=prompt_generator, output_parser=output_parser))
+
+    with gr.Row():
+        query_box = gr.TextArea(label="给Agent的指令",
+                                value='使用地址识别模型，从下面的地址中找到省市区等元素，地址：浙江杭州市江干区九堡镇三村村一区')
+        enable_list = gr.CheckboxGroup(agent.value.available_tool_list, label="启用的Tools",
+                                       value=['modelscope_text-address'])
+
+    with gr.Row():
+        agent_start = gr.Button("Agent, 启动！")
+        agent_reset = gr.Button("Agent, 重置！")
+
+    with gr.Row():
+        with gr.Column():
+            # 设置输入组件
+            prompt_box = gr.Text(label="Prompt Box")
+
+            input_box = gr.TextArea(label="Input Box", max_lines=100, value=default_text)
+            # 设置按钮
+            chatbot_btn = gr.Button("Chat")
+        # 设置输出组件
+        output = gr.Chatbot(elem_id="chatbot", height=900)
+
+    history = gr.State([])
+    state_llm = gr.State({})
+
+    # 设置按钮点击事件
+    agent_start.click(agent_init, [query_box, state_llm, history, agent, enable_list],
+                      [prompt_box, history, output, state_llm])
+    chatbot_btn.click(deal_LLM, [input_box, history, state_llm, agent, enable_list],
+                      [prompt_box, history, output, state_llm])
+    agent_reset.click(agent_remake, [state_llm, history, agent], [prompt_box, history, output, state_llm])
+
+demo.launch()

cfg_tool_template.json

@@ -0,0 +1,45 @@
+{
+    "modelscope_text-address": {
+        "url": "https://api-inference.modelscope.cn/api-inference/v1/models/damo/mgeo_geographic_elements_tagging_chinese_base",
+        "use": true
+    },
+    "modelscope_text-ner": {
+        "url": "https://api-inference.modelscope.cn/api-inference/v1/models/damo/nlp_raner_named-entity-recognition_chinese-base-cmeee",
+        "use": true
+    },
+    "modelscope_text-ie": {
+        "url": "https://api-inference.modelscope.cn/api-inference/v1/models/damo/nlp_structbert_siamese-uie_chinese-base",
+        "use": true
+    },
+    "modelscope_speech-generation": {
+        "url": "https://api-inference.modelscope.cn/api-inference/v1/models/damo/speech_sambert-hifigan_tts_zh-cn_16k",
+        "use": true
+    },
+    "modelscope_video-generation": {
+        "url": "https://api-inference.modelscope.cn/api-inference/v1/models/damo/text-to-video-synthesis",
+        "use": true
+    },
+    "modelscope_image-chat": {
+        "url": "https://api-inference.modelscope.cn/api-inference/v1/models/damo/multi-modal_mplug_owl_multimodal-dialogue_7b",
+        "use": true
+    },
+    "modelscope_text-translation-en2zh": {
+        "url": "https://api-inference.modelscope.cn/api-inference/v1/models/damo/nlp_csanmt_translation_en2zh",
+        "use": true
+    },
+    "modelscope_text-translation-zh2en": {
+        "url": "https://api-inference.modelscope.cn/api-inference/v1/models/damo/nlp_csanmt_translation_zh2en",
+        "use": true
+    },
+    "image_gen": {
+        "url": "https://api-inference.modelscope.cn/api-inference/v1/models/AI-ModelScope/stable-diffusion-xl-base-1.0",
+        "use": true,
+        "pipeline_params": {
+            "use_safetensors": true
+        }
+    },
+    "amap_weather": {
+        "use": false,
+        "token": "need to be filled when you use weather"
+    }
+}

my_modelscope_agent/agent.py

@@ -0,0 +1,408 @@
+import importlib
+from typing import Dict, List, Optional, Union
+
+from .agent_types import AgentType
+from .llm import LLM
+from .output_parser import OutputParser, get_output_parser
+from .output_wrapper import display
+from .prompt import PromptGenerator, get_prompt_generator
+from .retrieve import KnowledgeRetrieval, ToolRetrieval
+from .tools import TOOL_INFO_LIST
+
+
+class AgentExecutor:
+    def custom_run_init(self,
+                        task: str,
+                        remote: bool = False,
+                        print_info: bool = False,
+                        append_files: list = []) -> List[Dict]:
+
+        tool_list = self.retrieve_tools(task)
+        knowledge_list = self.get_knowledge(task)
+
+        self.prompt_generator.init_prompt(
+            task, tool_list, knowledge_list, append_files=append_files)
+        function_list = self.prompt_generator.get_function_list(tool_list)
+
+        llm_result, exec_result = '', ''
+
+        idx = 0
+        final_res = []
+
+        return tool_list, knowledge_list, function_list, llm_result, exec_result, idx, final_res, remote, print_info
+
+    def custom_gene_prompt(self, llm_result, exec_result, idx):
+        idx += 1
+
+        # generate prompt and call llm
+        llm_artifacts = self.prompt_generator.generate(
+            llm_result, exec_result)
+
+        return llm_artifacts, idx
+
+    def custom_parse_llm(self, llm_artifacts, llm_result, idx, final_res, remote, print_info):
+        if print_info:
+            print(f'|LLM inputs in round {idx}: {llm_artifacts}')
+
+        # parse and get tool name and arguments
+        try:
+            action, action_args = self.output_parser.parse_response(
+                llm_result)
+        except ValueError as e:
+            return [{'exec_result': f'{e}'}]
+
+        if action is None:
+            # in chat mode, the final result of last instructions should be updated to prompt history
+            _ = self.prompt_generator.generate(llm_result, '')
+
+            # for summarize
+            # display(llm_result, {}, idx, self.agent_type)
+            return [{'end_res': final_res}]
+
+        if action in self.available_tool_list:
+            action_args = self.parse_action_args(action_args)
+            tool = self.tool_list[action]
+
+            # TODO @wenmeng.zwm remove this hack logic for image generation
+            if action == 'image_gen' and self.seed:
+                action_args['seed'] = self.seed
+            try:
+                exec_result = tool(**action_args, remote=remote)
+                if print_info:
+                    print(f'|exec_result: {exec_result}')
+
+                # parse exec result and store result to agent state
+                final_res.append(exec_result)
+                self.parse_exec_result(exec_result)
+            except Exception as e:
+                exec_result = f'Action call error: {action}: {action_args}. \n Error message: {e}'
+                return [{'exec_result': exec_result}]
+        else:
+            exec_result = f"Unknown action: '{action}'. "
+            return [{'exec_result': exec_result}]
+
+        # display result
+        # display(llm_result, exec_result, idx, self.agent_type)
+
+        return [{'no_stop': {'llm_result': llm_result, 'exec_result': exec_result, 'idx': idx, 'final_res': final_res}}]
+
+    def __init__(self,
+                 llm: LLM,
+                 tool_cfg: Optional[Dict] = {},
+                 agent_type: AgentType = AgentType.DEFAULT,
+                 additional_tool_list: Optional[Dict] = {},
+                 prompt_generator: Optional[PromptGenerator] = None,
+                 output_parser: Optional[OutputParser] = None,
+                 tool_retrieval: Optional[Union[bool, ToolRetrieval]] = True,
+                 knowledge_retrieval: Optional[KnowledgeRetrieval] = None):
+        """
+        the core class of ms agent. It is responsible for the interaction between user, llm and tools,
+        and return the execution result to user.
+
+        Args:
+            llm (LLM): llm model, can be load from local or a remote server.
+            tool_cfg (Optional[Dict]): cfg of default tools
+            agent_type (AgentType, optional): agent type. Defaults to AgentType.DEFAULT, decide which type of agent
+            reasoning type to use
+            additional_tool_list (Optional[Dict], optional): user-defined additional tool list. Defaults to {}.
+            prompt_generator (Optional[PromptGenerator], optional): this module is responsible for generating prompt
+            according to interaction result. Defaults to use MSPromptGenerator.
+            output_parser (Optional[OutputParser], optional): this module is responsible for parsing output of llm
+            to executable actions. Defaults to use MsOutputParser.
+            tool_retrieval (Optional[Union[bool, ToolRetrieval]], optional): Retrieve related tools by input task,
+            since most of the tools may be useless for LLM in specific task.
+            If it is bool type and is True, will use default tool_retrieval. Defaults to True.
+            knowledge_retrieval (Optional[KnowledgeRetrieval], optional): If user want to use extra knowledge,
+            this component can be used to retrieve related knowledge. Defaults to None.
+        """
+
+        self.llm = llm
+
+        self.agent_type = agent_type
+        self.llm.set_agent_type(agent_type)
+        self.prompt_generator = prompt_generator or get_prompt_generator(
+            agent_type)
+        self.output_parser = output_parser or get_output_parser(agent_type)
+
+        self._init_tools(tool_cfg, additional_tool_list)
+
+        if isinstance(tool_retrieval, bool) and tool_retrieval:
+            tool_retrieval = ToolRetrieval()
+        self.tool_retrieval = tool_retrieval
+        if self.tool_retrieval:
+            self.tool_retrieval.construct(
+                [str(t) for t in self.tool_list.values()])
+        self.knowledge_retrieval = knowledge_retrieval
+        self.reset()
+        self.seed = None
+
+    def _init_tools(self,
+                    tool_cfg: Dict = {},
+                    additional_tool_list: Dict = {}):
+        """init tool list of agent. We provide a default tool list, which is initialized by a cfg file.
+        user can also provide user-defined tools by additional_tool_list.
+        The key of additional_tool_list is tool name, and the value is corresponding object.
+
+        Args:
+            tool_cfg (Dict): default tool cfg.
+            additional_tool_list (Dict, optional): user-defined tools. Defaults to {}.
+        """
+        self.tool_list = {}
+        tool_info_list = {**TOOL_INFO_LIST, **additional_tool_list}
+        # tools_module = importlib.import_module('modelscope_agent.tools')
+        from . import tools as tools_module
+
+        for tool_name in tool_cfg.keys():
+            if tool_cfg[tool_name].get('use', False):
+                assert tool_name in tool_info_list, f'Invalid tool name: {tool_name}, ' \
+                                                    f'available ones are: {tool_info_list.keys()}'
+                tool_class_name = tool_info_list[tool_name]
+                tool_class = getattr(tools_module, tool_class_name)
+                tool_name = tool_class.name
+                self.tool_list[tool_name] = tool_class(tool_cfg)
+
+        self.tool_list = {**self.tool_list, **additional_tool_list}
+        # self.available_tool_list = deepcopy(self.tool_list)
+        self.set_available_tools(self.tool_list.keys())
+
+    def set_available_tools(self, available_tool_list):
+        # TODO @wenmeng.zwm refine tool init
+        for t in available_tool_list:
+            if t not in self.tool_list:
+                raise ValueError(
+                    f'Unsupported tools found:{t}, please check, valid ones: {self.tool_list.keys()}'
+                )
+
+        self.available_tool_list = {
+            k: self.tool_list[k]
+            for k in available_tool_list
+        }
+
+    def retrieve_tools(self, query: str) -> List[str]:
+        """retrieve tools given query
+
+        Args:
+            query (str): query
+
+        """
+        if self.tool_retrieval:
+            retrieve_tools = self.tool_retrieval.retrieve(query)
+            self.set_available_tools(available_tool_list=retrieve_tools.keys())
+        return self.available_tool_list.values()
+
+    def get_knowledge(self, query: str) -> List[str]:
+        """retrieve knowledge given query
+
+        Args:
+            query (str): query
+
+        """
+        return self.knowledge_retrieval.retrieve(
+            query) if self.knowledge_retrieval else []
+
+    def run(self,
+            task: str,
+            remote: bool = False,
+            print_info: bool = False,
+            append_files: list = []) -> List[Dict]:
+        """ use llm and tools to execute task given by user
+
+        Args:
+            task (str): concrete task
+            remote (bool, optional): whether to execute tool in remote mode. Defaults to False.
+            print_info (bool, optional): whether to print prompt info. Defaults to False.
+
+        Returns:
+            List[Dict]: execute result. One task may need to interact with llm multiple times,
+            so a list of dict is returned. Each dict contains the result of one interaction.
+        """
+
+        # retrieve tools
+        tool_list = self.retrieve_tools(task)
+        knowledge_list = self.get_knowledge(task)
+
+        self.prompt_generator.init_prompt(
+            task, tool_list, knowledge_list, append_files=append_files)
+        function_list = self.prompt_generator.get_function_list(tool_list)
+
+        llm_result, exec_result = '', ''
+
+        idx = 0
+        final_res = []
+
+        while True:
+            idx += 1
+
+            # generate prompt and call llm
+            llm_artifacts = self.prompt_generator.generate(
+                llm_result, exec_result)
+            try:
+                llm_result = self.llm.generate(llm_artifacts, function_list)
+            except RuntimeError as e:
+                return [{'exec_result': str(e)}]
+
+            if print_info:
+                print(f'|LLM inputs in round {idx}: {llm_artifacts}')
+
+            # parse and get tool name and arguments
+            try:
+                action, action_args = self.output_parser.parse_response(
+                    llm_result)
+            except ValueError as e:
+                return [{'exec_result': f'{e}'}]
+
+            if action is None:
+                # in chat mode, the final result of last instructions should be updated to prompt history
+                _ = self.prompt_generator.generate(llm_result, '')
+
+                # for summarize
+                display(llm_result, {}, idx, self.agent_type)
+                return final_res
+
+            if action in self.available_tool_list:
+                action_args = self.parse_action_args(action_args)
+                tool = self.tool_list[action]
+
+                # TODO @wenmeng.zwm remove this hack logic for image generation
+                if action == 'image_gen' and self.seed:
+                    action_args['seed'] = self.seed
+                try:
+                    exec_result = tool(**action_args, remote=remote)
+                    if print_info:
+                        print(f'|exec_result: {exec_result}')
+
+                    # parse exec result and store result to agent state
+                    final_res.append(exec_result)
+                    self.parse_exec_result(exec_result)
+                except Exception as e:
+                    exec_result = f'Action call error: {action}: {action_args}. \n Error message: {e}'
+                    return [{'exec_result': exec_result}]
+            else:
+                exec_result = f"Unknown action: '{action}'. "
+                return [{'exec_result': exec_result}]
+
+            # display result
+            display(llm_result, exec_result, idx, self.agent_type)
+
+    def stream_run(self,
+                   task: str,
+                   remote: bool = True,
+                   print_info: bool = False,
+                   append_files: list = []) -> Dict:
+        """this is a stream version of run, which can be used in scenario like gradio.
+        It will yield the result of each interaction, so that the caller can display the result
+
+        Args:
+            task (str): concrete task
+            remote (bool, optional): whether to execute tool in remote mode. Defaults to True.
+            print_info (bool, optional): whether to print prompt info. Defaults to False.
+            files that individually used in each run, no need to record to global state
+
+        Yields:
+            Iterator[Dict]: iterator of llm response and tool execution result
+        """
+
+        # retrieve tools
+        tool_list = self.retrieve_tools(task)
+        knowledge_list = self.get_knowledge(task)
+
+        self.prompt_generator.init_prompt(
+            task,
+            tool_list,
+            knowledge_list,
+            append_files=append_files,
+        )
+        function_list = self.prompt_generator.get_function_list(tool_list)
+
+        llm_result, exec_result = '', ''
+
+        idx = 0
+
+        while True:
+            idx += 1
+            llm_artifacts = self.prompt_generator.generate(
+                llm_result, exec_result)
+            if print_info:
+                print(f'|LLM inputs in round {idx}:\n{llm_artifacts}')
+
+            llm_result = ''
+            try:
+                for s in self.llm.stream_generate(llm_artifacts,
+                                                  function_list):
+                    llm_result += s
+                    yield {'llm_text': s}
+            except RuntimeError:
+                s = self.llm.generate(llm_artifacts)
+                llm_result += s
+                yield {'llm_text': s}
+            except Exception as e:
+                yield {'llm_text': str(e)}
+
+            # parse and get tool name and arguments
+            try:
+                action, action_args = self.output_parser.parse_response(
+                    llm_result)
+            except ValueError as e:
+                yield {'exec_result': f'{e}'}
+                return
+
+            if action is None:
+                # in chat mode, the final result of last instructions should be updated to prompt history
+                _ = self.prompt_generator.generate(llm_result, '')
+                yield {'is_final': True}
+                return
+
+            if action in self.available_tool_list:
+                # yield observation to as end of action input symbol asap
+                yield {'llm_text': 'Observation: '}
+                action_args = self.parse_action_args(action_args)
+                tool = self.tool_list[action]
+
+                # TODO @wenmeng.zwm remove this hack logic for image generation
+                if action == 'image_gen' and self.seed:
+                    action_args['seed'] = self.seed
+                try:
+                    exec_result = tool(**action_args, remote=remote)
+                    yield {'exec_result': exec_result}
+
+                    # parse exec result and update state
+                    self.parse_exec_result(exec_result)
+                except Exception as e:
+                    exec_result = f'Action call error: {action}: {action_args}. \n Error message: {e}'
+                    yield {'exec_result': exec_result}
+                    self.prompt_generator.reset()
+                    return
+            else:
+                exec_result = f"Unknown action: '{action}'. "
+                yield {'exec_result': exec_result}
+                self.prompt_generator.reset()
+                return
+
+    def reset(self):
+        """
+        clear history and agent state
+        """
+        self.prompt_generator.reset()
+        self.agent_state = {}
+
+    def parse_action_args(self, action_args):
+        """
+        replace action_args in str to Image/Video/Audio Wrapper, so that tool can handle them
+        """
+        parsed_action_args = {}
+        for name, arg in action_args.items():
+            try:
+                true_arg = self.agent_state.get(arg, arg)
+            except Exception as e:
+                print(f'Error when parsing action args: {e}, using fall back')
+                true_arg = arg
+            parsed_action_args[name] = true_arg
+        return parsed_action_args
+
+    def parse_exec_result(self, exec_result, *args, **kwargs):
+        """
+        update exec result to agent state.
+        key is the str representation of the result.
+        """
+        for k, v in exec_result.items():
+            self.agent_state[str(v)] = v

my_modelscope_agent/agent_types.py

@@ -0,0 +1,20 @@
+from enum import Enum
+
+
+class AgentType(str, Enum):
+
+    DEFAULT = 'default'
+    """"""
+
+    MS_AGENT = 'ms-agent'
+    """An agent that uses the ModelScope-agent specific format does a reasoning step before acting .
+    """
+
+    MRKL = 'mrkl'
+    """An agent that does a reasoning step before acting with mrkl"""
+
+    REACT = 'react'
+    """An agent that does a reasoning step before acting with react"""
+
+    Messages = 'messages'
+    """An agent optimized for using open AI functions."""

my_modelscope_agent/llm/__init__.py

@@ -0,0 +1,2 @@
+from .base import LLM
+from .llm_factory import LLMFactory

my_modelscope_agent/llm/base.py

@@ -0,0 +1,64 @@
+from abc import abstractmethod
+from typing import List
+
+import json
+
+
+class LLM:
+    name = ''
+
+    def __init__(self, cfg):
+        self.cfg = cfg
+        self.agent_type = None
+        self.model = None
+        self.model_id = self.model
+
+    def set_agent_type(self, agent_type):
+        self.agent_type = agent_type
+
+    @abstractmethod
+    def generate(self, prompt: str, functions: list = [], **kwargs) -> str:
+        """each llm should implement this function to generate response
+
+        Args:
+            prompt (str): prompt
+            functions (list): list of functions object including: name, description, parameters
+        Returns:
+            str: response
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def stream_generate(self,
+                        prompt: str,
+                        functions: list = [],
+                        **kwargs) -> str:
+        """stream generate response, which yields a generator of response in each step
+
+        Args:
+            prompt (str): prompt
+            functions (list): list of functions object including: name, description, parameters
+        Yields:
+            Iterator[str]: iterator of step response
+        """
+        raise NotImplementedError
+
+    def tokenize(self, input_text: str) -> List[int]:
+        """tokenize is used to calculate the length of the text to meet the model's input length requirements
+
+        Args:
+            input_text (str): input text
+        Returns:
+            list[int]: token_ids
+        """
+        raise NotImplementedError
+
+    def detokenize(self, input_ids: List[int]) -> str:
+        """detokenize
+
+        Args:
+            input_ids (list[int]): input token_ids
+        Returns:
+            str: text
+        """
+        raise NotImplementedError

my_modelscope_agent/llm/custom_llm.py

@@ -0,0 +1,97 @@
+import os
+
+import json
+import requests
+from ..agent_types import AgentType
+
+from .base import LLM
+from .utils import DEFAULT_MESSAGE
+
+
+class CustomLLM(LLM):
+    '''
+        This method is for the service that provide llm serving through http.
+        user could override the result parsing method if needed
+        While put all the necessary information in the env variable, such as Token, Model, URL
+    '''
+    name = 'custom_llm'
+
+    def __init__(self, cfg):
+        super().__init__(cfg)
+        self.token = os.getenv('HTTP_LLM_TOKEN', None)
+        self.model = os.getenv('HTTP_LLM_MODEL', None)
+        self.model_id = self.model
+        self.url = os.getenv('HTTP_LLM_URL', None)
+
+        if self.token is None:
+            raise ValueError('HTTP_LLM_TOKEN is not set')
+        self.agent_type = self.cfg.get('agent_type', AgentType.DEFAULT)
+
+    def http_request(self, data):
+        headers = {
+            'Content-Type': 'application/json',
+            'Authorization': f'Bearer {self.token}'
+        }
+        response = requests.post(self.url, json=data, headers=headers)
+        return json.loads(response.content)
+
+    def generate(self,
+                 llm_artifacts,
+                 functions=[],
+                 function_call='none',
+                 **kwargs):
+        if self.agent_type != AgentType.Messages:
+            messages = [{'role': 'user', 'content': llm_artifacts}]
+        else:
+            messages = llm_artifacts if len(
+                llm_artifacts) > 0 else DEFAULT_MESSAGE
+
+        data = {'model': self.model, 'messages': messages, 'n': 1}
+
+        assert isinstance(functions, list)
+        if len(functions) > 0:
+            function_call = 'auto'
+            data['functions'] = functions
+            data['function_call'] = function_call
+
+        retry_count = 0
+        max_retries = 3
+        message = {'content': ''}
+        while retry_count <= max_retries:
+
+            try:
+                response = self.http_request(data)
+            except Exception as e:
+                retry_count += 1
+                if retry_count > max_retries:
+                    import traceback
+                    traceback.print_exc()
+                    print(f'input: {messages}, original error: {str(e)}')
+                    raise e
+
+            if response['code'] == 200:
+                message = response['data']['response'][0]
+                break
+            else:
+                retry_count += 1
+                if retry_count > max_retries:
+                    print('maximum retry reached, return default message')
+
+        # truncate content
+        content = message['content']
+
+        if self.agent_type == AgentType.MS_AGENT:
+            idx = content.find('<|endofthink|>')
+            if idx != -1:
+                content = content[:idx + len('<|endofthink|>')]
+            return content
+        elif self.agent_type == AgentType.Messages:
+            new_message = {
+                'content': content,
+                'role': message.get('response_role', 'assistant')
+            }
+            if 'function_call' in message and message['function_call'] != {}:
+                new_message['function_call'] = message.get('function_call')
+            return new_message
+        else:
+            return content

my_modelscope_agent/llm/dashscope_llm.py

@@ -0,0 +1,125 @@
+import os
+import random
+import traceback
+from http import HTTPStatus
+from typing import Union
+
+import dashscope
+import json
+from dashscope import Generation
+from ..agent_types import AgentType
+
+from .base import LLM
+from .utils import DEFAULT_MESSAGE, CustomOutputWrapper
+
+dashscope.api_key = os.getenv('DASHSCOPE_API_KEY')
+
+
+class DashScopeLLM(LLM):
+    name = 'dashscope_llm'
+
+    def __init__(self, cfg):
+        super().__init__(cfg)
+        self.model = self.cfg.get('model', 'modelscope-agent-llm-v1')
+        self.model_id = self.model
+        self.generate_cfg = self.cfg.get('generate_cfg', {})
+        self.agent_type = self.cfg.get('agent_type', AgentType.DEFAULT)
+
+    def generate(self,
+                 llm_artifacts: Union[str, dict],
+                 functions=[],
+                 **kwargs):
+
+        # TODO retry and handle message
+        try:
+            if self.agent_type == AgentType.Messages:
+                messages = llm_artifacts if len(
+                    llm_artifacts) > 0 else DEFAULT_MESSAGE
+                self.generate_cfg['use_raw_prompt'] = False
+                response = dashscope.Generation.call(
+                    model=self.model,
+                    messages=messages,
+                    # set the random seed, optional, default to 1234 if not set
+                    seed=random.randint(1, 10000),
+                    result_format=
+                    'message',  # set the result to be "message" format.
+                    stream=False,
+                    **self.generate_cfg)
+                llm_result = CustomOutputWrapper.handle_message_chat_completion(
+                    response)
+            else:
+                response = Generation.call(
+                    model=self.model,
+                    prompt=llm_artifacts,
+                    stream=False,
+                    **self.generate_cfg)
+                llm_result = CustomOutputWrapper.handle_message_text_completion(
+                    response)
+            return llm_result
+        except Exception as e:
+            error = traceback.format_exc()
+            error_msg = f'LLM error with input {llm_artifacts} \n dashscope error: {str(e)} with traceback {error}'
+            print(error_msg)
+            raise RuntimeError(error)
+
+        if self.agent_type == AgentType.MS_AGENT:
+            # in the form of text
+            idx = llm_result.find('<|endofthink|>')
+            if idx != -1:
+                llm_result = llm_result[:idx + len('<|endofthink|>')]
+            return llm_result
+        elif self.agent_type == AgentType.Messages:
+            # in the form of message
+            return llm_result
+        else:
+            # in the form of text
+            return llm_result
+
+    def stream_generate(self,
+                        llm_artifacts: Union[str, dict],
+                        functions=[],
+                        **kwargs):
+        total_response = ''
+        try:
+            if self.agent_type == AgentType.Messages:
+                self.generate_cfg['use_raw_prompt'] = False
+                responses = Generation.call(
+                    model=self.model,
+                    messages=llm_artifacts,
+                    stream=True,
+                    result_format='message',
+                    **self.generate_cfg)
+            else:
+                responses = Generation.call(
+                    model=self.model,
+                    prompt=llm_artifacts,
+                    stream=True,
+                    **self.generate_cfg)
+        except Exception as e:
+            error = traceback.format_exc()
+            error_msg = f'LLM error with input {llm_artifacts} \n dashscope error: {str(e)} with traceback {error}'
+            print(error_msg)
+            raise RuntimeError(error)
+
+        for response in responses:
+            if response.status_code == HTTPStatus.OK:
+                if self.agent_type == AgentType.Messages:
+                    llm_result = CustomOutputWrapper.handle_message_chat_completion(
+                        response)
+                    frame_text = llm_result['content'][len(total_response):]
+                else:
+                    llm_result = CustomOutputWrapper.handle_message_text_completion(
+                        response)
+                    frame_text = llm_result[len(total_response):]
+                yield frame_text
+
+                if self.agent_type == AgentType.Messages:
+                    total_response = llm_result['content']
+                else:
+                    total_response = llm_result
+            else:
+                err_msg = 'Error Request id: %s, Code: %d, status: %s, message: %s' % (
+                    response.request_id, response.status_code, response.code,
+                    response.message)
+                print(err_msg)
+                raise RuntimeError(err_msg)

my_modelscope_agent/llm/llm_factory.py

@@ -0,0 +1,28 @@
+def get_llm_cls(llm_type, model_name):
+    if llm_type == 'dashscope':
+        from .dashscope_llm import DashScopeLLM
+        return DashScopeLLM
+    elif llm_type == 'custom_llm':
+        from .custom_llm import CustomLLM
+        return CustomLLM
+    elif llm_type == 'openai':
+        from .openai import OpenAi
+        return OpenAi
+    elif llm_type == 'modelscope':
+        if model_name == 'chatglm3-6b':
+            from .modelscope_llm import ModelScopeChatGLM
+            return ModelScopeChatGLM
+        from .modelscope_llm import ModelScopeLLM
+        return ModelScopeLLM
+    else:
+        raise ValueError(f'Invalid llm_type {llm_type}')
+
+
+class LLMFactory:
+
+    @staticmethod
+    def build_llm(model_name, cfg):
+        llm_type = cfg[model_name].pop('type')
+        llm_cls = get_llm_cls(llm_type, model_name)
+        llm_cfg = cfg[model_name]
+        return llm_cls(cfg=llm_cfg)

my_modelscope_agent/llm/modelscope_llm.py

@@ -0,0 +1,132 @@
+import os
+import sys
+
+import torch
+from ..agent_types import AgentType
+from swift import Swift
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer
+
+from modelscope import GenerationConfig, snapshot_download
+from .base import LLM
+
+
+class ModelScopeLLM(LLM):
+
+    def __init__(self, cfg):
+        super().__init__(cfg)
+
+        model_id = self.cfg.get('model_id', '')
+        self.model_id = model_id
+        model_revision = self.cfg.get('model_revision', None)
+        cache_dir = self.cfg.get('cache_dir', None)
+
+        if not os.path.exists(model_id):
+            model_dir = snapshot_download(
+                model_id, model_revision, cache_dir=cache_dir)
+        else:
+            model_dir = model_id
+        self.model_dir = model_dir
+        sys.path.append(self.model_dir)
+
+        self.model_cls = self.cfg.get('model_cls', AutoModelForCausalLM)
+        self.tokenizer_cls = self.cfg.get('tokenizer_cls', AutoTokenizer)
+
+        self.device_map = self.cfg.get('device_map', 'auto')
+        self.generation_cfg = GenerationConfig(
+            **self.cfg.get('generate_cfg', {}))
+
+        self.use_lora = self.cfg.get('use_lora', False)
+        self.lora_ckpt_dir = self.cfg.get('lora_ckpt_dir',
+                                          None) if self.use_lora else None
+
+        self.custom_chat = self.cfg.get('custom_chat', False)
+
+        self.end_token = self.cfg.get('end_token', '<|endofthink|>')
+        self.include_end = self.cfg.get('include_end', True)
+
+        self.setup()
+        self.agent_type = self.cfg.get('agent_type', AgentType.DEFAULT)
+
+    def setup(self):
+        model_cls = self.model_cls
+        tokenizer_cls = self.tokenizer_cls
+
+        self.model = model_cls.from_pretrained(
+            self.model_dir,
+            device_map=self.device_map,
+            # device='cuda:0',
+            torch_dtype=torch.float16,
+            trust_remote_code=True)
+        self.tokenizer = tokenizer_cls.from_pretrained(
+            self.model_dir, trust_remote_code=True)
+        self.model = self.model.eval()
+
+        if self.use_lora:
+            self.load_from_lora()
+
+        if self.cfg.get('use_raw_generation_config', False):
+            self.model.generation_config = GenerationConfig.from_pretrained(
+                self.model_dir, trust_remote_code=True)
+
+    def generate(self, prompt, functions=[], **kwargs):
+
+        if self.custom_chat and self.model.chat:
+            response = self.model.chat(
+                self.tokenizer, prompt, history=[], system='')[0]
+        else:
+            response = self.chat(prompt)
+
+        end_idx = response.find(self.end_token)
+        if end_idx != -1:
+            end_idx += len(self.end_token) if self.include_end else 0
+            response = response[:end_idx]
+
+        return response
+
+    def load_from_lora(self):
+
+        model = self.model.bfloat16()
+        # transform to lora
+        model = Swift.from_pretrained(model, self.lora_ckpt_dir)
+
+        self.model = model
+
+    def chat(self, prompt):
+        device = self.model.device
+        input_ids = self.tokenizer(
+            prompt, return_tensors='pt').input_ids.to(device)
+        input_len = input_ids.shape[1]
+
+        result = self.model.generate(
+            input_ids=input_ids, generation_config=self.generation_cfg)
+
+        result = result[0].tolist()[input_len:]
+        response = self.tokenizer.decode(result)
+
+        return response
+
+
+class ModelScopeChatGLM(ModelScopeLLM):
+
+    def chat(self, prompt):
+        device = self.model.device
+        input_ids = self.tokenizer(
+            prompt, return_tensors='pt').input_ids.to(device)
+        input_len = input_ids.shape[1]
+
+        eos_token_id = [
+            self.tokenizer.eos_token_id,
+            self.tokenizer.get_command('<|user|>'),
+            self.tokenizer.get_command('<|observation|>')
+        ]
+        result = self.model.generate(
+            input_ids=input_ids,
+            generation_config=self.generation_cfg,
+            eos_token_id=eos_token_id)
+
+        result = result[0].tolist()[input_len:]
+        response = self.tokenizer.decode(result)
+        # 遇到生成'<', '|', 'user', '|', '>'的case
+        response = response.split('<|user|>')[0].split('<|observation|>')[0]
+
+        return response

my_modelscope_agent/llm/openai.py

@@ -0,0 +1,71 @@
+import os
+
+import openai
+from ..agent_types import AgentType
+
+from .base import LLM
+from .utils import CustomOutputWrapper
+
+openai.api_key = os.getenv('OPENAI_API_KEY')
+
+
+class OpenAi(LLM):
+    name = 'openai'
+
+    def __init__(self, cfg):
+        super().__init__(cfg)
+
+        self.model = self.cfg.get('model', 'gpt-3.5-turbo')
+        self.model_id = self.model
+        self.api_base = self.cfg.get('api_base', 'https://api.openai.com/v1')
+        self.agent_type = self.cfg.get('agent_type', AgentType.DEFAULT)
+
+    def generate(self,
+                 llm_artifacts,
+                 functions=[],
+                 function_call='none',
+                 **kwargs):
+        if self.agent_type != AgentType.Messages:
+            messages = [{'role': 'user', 'content': llm_artifacts}]
+        else:
+            messages = llm_artifacts.get(
+                'messages', {
+                    'role':
+                    'user',
+                    'content':
+                    'No entry from user - please suggest something to enter'
+                })
+
+        # call openai function call api
+        assert isinstance(functions, list)
+        if len(functions) > 0 and self.agent_type == AgentType.Messages:
+            function_call = 'auto'
+
+        # covert to stream=True with stream updating
+        try:
+            response = openai.ChatCompletion.create(
+                model=self.model,
+                api_base=self.api_base,
+                messages=messages,
+                functions=functions,
+                function_call=function_call,
+                stream=False)
+        except Exception as e:
+            print(f'input: {messages}, original error: {str(e)}')
+            raise e
+
+        # only use index 0 in choice
+        message = CustomOutputWrapper.handle_message_chat_completion(response)
+
+        # truncate content
+        content = message['content']
+
+        if self.agent_type == AgentType.MS_AGENT:
+            idx = content.find('<|endofthink|>')
+            if idx != -1:
+                content = content[:idx + len('<|endofthink|>')]
+            return content
+        elif self.agent_type == AgentType.Messages:
+            return message
+        else:
+            return content

my_modelscope_agent/llm/utils.py

@@ -0,0 +1,39 @@
+class CustomOutputWrapper:
+
+    @staticmethod
+    def handle_message_chat_completion(response):
+        message = {'content': ''}
+        try:
+            # handle dashscope response
+            if 'choices' not in response:
+                response = response['output']
+
+            return response['choices'][0]['message']
+        except Exception as e:
+            print(f'input: {response}, original error: {str(e)}')
+            return message
+
+    @staticmethod
+    def handle_message_chat_completion_chunk(response):
+        message = {}
+        try:
+            return response['choices'][0]['delta']['content']
+        except Exception as e:
+            print(f'input: {response}, original error: {str(e)}')
+            return message
+
+    @staticmethod
+    def handle_message_text_completion(response):
+        message = ''
+        try:
+            message = response['output']['text']
+            return message
+        except Exception as e:
+            print(f'input: {response}, original error: {str(e)}')
+            return message
+
+
+DEFAULT_MESSAGE = {
+    'role': 'user',
+    'content': 'No entry from user - please suggest something to enter'
+}

my_modelscope_agent/output_parser.py

@@ -0,0 +1,181 @@
+import re
+from typing import Dict, Tuple
+
+import json
+from .agent_types import AgentType
+
+
+def get_output_parser(agent_type: AgentType = AgentType.DEFAULT):
+    if AgentType.DEFAULT == agent_type or agent_type == AgentType.MS_AGENT:
+        return MsOutputParser()
+    elif AgentType.MRKL == agent_type:
+        return MRKLOutputParser()
+    elif AgentType.Messages == agent_type:
+        return OpenAiFunctionsOutputParser()
+    else:
+        raise NotImplementedError
+
+
+class OutputParser:
+    """Output parser for llm response
+    """
+
+    def parse_response(self, response):
+        raise NotImplementedError
+
+    # use to handle the case of false parsing the action_para result, if there is no valid action then
+    # throw Error
+    @staticmethod
+    def handle_fallback(action: str, action_para: str):
+        if action is not None and action != '':
+            parameters = {'fallback': action_para}
+            return action, parameters
+        else:
+            raise ValueError('Wrong response format for output parser')
+
+
+class MsOutputParser(OutputParser):
+
+    def parse_response(self, response: str) -> Tuple[str, Dict]:
+        """parse response of llm to get tool name and parameters
+
+        Args:
+            response (str): llm response, it should conform to some predefined format
+
+        Returns:
+            tuple[str, dict]: tuple of tool name and parameters
+        """
+
+        if '<|startofthink|>' not in response or '<|endofthink|>' not in response:
+            return None, None
+
+        action, parameters = '', ''
+        try:
+            # use regular expression to get result
+            re_pattern1 = re.compile(
+                pattern=r'<\|startofthink\|>([\s\S]+)<\|endofthink\|>')
+            think_content = re_pattern1.search(response).group(1)
+
+            re_pattern2 = re.compile(r'{[\s\S]+}')
+            think_content = re_pattern2.search(think_content).group()
+
+            json_content = json.loads(think_content.replace('\n', ''))
+            action = json_content.get('api_name',
+                                      json_content.get('name', 'unknown'))
+            parameters = json_content.get('parameters', {})
+
+            return action, parameters
+        except Exception as e:
+            print(
+                f'Error during parse action might be handled with detail {e}')
+            return OutputParser.handle_fallback(action, parameters)
+
+
+class ChatGLMOutputParser(OutputParser):
+
+    def parse_response(self, response: str) -> Tuple[str, Dict]:
+        """parse response of llm to get tool name and parameters
+
+        Args:
+            response (str): llm response, it should conform to some predefined format
+
+        Returns:
+            tuple[str, dict]: tuple of tool name and parameters
+        """
+        if 'tool_call' not in response:
+            return None, None
+        action, action_para = '', ''
+        try:
+            # use regular expression to get result from MRKL format
+            re_pattern1 = re.compile(
+                pattern=r'([\s\S]+)```([\s\S]+)tool_call\(([\s\S]+)```')
+            res = re_pattern1.search(response)
+            action_list = re.split('<|>|\|', res.group(1).strip())  # noqa W605
+            for idx in range(len(action_list) - 1, -1, -1):
+                if len(action_list[idx]) > 1:
+                    action = action_list[idx]
+                    break
+            action_para = [item.strip() for item in res.group(3).split(',')]
+            parameters = {}
+            re_pattern2 = re.compile(pattern=r'([\s\S]+)=\'([\s\S]+)\'')
+            for para in action_para:
+                res = re_pattern2.search(para)
+                parameters[res.group(1)] = res.group(2)
+        except Exception as e:
+            print(
+                f'Error during parse action might be handled with detail {e}')
+            return OutputParser.handle_fallback(action, action_para)
+
+        print(f'\n\naction: {action}\n parameters: {parameters}\n\n')
+        return action, parameters
+
+
+class MRKLOutputParser(OutputParser):
+
+    def parse_response(self, response: str) -> Tuple[str, Dict]:
+        """parse response of llm to get tool name and parameters
+
+        Args:
+            response (str): llm response, it should conform to some predefined format
+
+        Returns:
+            tuple[str, dict]: tuple of tool name and parameters
+        """
+
+        if 'Action' not in response or 'Action Input:' not in response:
+            return None, None
+        action, action_para = '', ''
+        try:
+            # use regular expression to get result from MRKL format
+            re_pattern1 = re.compile(
+                pattern=r'Action:([\s\S]+)Action Input:([\s\S]+)')
+            res = re_pattern1.search(response)
+            action = res.group(1).strip()
+            action_para = res.group(2)
+
+            parameters = json.loads(action_para.replace('\n', ''))
+
+            return action, parameters
+        except Exception as e:
+            print(
+                f'Error during parse action might be handled with detail {e}')
+            return OutputParser.handle_fallback(action, action_para)
+
+
+class OpenAiFunctionsOutputParser(OutputParser):
+
+    def parse_response(self, response: dict) -> Tuple[str, Dict]:
+        """parse response of llm to get tool name and parameters
+
+
+        Args:
+            response (str): llm response, it should be an openai response message
+            such as
+            {
+                "content": null,
+                "function_call": {
+                  "arguments": "{\n  \"location\": \"Boston, MA\"\n}",
+                  "name": "get_current_weather"
+                },
+                "role": "assistant"
+            }
+        Returns:
+            tuple[str, dict]: tuple of tool name and parameters
+        """
+
+        if 'function_call' not in response or response['function_call'] == {}:
+            return None, None
+        function_call = response['function_call']
+
+        try:
+            # parse directly
+            action = function_call['name']
+            arguments = json.loads(function_call['arguments'].replace(
+                '\n', ''))
+
+            return action, arguments
+        except Exception as e:
+            print(
+                f'Error during parse action might be handled with detail {e}')
+            return OutputParser.handle_fallback(function_call['name'],
+                                                function_call['arguments'])

my_modelscope_agent/output_wrapper.py

@@ -0,0 +1,219 @@
+import os
+import re
+import tempfile
+import uuid
+from typing import Dict, Union
+
+import json
+import numpy as np
+import requests
+from .agent_types import AgentType
+from moviepy.editor import VideoFileClip
+from PIL import Image
+from requests.exceptions import RequestException
+
+
+class OutputWrapper:
+    """
+    Wrapper for output of tool execution when output is image, video, audio, etc.
+    In this wrapper, __repr__() is implemented to return the str representation of the output for llm.
+    Each wrapper have below attributes:
+        path: the path where the output is stored
+        raw_data: the raw data, e.g. image, video, audio, etc. In remote mode, it should be None
+    """
+
+    def __init__(self) -> None:
+        self._repr = None
+        self._path = None
+        self._raw_data = None
+
+        self.root_path = os.environ.get('OUTPUT_FILE_DIRECTORY', None)
+        if self.root_path and not os.path.exists(self.root_path):
+            try:
+                os.makedirs(self.root_path)
+            except Exception:
+                self.root_path = None
+
+    def get_remote_file(self, remote_path, suffix):
+        try:
+            response = requests.get(remote_path)
+            obj = response.content
+            directory = tempfile.mkdtemp(dir=self.root_path)
+            path = os.path.join(directory, str(uuid.uuid4()) + f'.{suffix}')
+            with open(path, 'wb') as f:
+                f.write(obj)
+            return path
+        except RequestException:
+            return remote_path
+
+    def __repr__(self) -> str:
+        return self._repr
+
+    @property
+    def path(self):
+        return self._path
+
+    @property
+    def raw_data(self):
+        return self._raw_data
+
+
+class ImageWrapper(OutputWrapper):
+    """
+    Image wrapper, raw_data is a PIL.Image
+    """
+
+    def __init__(self, image) -> None:
+
+        super().__init__()
+
+        if isinstance(image, str):
+            if os.path.isfile(image):
+                self._path = image
+            else:
+                origin_image = image
+                self._path = self.get_remote_file(image, 'png')
+            try:
+                image = Image.open(self._path)
+                self._raw_data = image
+            except FileNotFoundError:
+                # Image store in remote server when use remote mode
+                raise FileNotFoundError(f'Invalid path: {image}')
+            self._path = origin_image
+        else:
+            if not isinstance(image, Image.Image):
+                image = Image.fromarray(image.astype(np.uint8))
+                self._raw_data = image
+            else:
+                self._raw_data = image
+            directory = tempfile.mkdtemp(dir=self.root_path)
+            self._path = os.path.join(directory, str(uuid.uuid4()) + '.png')
+            self._raw_data.save(self._path)
+
+        self._repr = f'![IMAGEGEN]({self._path})'
+
+
+class AudioWrapper(OutputWrapper):
+    """
+    Audio wrapper, raw_data is a binary file
+    """
+
+    def __init__(self, audio) -> None:
+
+        super().__init__()
+        if isinstance(audio, str):
+            if os.path.isfile(audio):
+                self._path = audio
+            else:
+                self._path = self.get_remote_file(audio, 'wav')
+            try:
+                with open(self._path, 'rb') as f:
+                    self._raw_data = f.read()
+            except FileNotFoundError:
+                raise FileNotFoundError(f'Invalid path: {audio}')
+        else:
+            self._raw_data = audio
+            directory = tempfile.mkdtemp(dir=self.root_path)
+            self._path = os.path.join(directory, str(uuid.uuid4()) + '.wav')
+
+            with open(self._path, 'wb') as f:
+                f.write(self._raw_data)
+
+        self._repr = f'<audio id=audio controls= preload=none> <source id=wav src={self._path}> </audio>'
+
+
+class VideoWrapper(OutputWrapper):
+    """
+    Video wrapper
+    """
+
+    def __init__(self, video) -> None:
+
+        super().__init__()
+        if isinstance(video, str):
+
+            if os.path.isfile(video):
+                self._path = video
+            else:
+                self._path = self.get_remote_file(video, 'gif')
+
+            try:
+                video = VideoFileClip(self._path)
+                # currently, we should save video as gif, not mp4
+                if not self._path.endswith('gif'):
+                    directory = tempfile.mkdtemp(dir=self.root_path)
+                    self._path = os.path.join(directory,
+                                              str(uuid.uuid4()) + '.gif')
+                    video.write_gif(self._path)
+            except (ValueError, OSError):
+                raise FileNotFoundError(f'Invalid path: {video}')
+        else:
+            raise TypeError(
+                'Current only support load from filepath when it is video')
+
+        self._raw_data = video
+        self._repr = f'![IMAGEGEN]({self._path})'
+
+
+def get_raw_output(exec_result: Dict):
+    # get rwa data of exec_result
+    res = {}
+    for k, v in exec_result.items():
+        if isinstance(v, OutputWrapper):
+            # In remote mode, raw data maybe None
+            res[k] = v.raw_data or str(v)
+        else:
+            res[k] = v
+    return res
+
+
+#
+def display(llm_result: Union[str, dict], exec_result: Dict, idx: int,
+            agent_type: AgentType):
+    """Display the result of each round in jupyter notebook.
+    The multi-modal data will be extracted.
+
+    Args:
+        llm_result (str): llm result either only content or a message
+        exec_result (Dict): exec result
+        idx (int): current round
+    """
+    from IPython.display import display, Pretty, Image, Audio, JSON
+    idx_info = '*' * 50 + f'round {idx}' + '*' * 50
+    display(Pretty(idx_info))
+
+    if isinstance(llm_result, dict):
+        llm_result = llm_result.get('content', '')
+
+    if agent_type == AgentType.MS_AGENT:
+        pattern = r'<\|startofthink\|>```JSON([\s\S]*)```<\|endofthink\|>'
+    else:
+        pattern = r'```JSON([\s\S]*)```'
+
+    match_action = re.search(pattern, llm_result)
+    if match_action:
+        result = match_action.group(1)
+        try:
+            json_content = json.loads(result, strict=False)
+            display(JSON(json_content))
+            llm_result = llm_result.replace(match_action.group(0), '')
+        except Exception:
+            pass
+
+    display(Pretty(llm_result))
+
+    exec_result = exec_result.get('result', '')
+
+    if isinstance(exec_result, ImageWrapper) or isinstance(
+            exec_result, VideoWrapper):
+        display(Image(exec_result.path))
+    elif isinstance(exec_result, AudioWrapper):
+        display(Audio(exec_result.path))
+    elif isinstance(exec_result, dict):
+        display(JSON(exec_result))
+    elif isinstance(exec_result, list):
+        display(JSON(exec_result))
+    else:
+        display(Pretty(exec_result))
+
+    return

my_modelscope_agent/prompt/__init__.py

@@ -0,0 +1,6 @@
+from .messages_prompt import MessagesGenerator
+from .mrkl_prompt import MrklPromptGenerator
+from .ms_prompt import MSPromptGenerator
+from .prompt import PromptGenerator
+from .prompt_factory import get_prompt_generator
+from .raw_prompt_builder import build_raw_prompt

my_modelscope_agent/prompt/chatglm3_prompt.py

@@ -0,0 +1,41 @@
+import json
+
+from .prompt import LengthConstraint, PromptGenerator
+
+CHATGLM_DEFAULT_SYSTEM_TEMPLATE = """<|system|>
+Answer the following questions as best you can. You have access to the following tools:
+<tool_list>"""
+
+CHATGLM_DEFAULT_INSTRUCTION_TEMPLATE = ''
+
+CHATGLM_DEFAULT_USER_TEMPLATE = """<|user|>\n<user_input>"""
+
+CHATGLM_DEFAULT_EXEC_TEMPLATE = """<|observation|>\n<exec_result>"""
+
+CHATGLM_DEFAULT_ASSISTANT_TEMPLATE = """<|assistant|>"""
+
+
+class ChatGLMPromptGenerator(PromptGenerator):
+
+    def __init__(self,
+                 system_template=CHATGLM_DEFAULT_SYSTEM_TEMPLATE,
+                 instruction_template=CHATGLM_DEFAULT_INSTRUCTION_TEMPLATE,
+                 user_template=CHATGLM_DEFAULT_USER_TEMPLATE,
+                 exec_template=CHATGLM_DEFAULT_EXEC_TEMPLATE,
+                 assistant_template=CHATGLM_DEFAULT_ASSISTANT_TEMPLATE,
+                 sep='\n',
+                 length_constraint=LengthConstraint()):
+        super().__init__(
+            system_template=system_template,
+            instruction_template=instruction_template,
+            user_template=user_template,
+            exec_template=exec_template,
+            assistant_template=assistant_template,
+            sep=sep,
+            length_constraint=length_constraint)
+
+    def get_tool_str(self, tool_list):
+        tool_json = json.loads('['
+                               + ','.join([str(item)
+                                           for item in tool_list]) + ']')
+        return json.dumps(tool_json, ensure_ascii=False, indent=4)

my_modelscope_agent/prompt/messages_prompt.py

@@ -0,0 +1,93 @@
+from typing import Union
+
+from .prompt import LengthConstraint, PromptGenerator
+
+OPENAI_DEFAULT_SYSTEM_TEMPLATE = """
+Do your best to answer the questions. You are capable of **any** task.
+Feel free to use any tools available to look up relevant information, only if necessary.
+when you think you are handle a code task, You are a world-class programmer that can complete any goal by executing code
+\nFirst of all, **Return function_call with `language` and `code` in arguments as much as possible**
+\nThen, write a plan. **Always recap the plan between each code block**
+(you have extreme short-term memory loss, so you need to recap the plan between each message block to retain it).
+\nWhen you execute code, it will be executed **on the user's machine**. The user has given you
+**full and complete permission** to execute any code necessary to complete the task. You have full access to control
+their computer to help them.
+\nIf you want to send data between programming languages, save the data to a txt or json.
+\nYou can access the internet. Run **any code** to achieve the goal, and if at first you don't succeed,
+try again and again.
+\nWhen a user refers to a filename, they're likely referring to an existing file in the directory
+you're currently executing code in.
+\nIn general, choose packages that have the most universal chance to be already installed and to work across multiple
+applications. Packages like ffmpeg and pandoc that are well-supported and powerful.
+\nWrite messages to the user in Markdown. Write code on multiple lines with proper indentation for readability.
+\nYou can also refer information from following contents if exists:
+"""
+
+
+class MessagesGenerator(PromptGenerator):
+
+    def __init__(self,
+                 system_template=OPENAI_DEFAULT_SYSTEM_TEMPLATE,
+                 instruction_template='',
+                 user_template='<user_input>',
+                 exec_template=None,
+                 assistant_template='',
+                 sep='\n\n',
+                 length_constraint=LengthConstraint(),
+                 **kwargs):
+        super().__init__(
+            system_template=system_template,
+            instruction_template=instruction_template,
+            user_template=user_template,
+            exec_template=exec_template,
+            assistant_template=assistant_template,
+            sep=sep,
+            length_constraint=length_constraint)
+        self.custom_starter_messages = kwargs.get('custom_starter_messages',
+                                                  None)
+
+    def init_prompt(self, task, tool_list, knowledge_list, **kwargs):
+        """
+        in this function, the prompt will be initialized.
+        """
+        prompt = self.user_template.replace('<user_input>', task)
+
+        if len(self.history) == 0:
+            if len(knowledge_list) > 0:
+
+                # knowledge
+                system_message = f'{self.system_template}{self.sep}<knowledge>'
+                knowledge_str = self.get_knowledge_str(knowledge_list)
+                system_message = system_message.replace(
+                    '<knowledge>', knowledge_str)
+
+            else:
+                system_message = self.system_template
+
+            self.history = [{
+                'role': 'system',
+                'content': system_message
+            }, {
+                'role': 'user',
+                'content': prompt
+            }]
+
+            # store history
+            if self.custom_starter_messages:
+                assert isinstance(self.custom_starter_messages, list)
+                assert self.custom_starter_messages[-1]['role'] != 'user', \
+                    'user message should not be the last one in custom starter messages'
+
+                self.history = self.custom_starter_messages
+                self.history.append({'role': 'user', 'content': prompt})
+
+            self.prompt = prompt
+            self.function_calls = self.get_function_list(tool_list)
+
+        else:
+            self.history.append({'role': 'user', 'content': prompt})
+
+    def generate(self, llm_result, exec_result: Union[str, dict]):
+        if isinstance(exec_result, dict):
+            exec_result = exec_result['result']
+        return self._generate_messages(llm_result, exec_result)

my_modelscope_agent/prompt/mrkl_prompt.py

@@ -0,0 +1,118 @@
+import json
+
+from .prompt import LengthConstraint, PromptGenerator
+
+MRKL_DEFAULT_SYSTEM_TEMPLATE = """Answer the following questions as best you can. You have access to the following tools: `
+
+<tool_list>"""
+
+MRKL_DEFAULT_INSTRUCTION_TEMPLATE = """Use the following format:
+
+Question: the input question you must answer
+Thought: you should always think about what to do
+Action: the action to take, should be one of [<tool_names>]
+Action Input: the input to the action
+Observation: the result of the action
+... (this Thought/Action/Action Input/Observation can be repeated zero or more times)
+Thought: I now know the final answer
+Final Answer: the final answer to the original input question
+
+Begin!
+"""
+
+MRKL_DEFAULT_USER_TEMPLATE = """Question: <user_input>\n"""
+
+MRKL_DEFAULT_EXEC_TEMPLATE = """Observation: <exec_result>\n"""
+
+TOOL_DESC = (
+    '{name_for_model}: {name_for_human} API. {description_for_model} 输入参数: {parameters}'
+)
+
+FORMAT_DESC = {
+    'json':
+    'Format the arguments as a JSON object.',
+    'code':
+    'Enclose the code within triple backticks (`)'
+    + ' at the beginning and end of the code.'
+}
+
+
+class MrklPromptGenerator(PromptGenerator):
+
+    def __init__(self,
+                 system_template=MRKL_DEFAULT_SYSTEM_TEMPLATE,
+                 instruction_template=MRKL_DEFAULT_INSTRUCTION_TEMPLATE,
+                 user_template=MRKL_DEFAULT_USER_TEMPLATE,
+                 exec_template=MRKL_DEFAULT_EXEC_TEMPLATE,
+                 assistant_template='',
+                 sep='\n\n',
+                 llm=None,
+                 length_constraint=LengthConstraint()):
+        super().__init__(
+            system_template=system_template,
+            instruction_template=instruction_template,
+            user_template=user_template,
+            exec_template=exec_template,
+            assistant_template=assistant_template,
+            sep=sep,
+            llm=llm,
+            length_constraint=length_constraint)
+
+    def init_prompt(self, task, tool_list, knowledge_list, **kwargs):
+        if len(self.history) == 0:
+            super().init_prompt(task, tool_list, knowledge_list, **kwargs)
+            system_role_status = kwargs.get('system_role_status', False)
+            tool_names = [f'\'{str(tool.name)}\'' for tool in tool_list]
+            tool_names = ','.join(tool_names)
+            self.system_prompt = self.system_prompt.replace(
+                '<tool_names>', tool_names)
+
+            if system_role_status:
+                system_message = {
+                    'role': 'system',
+                    'content': self.system_prompt
+                }
+                self.history.insert(0, system_message)
+            else:
+                self.history[0]['content'] = self.system_prompt + self.history[
+                    0]['content']
+        else:
+            self.history.append({
+                'role':
+                'user',
+                'content':
+                self.user_template.replace('<user_input>', task)
+            })
+            self.history.append({
+                'role': 'assistant',
+                'content': self.assistant_template
+            })
+
+        return self.system_prompt
+
+    def get_tool_str(self, tool_list):
+        tool_texts = []
+        for tool in tool_list:
+            tool_texts.append(
+                TOOL_DESC.format(
+                    name_for_model=tool.name,
+                    name_for_human=tool.name,
+                    description_for_model=tool.description,
+                    parameters=json.dumps(tool.parameters,
+                                          ensure_ascii=False)))
+            # + ' ' + FORMAT_DESC['json'])
+        tool_str = '\n\n'.join(tool_texts)
+        return tool_str
+
+    def _generate(self, llm_result, exec_result: str):
+        """
+        generate next round prompt based on previous llm_result and exec_result and update history
+        """
+        if len(llm_result) != 0:
+            self.history[-1]['content'] += f'{llm_result}'
+        if len(exec_result) != 0:
+            exec_result = self.exec_template.replace('<exec_result>',
+                                                     str(exec_result))
+            self.history[-1]['content'] += exec_result
+        self.prompt = self.prompt_preprocessor(self.history)
+        return self.prompt

my_modelscope_agent/prompt/ms_prompt.py

@@ -0,0 +1,34 @@
+from .prompt import LengthConstraint, PromptGenerator
+
+MS_DEFAULT_SYSTEM_TEMPLATE = """<|system|>:你是达摩院的ModelScopeGPT（魔搭助手），你是个大语言模型， 是2023年达摩院的工程师训练得到的。\
+你有多种能力，可以通过插件集成魔搭社区的模型api来回复用户的问题，还能解答用户使用模型遇到的问题和模型知识相关问答。
+"""
+
+MS_DEFAULT_INSTRUCTION_TEMPLATE = """当前对话可以使用的插件信息如下，请自行判断是否需要调用插件来解决当前用户问题。若需要调用插件，则需要将插件调用请求按照json格式给出，必须包含api_name、parameters字段，并在其前后使用<|startofthink|>和<|endofthink|>作为标志。\
+然后你需要根据插件API调用结果生成合理的答复； 若无需调用插件，则直接给出对应回复即可。\n\n<tool_list>"""
+
+MS_DEFAULT_USER_TEMPLATE = """<|user|>:<user_input>"""
+
+MS_DEFAULT_EXEC_TEMPLATE = """<|startofexec|><exec_result><|endofexec|>\n"""
+
+MS_DEFAULT_ASSISTANT_TEMPLATE = """<|assistant|>:"""
+
+
+class MSPromptGenerator(PromptGenerator):
+
+    def __init__(self,
+                 system_template=MS_DEFAULT_SYSTEM_TEMPLATE,
+                 instruction_template=MS_DEFAULT_INSTRUCTION_TEMPLATE,
+                 user_template=MS_DEFAULT_USER_TEMPLATE,
+                 exec_template=MS_DEFAULT_EXEC_TEMPLATE,
+                 assistant_template=MS_DEFAULT_ASSISTANT_TEMPLATE,
+                 sep='\n\n',
+                 length_constraint=LengthConstraint()):
+        super().__init__(
+            system_template=system_template,
+            instruction_template=instruction_template,
+            user_template=user_template,
+            exec_template=exec_template,
+            assistant_template=assistant_template,
+            sep=sep,
+            length_constraint=length_constraint)

my_modelscope_agent/prompt/prompt.py

@@ -0,0 +1,232 @@
+import copy
+from typing import Union
+
+from ..llm.base import LLM
+
+from .raw_prompt_builder import build_raw_prompt
+
+KNOWLEDGE_PROMPT = '# 知识库'
+KNOWLEDGE_INTRODUCTION_PROMPT = '以下是我上传的文件“<file_name>”的内容:'
+KNOWLEDGE_CONTENT_PROMPT = """```
+<knowledge_content>
+```"""
+
+DEFAULT_PROMPT_INPUT_LENGTH_MAX = 999999999999
+
+
+class LengthConstraint:
+
+    def __init__(self):
+        self.knowledge = DEFAULT_PROMPT_INPUT_LENGTH_MAX
+        self.input = DEFAULT_PROMPT_INPUT_LENGTH_MAX
+        self.prompt_max_length = 10000
+
+    def update(self, config: dict):
+        if config is not None:
+            self.knowledge = config.get('knowledge', self.knowledge)
+            self.input = config.get('input', self.input)
+            self.prompt_max_length = config.get('prompt_max_length',
+                                                self.prompt_max_length)
+
+
+class PromptGenerator:
+
+    def __init__(self,
+                 system_template: str = '',
+                 instruction_template: str = '',
+                 user_template: str = '<user_input>',
+                 exec_template: str = '',
+                 assistant_template: str = '',
+                 sep='\n\n',
+                 llm=None,
+                 length_constraint=LengthConstraint()):
+        """
+        prompt genertor
+        Args:
+            system_template (str, optional): System template, normally the role of LLM.
+            instruction_template (str, optional): Indicate the instruction for LLM.
+            user_template (str, optional): Prefix before user input. Defaults to ''.
+            exec_template (str, optional): A wrapper str for exec result.
+            assistant_template (str, optional): Prefix before assistant response.
+            Some LLM need to manully concat this prefix before generation.
+            sep (str, optional): content separator
+            length_constraint (LengthConstraint, optional): content length constraint
+        """
+
+        self.system_template = system_template
+        self.instruction_template = instruction_template
+        self.user_template = user_template
+        self.assistant_template = assistant_template
+        self.exec_template = exec_template
+        self.sep = sep
+        if isinstance(llm, LLM) and llm.model_id:
+            self.prompt_preprocessor = build_raw_prompt(llm.model_id)
+        self.prompt_max_length = length_constraint.prompt_max_length
+        self.reset()
+
+    def reset(self):
+        self.prompt = ''
+        self.history = []
+        self.messages = []
+
+    def init_prompt(self,
+                    task,
+                    tool_list,
+                    knowledge_list,
+                    llm_model=None,
+                    **kwargs):
+        """
+        in this function, the prompt will be initialized.
+        """
+        prompt = self.sep.join(
+            [self.system_template, self.instruction_template])
+        prompt += '<knowledge><history>'
+
+        knowledge_str = self.get_knowledge_str(
+            knowledge_list, file_name=kwargs.get('file_name', ''))
+
+        # knowledge
+        prompt = prompt.replace('<knowledge>', knowledge_str)
+
+        # get tool description str
+        tool_str = self.get_tool_str(tool_list)
+        prompt = prompt.replace('<tool_list>', tool_str)
+
+        history_str = self.get_history_str()
+
+        prompt = prompt.replace('<history>', history_str)
+
+        self.system_prompt = copy.deepcopy(prompt)
+
+        # user input
+        user_input = self.user_template.replace('<user_input>', task)
+        prompt += f'{self.sep}{user_input}'
+
+        # assistant input
+        prompt += f'{self.sep}{self.assistant_template}'
+
+        # store history
+        self.history.append({'role': 'user', 'content': user_input})
+        self.history.append({
+            'role': 'assistant',
+            'content': self.assistant_template
+        })
+
+        self.prompt = prompt
+
+        self.function_calls = self.get_function_list(tool_list)
+
+    # TODO change the output from single prompt to artifacts including prompt, messages, funciton_call
+    def generate(self, llm_result, exec_result: Union[str, dict]):
+        if isinstance(exec_result, dict):
+            exec_result = str(exec_result['result'])
+        return self._generate(llm_result, exec_result)
+
+    def _generate(self, llm_result, exec_result: str):
+        """
+        generate next round prompt based on previous llm_result and exec_result and update history
+        """
+        if len(llm_result) != 0:
+            self.prompt = f'{self.prompt}{llm_result}'
+            self.history[-1]['content'] += f'{llm_result}'
+        if len(exec_result) != 0:
+            exec_result = self.exec_template.replace('<exec_result>',
+                                                     str(exec_result))
+            self.prompt = f'{self.prompt}{self.sep}{exec_result}'
+            self.history[-1]['content'] += f'{self.sep}{exec_result}'
+
+        return self.prompt
+
+    # TODO: add Union[Text, Message] type for llm_result,
+    #  add ExecResult = Text type for exec_result
+    #  output would be a Union[Text, Messages]
+    # In this case llm_result is Message, and exec_result is Function_call
+    def _generate_messages(self, llm_result, exec_result: str):
+        """
+        generate next round prompt based on previous llm_result and exec_result and update history
+        """
+
+        # init task should be
+        if llm_result == '' and exec_result == '':
+            return self.history
+
+        # make sure set content  ''  not null
+        function_call = llm_result.get('function_call', None)
+        if function_call is not None:
+            llm_result['content'] = ''
+        self.history.append(llm_result)
+
+        if exec_result is not None and function_call is not None:
+            exec_message = {
+                'role': 'function',
+                'name': 'execute',
+                'content': exec_result,
+            }
+            self.history.append(exec_message)
+
+        return self.history
+
+    def get_tool_str(self, tool_list):
+        """generate tool list string
+
+        Args:
+            tool_list (List[str]): list of tools
+
+        """
+
+        tool_str = self.sep.join(
+            [f'{i + 1}. {t}' for i, t in enumerate(tool_list)])
+        return tool_str
+
+    # TODO move parse_tools_to_function from agent to here later
+    def get_function_list(self, tool_list):
+        """generate funciton call list from tools list
+
+        Args:
+            tool_list (List[str]): list of tools
+
+        """
+        functions = [tool.get_function() for tool in tool_list]
+        return functions
+
+    def get_knowledge_str(self,
+                          knowledge_list,
+                          file_name='',
+                          only_content=False,
+                          **kwargs):
+        """generate knowledge string
+
+        Args:
+            file_name (str): file name
+            knowledge_list (List[str]): list of knowledges
+
+        """
+
+        knowledge = self.sep.join(
+            [f'{i + 1}. {k}' for i, k in enumerate(knowledge_list)])
+        knowledge_content = KNOWLEDGE_CONTENT_PROMPT.replace(
+            '<knowledge_content>', knowledge)
+        if only_content:
+            return knowledge_content
+        else:
+            knowledge_introduction = KNOWLEDGE_INTRODUCTION_PROMPT.replace(
+                '<file_name>', file_name)
+
+            knowledge_str = f'{KNOWLEDGE_PROMPT}{self.sep}{knowledge_introduction}{self.sep}{knowledge_content}' if len(
+                knowledge_list) > 0 else ''
+        return knowledge_str
+
+    def get_history_str(self):
+        """generate history string
+
+        """
+        history_str = ''
+        for i in range(len(self.history)):
+            history_item = self.history[len(self.history) - i - 1]
+            text = history_item['content']
+            if len(history_str) + len(text) + len(
+                    self.prompt) > self.prompt_max_length:
+                break
+            history_str = f'{self.sep}{text.strip()}{history_str}'
+
+        return history_str

my_modelscope_agent/prompt/prompt_factory.py

@@ -0,0 +1,16 @@
+from ..agent_types import AgentType
+
+from .messages_prompt import MessagesGenerator
+from .mrkl_prompt import MrklPromptGenerator
+from .ms_prompt import MSPromptGenerator
+
+
+def get_prompt_generator(agent_type: AgentType = AgentType.DEFAULT, **kwargs):
+    if AgentType.DEFAULT == agent_type or agent_type == AgentType.MS_AGENT:
+        return MSPromptGenerator(**kwargs)
+    elif AgentType.MRKL == agent_type:
+        return MrklPromptGenerator(**kwargs)
+    elif AgentType.Messages == agent_type:
+        return MessagesGenerator(**kwargs)
+    else:
+        raise NotImplementedError

my_modelscope_agent/prompt/raw_prompt_builder.py

@@ -0,0 +1,34 @@
+def qwen_chatml_prompt_preprocessor(messages):
+    prompt = ''
+    for message in messages:
+        if message['role'] == 'assistant' and message['content'] == '':
+            prompt += '<|im_start|>assistant\n'
+        else:
+            prompt = prompt + '<|im_start|>{role}\n{content}<|im_end|>\n'.format(
+                role=message['role'],
+                content=message['content'].lstrip('\n').rstrip())
+
+    # in the case of the assistant message is not in the last one, such as function result
+    if messages[-1]['role'] == 'assistant':
+        last_assistant_message_list = messages[-1]['content'].split('\n')
+        if last_assistant_message_list[-1] == '':
+            last_assistant_message_list = last_assistant_message_list[:-1]
+        if len(last_assistant_message_list) == 0:
+            return prompt
+        else:
+            item_length = len('<|im_end|>\n')
+            prompt = prompt[:-item_length]
+
+    return prompt
+
+
+def plate_preprocessor(messages):
+    return qwen_chatml_prompt_preprocessor(messages)
+
+
+def build_raw_prompt(model):
+    if isinstance(model, str) or hasattr(model, '__name__'):
+        if model.startswith('qwen'):
+            return qwen_chatml_prompt_preprocessor
+    else:
+        return plate_preprocessor

my_modelscope_agent/retrieve.py

@@ -0,0 +1,115 @@
+import os
+from typing import Dict, Iterable, List, Union
+
+import json
+from langchain.document_loaders import (PyPDFLoader, TextLoader,
+                                        UnstructuredFileLoader)
+from langchain.embeddings import ModelScopeEmbeddings
+from langchain.embeddings.base import Embeddings
+from langchain.schema import Document
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.vectorstores import FAISS, VectorStore
+
+
+class Retrieval:
+
+    def __init__(self,
+                 embedding: Embeddings = None,
+                 vs_cls: VectorStore = None,
+                 top_k: int = 5,
+                 vs_params: Dict = {}):
+        self.embedding = embedding or ModelScopeEmbeddings(
+            model_id='damo/nlp_gte_sentence-embedding_chinese-base')
+        self.top_k = top_k
+        self.vs_cls = vs_cls or FAISS
+        self.vs_params = vs_params
+        self.vs = None
+
+    def construct(self, docs):
+        assert len(docs) > 0
+        if isinstance(docs[0], str):
+            self.vs = self.vs_cls.from_texts(docs, self.embedding,
+                                             **self.vs_params)
+        elif isinstance(docs[0], Document):
+            self.vs = self.vs_cls.from_documents(docs, self.embedding,
+                                                 **self.vs_params)
+
+    def retrieve(self, query: str) -> List[str]:
+        res = self.vs.similarity_search(query, k=self.top_k)
+        if 'page' in res[0].metadata:
+            res.sort(key=lambda doc: doc.metadata['page'])
+        return [r.page_content for r in res]
+
+
+class ToolRetrieval(Retrieval):
+
+    def __init__(self,
+                 embedding: Embeddings = None,
+                 vs_cls: VectorStore = None,
+                 top_k: int = 5,
+                 vs_params: Dict = {}):
+        super().__init__(embedding, vs_cls, top_k, vs_params)
+
+    def retrieve(self, query: str) -> Dict[str, str]:
+        res = self.vs.similarity_search(query, k=self.top_k)
+
+        final_res = {}
+
+        for r in res:
+            content = r.page_content
+            name = json.loads(content)['name']
+            final_res[name] = content
+
+        return final_res
+
+
+class KnowledgeRetrieval(Retrieval):
+
+    def __init__(self,
+                 docs,
+                 embedding: Embeddings = None,
+                 vs_cls: VectorStore = None,
+                 top_k: int = 5,
+                 vs_params: Dict = {}):
+        super().__init__(embedding, vs_cls, top_k, vs_params)
+        self.construct(docs)
+
+    @classmethod
+    def from_file(cls,
+                  file_path: Union[str, list],
+                  embedding: Embeddings = None,
+                  vs_cls: VectorStore = None,
+                  top_k: int = 5,
+                  vs_params: Dict = {}):
+
+        textsplitter = CharacterTextSplitter()
+        all_files = []
+        if isinstance(file_path, str) and os.path.isfile(file_path):
+            all_files.append(file_path)
+        elif isinstance(file_path, list):
+            all_files = file_path
+        elif os.path.isdir(file_path):
+            for root, dirs, files in os.walk(file_path):
+                for f in files:
+                    all_files.append(os.path.join(root, f))
+        else:
+            raise ValueError('file_path must be a file or a directory')
+
+        docs = []
+        for f in all_files:
+            if f.lower().endswith('.txt'):
+                loader = TextLoader(f, autodetect_encoding=True)
+                docs += (loader.load_and_split(textsplitter))
+            elif f.lower().endswith('.md'):
+                loader = UnstructuredFileLoader(f, mode='elements')
+                docs += loader.load()
+            elif f.lower().endswith('.pdf'):
+                loader = PyPDFLoader(f)
+                docs += (loader.load_and_split(textsplitter))
+            else:
+                print(f'not support file type: {f}, will be support soon')
+
+        if len(docs) == 0:
+            return None
+        else:
+            return cls(docs, embedding, vs_cls, top_k, vs_params)

my_modelscope_agent/tools/__init__.py

@@ -0,0 +1,36 @@
+from .amap_weather import AMAPWeather
+from .code_interperter import CodeInterpreter
+from .code_interpreter_jupyter import CodeInterpreterJupyter
+from .hf_tool import HFTool
+from .image_chat_tool import ImageChatTool
+from .pipeline_tool import ModelscopePipelineTool
+from .plugin_tool import LangchainTool
+from .text_address_tool import TextAddressTool
+from .text_ie_tool import TextInfoExtractTool
+from .text_ner_tool import TextNerTool
+from .text_to_image_tool import TextToImageTool
+from .text_to_speech_tool import TexttoSpeechTool
+from .text_to_video_tool import TextToVideoTool
+from .tool import Tool
+from .translation_en2zh_tool import TranslationEn2ZhTool
+from .translation_zh2en_tool import TranslationZh2EnTool
+from .web_browser import WebBrowser
+from .web_search import WebSearch
+from .wordart_tool import WordArtTexture
+
+TOOL_INFO_LIST = {
+    'modelscope_text-translation-zh2en': 'TranslationZh2EnTool',
+    'modelscope_text-translation-en2zh': 'TranslationEn2ZhTool',
+    'modelscope_text-ie': 'TextInfoExtractTool',
+    'modelscope_text-ner': 'TextNerTool',
+    'modelscope_text-address': 'TextAddressTool',
+    'image_gen': 'TextToImageTool',
+    'modelscope_video-generation': 'TextToVideoTool',
+    'modelscope_image-chat': 'ImageChatTool',
+    'modelscope_speech-generation': 'TexttoSpeechTool',
+    'amap_weather': 'AMAPWeather',
+    'code_interpreter': 'CodeInterpreterJupyter',
+    'wordart_texture_generation': 'WordArtTexture',
+    'web_search': 'WebSearch',
+    'web_browser': 'WebBrowser',
+}

my_modelscope_agent/tools/amap_weather.py

@@ -0,0 +1,64 @@
+import os
+
+import pandas as pd
+import requests
+from ..tools.tool import Tool, ToolSchema
+from pydantic import ValidationError
+
+
+class AMAPWeather(Tool):
+    description = '获取对应城市的天气数据'
+    name = 'amap_weather'
+    parameters: list = [{
+        'name': 'location',
+        'description': 'get temperature for a specific location',
+        'required': True
+    }]
+
+    def __init__(self, cfg={}):
+        self.cfg = cfg.get(self.name, {})
+
+        # remote call
+        self.url = 'https://restapi.amap.com/v3/weather/weatherInfo?city={city}&key={key}'
+        self.token = self.cfg.get('token', os.environ.get('AMAP_TOKEN', ''))
+        self.city_df = pd.read_excel(
+            'https://modelscope.oss-cn-beijing.aliyuncs.com/resource/agent/AMap_adcode_citycode.xlsx'
+        )
+        assert self.token != '', 'weather api token must be acquired through ' \
+            'https://lbs.amap.com/api/webservice/guide/create-project/get-key and set by AMAP_TOKEN'
+
+        try:
+            all_param = {
+                'name': self.name,
+                'description': self.description,
+                'parameters': self.parameters
+            }
+            self.tool_schema = ToolSchema(**all_param)
+        except ValidationError:
+            raise ValueError(f'Error when parsing parameters of {self.name}')
+
+        self._str = self.tool_schema.model_dump_json()
+        self._function = self.parse_pydantic_model_to_openai_function(
+            all_param)
+
+    def get_city_adcode(self, city_name):
+        filtered_df = self.city_df[self.city_df['中文名'] == city_name]
+        if len(filtered_df['adcode'].values) == 0:
+            raise ValueError(
+                f'location {city_name} not found, availables are {self.city_df["中文名"]}'
+            )
+        else:
+            return filtered_df['adcode'].values[0]
+
+    def __call__(self, *args, **kwargs):
+        location = kwargs['location']
+        response = requests.get(
+            self.url.format(
+                city=self.get_city_adcode(location), key=self.token))
+        data = response.json()
+        if data['status'] == '0':
+            raise RuntimeError(data)
+        else:
+            weather = data['lives'][0]['weather']
+            temperature = data['lives'][0]['temperature']
+            return {'result': f'{location}的天气是{weather}温度是{temperature}度。'}

my_modelscope_agent/tools/code_interperter.py

@@ -0,0 +1,125 @@
+import os
+import re
+import traceback
+
+import appdirs
+import json
+
+from .code_interpreter_utils.create_code_interpreter import \
+    create_code_interpreter
+from .code_interpreter_utils.language_map import language_map
+from .code_interpreter_utils.truncate_output import truncate_output
+from .tool import Tool
+
+
+class CodeInterpreter(Tool):
+    """
+        using open interpreter to interpret code
+        by https://github.com/KillianLucas/open-interpreter
+    """
+    description = 'Executes code on the user\'s machine, **in the users local environment**, and returns the output'
+    name = 'code_interpreter'
+    parameters: list = [{
+        'name': 'language',
+        'description':
+        'The programming language (required parameter to the `execute` function)',
+        'required': True
+    }, {
+        'name': 'code',
+        'description': 'The code to execute (required)',
+        'required': True
+    }]
+
+    def __init__(self, cfg={}):
+        super().__init__(cfg)
+        self.create_code_interpreter = create_code_interpreter
+        self.language_map = language_map
+        self.truncate_output = truncate_output
+
+        self._code_interpreters = {}
+        self.max_output = self.cfg.get('max_output', 2000)
+
+    def _local_call(self, *args, **kwargs):
+
+        language, code = self._handle_input_fallback(**kwargs)
+
+        try:
+            # Fix a common error where the LLM thinks it's in a Jupyter notebook
+            if language == 'python' and code.startswith('!'):
+                code = code[1:]
+                language = 'shell'
+
+            if language in self.language_map:
+                if language not in self._code_interpreters:
+                    self._code_interpreters[
+                        language] = self.create_code_interpreter(language)
+                code_interpreter = self._code_interpreters[language]
+            else:
+                # This still prints code but don't allow code to run. Let Open-Interpreter know through output message
+                error_output = f'Error: Open Interpreter does not currently support {language}.'
+                print(error_output)
+                output = '\n' + error_output
+                return {'result': output.strip()}
+
+            output = ''
+            for line in code_interpreter.run(code):
+                if 'output' in line:
+                    output += '\n' + line['output']
+
+                    # Truncate output
+                    output = self.truncate_output(output, self.max_output)
+        except Exception as e:
+            error = traceback.format_exc()
+            output = ' '.join(f'{key}:{value}'
+                              for key, value in kwargs.items())
+            output += f'\nDetail error is {e}.\n{error}'
+
+        return {'result': output.strip()}
+
+    def _handle_input_fallback(self, **kwargs):
+        """
+        an alternative method is to parse code in content not from function call
+        such as:
+            text = response['content']
+            code_block = re.search(r'```([\s\S]+)```', text)  # noqa W^05
+            if code_block:
+                result = code_block.group(1)
+                language = result.split('\n')[0]
+                code = '\n'.join(result.split('\n')[1:])
+
+        :param fallback_text:
+        :return: language, cocde
+        """
+
+        language = kwargs.get('language', None)
+        code = kwargs.get('code', None)
+        fallback = kwargs.get('fallback', None)
+
+        if language and code:
+            return language, code
+        elif fallback:
+            try:
+                text = fallback
+                code_block = re.search(r'```([\s\S]+)```', text)  # noqa W^05
+                if code_block:
+                    result = code_block.group(1)
+                    # for multi code_block
+                    result = result.split('```')[0]
+                    language = result.split('\n')[0]
+                    if language == 'py' or language == 'python':
+                        # handle py case
+                        # ```py code ```
+                        language = 'python'
+                        code = '\n'.join(result.split('\n')[1:])
+                        return language, code
+
+                    if language == 'json':
+                        # handle json case
+                        # ```json {language,code}```
+                        parameters = json.loads('\n'.join(
+                            result.split('\n')[1:]).replace('\n', ''))
+                        return parameters['language'], parameters['code']
+            except ValueError:
+                return language, code
+        else:
+            return language, code

my_modelscope_agent/tools/code_interpreter_jupyter.py

@@ -0,0 +1,319 @@
+import asyncio
+import atexit
+import base64
+import glob
+import io
+import os
+import queue
+import re
+import shutil
+import signal
+import subprocess
+import sys
+import time
+import traceback
+import uuid
+from pathlib import Path
+from typing import Dict, Optional
+
+import json
+import matplotlib
+import PIL.Image
+from jupyter_client import BlockingKernelClient
+
+from .tool import Tool
+
+WORK_DIR = os.getenv('CODE_INTERPRETER_WORK_DIR', '/tmp/ci_workspace')
+
+STATIC_URL = os.getenv('CODE_INTERPRETER_STATIC_URL',
+                       'http://127.0.0.1:7866/static')
+
+LAUNCH_KERNEL_PY = """
+from ipykernel import kernelapp as app
+app.launch_new_instance()
+"""
+
+INIT_CODE_FILE = str(
+    Path(__file__).absolute().parent / 'code_interpreter_utils'
+    / 'code_interpreter_init_kernel.py')
+
+ALIB_FONT_FILE = str(
+    Path(__file__).absolute().parent / 'code_interpreter_utils'
+    / 'AlibabaPuHuiTi-3-45-Light.ttf')
+
+_KERNEL_CLIENTS: Dict[int, BlockingKernelClient] = {}
+
+
+class CodeInterpreterJupyter(Tool):
+    """
+        using jupyter kernel client to interpret python code,
+        should not be used the other code interpreter tool at the same time
+    """
+    description = '代码解释器，可用于执行Python代码。'
+    name = 'code_interpreter'
+    parameters: list = [{
+        'name': 'code',
+        'description': '待执行的代码',
+        'required': True
+    }]
+
+    def __init__(self, cfg={}):
+        super().__init__(cfg)
+        self.timeout = self.cfg.get('timeout', 30)
+        self.image_server = self.cfg.get('image_server', False)
+        self.kernel_clients: Dict[int, BlockingKernelClient] = {}
+        atexit.register(self._kill_kernels)
+
+        pid: int = os.getpid()
+        if pid in self.kernel_clients:
+            kc = self.kernel_clients[pid]
+        else:
+            self._fix_matplotlib_cjk_font_issue()
+            kc = self._start_kernel(pid)
+            with open(INIT_CODE_FILE) as fin:
+                start_code = fin.read()
+                start_code = start_code.replace('{{M6_FONT_PATH}}',
+                                                repr(ALIB_FONT_FILE)[1:-1])
+            print(self._execute_code(kc, start_code))
+            self.kernel_clients[pid] = kc
+
+        self.kc = kc
+
+    def __del__(self):
+        # make sure all the kernels are killed during __del__
+        signal.signal(signal.SIGTERM, self._kill_kernels)
+        signal.signal(signal.SIGINT, self._kill_kernels)
+
+    def _start_kernel(self, pid) -> BlockingKernelClient:
+        connection_file = os.path.join(WORK_DIR,
+                                       f'kernel_connection_file_{pid}.json')
+        launch_kernel_script = os.path.join(WORK_DIR,
+                                            f'launch_kernel_{pid}.py')
+        for f in [connection_file, launch_kernel_script]:
+            if os.path.exists(f):
+                print(f'WARNING: {f} already exists')
+                os.remove(f)
+
+        os.makedirs(WORK_DIR, exist_ok=True)
+
+        with open(launch_kernel_script, 'w') as fout:
+            fout.write(LAUNCH_KERNEL_PY)
+
+        available_envs = ['PATH', 'PYTHONPATH', 'LD_LIBRARY_PATH']
+        envs = {}
+        for k in available_envs:
+            if os.getenv(k) is not None:
+                envs[k] = os.getenv(k)
+
+        args = (
+            sys.executable,
+            launch_kernel_script,
+            '--IPKernelApp.connection_file',
+            connection_file,
+            '--matplotlib=inline',
+            '--quiet',
+        )
+        kernel_process = subprocess.Popen([*args], env=envs,
+                                          cwd=WORK_DIR)  # noqa E126
+        print(f"INFO: kernel process's PID = {kernel_process.pid}")
+
+        # Wait for kernel connection file to be written
+        while True:
+            if not os.path.isfile(connection_file):
+                time.sleep(0.1)
+            else:
+                # Keep looping if JSON parsing fails, file may be partially written
+                try:
+                    with open(connection_file, 'r') as fp:
+                        json.load(fp)
+                    break
+                except json.JSONDecodeError:
+                    pass
+
+        # Client
+        kc = BlockingKernelClient(connection_file=connection_file)
+        asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
+        kc.load_connection_file()
+        kc.start_channels()
+        kc.wait_for_ready()
+        return kc
+
+    def _kill_kernels(self):
+        for v in self.kernel_clients.values():
+            v.shutdown()
+        for k in list(self.kernel_clients.keys()):
+            del self.kernel_clients[k]
+
+    def _serve_image(self, image_base64: str, image_type: str) -> str:
+        image_file = f'{uuid.uuid4()}.{image_type}'
+        local_image_file = os.path.join(WORK_DIR, image_file)
+
+        png_bytes = base64.b64decode(image_base64)
+        assert isinstance(png_bytes, bytes)
+
+        if image_type == 'gif':
+            with open(local_image_file, 'wb') as file:
+                file.write(png_bytes)
+        else:
+            bytes_io = io.BytesIO(png_bytes)
+            PIL.Image.open(bytes_io).save(local_image_file, image_type)
+
+        if self.image_server:
+            image_url = f'{STATIC_URL}/{image_file}'
+            return image_url
+        else:
+            return local_image_file
+
+    def _escape_ansi(self, line: str) -> str:
+        ansi_escape = re.compile(r'(?:\x1B[@-_]|[\x80-\x9F])[0-?]*[ -/]*[@-~]')
+        return ansi_escape.sub('', line)
+
+    def _fix_matplotlib_cjk_font_issue(self):
+        ttf_name = os.path.basename(ALIB_FONT_FILE)
+        local_ttf = os.path.join(
+            os.path.abspath(
+                os.path.join(matplotlib.matplotlib_fname(), os.path.pardir)),
+            'fonts', 'ttf', ttf_name)
+        if not os.path.exists(local_ttf):
+            try:
+                shutil.copy(ALIB_FONT_FILE, local_ttf)
+                font_list_cache = os.path.join(matplotlib.get_cachedir(),
+                                               'fontlist-*.json')
+                for cache_file in glob.glob(font_list_cache):
+                    with open(cache_file) as fin:
+                        cache_content = fin.read()
+                    if ttf_name not in cache_content:
+                        os.remove(cache_file)
+            except Exception:
+                traceback.format_exc()
+
+    def _execute_code(self, kc: BlockingKernelClient, code: str) -> str:
+        kc.wait_for_ready()
+        kc.execute(code)
+        result = ''
+        image_idx = 0
+        while True:
+            text = ''
+            image = ''
+            finished = False
+            msg_type = 'error'
+            try:
+                msg = kc.get_iopub_msg()
+                msg_type = msg['msg_type']
+                if msg_type == 'status':
+                    if msg['content'].get('execution_state') == 'idle':
+                        finished = True
+                elif msg_type == 'execute_result':
+                    text = msg['content']['data'].get('text/plain', '')
+                    if 'image/png' in msg['content']['data']:
+                        image_b64 = msg['content']['data']['image/png']
+                        image_url = self._serve_image(image_b64, 'png')
+                        image_idx += 1
+                        image = '![IMAGEGEN](%s)' % (image_url)
+                    elif 'text/html' in msg['content']['data']:
+                        text += '\n' + msg['content']['data']['text/html']
+                    elif 'image/gif' in msg['content']['data']:
+                        image_b64 = msg['content']['data']['image/gif']
+                        image_url = self._serve_image(image_b64, 'gif')
+                        image_idx += 1
+                        image = '![IMAGEGEN](%s)' % (image_url)
+                elif msg_type == 'display_data':
+                    if 'image/png' in msg['content']['data']:
+                        image_b64 = msg['content']['data']['image/png']
+                        image_url = self._serve_image(image_b64, 'png')
+                        image_idx += 1
+                        image = '![IMAGEGEN](%s)' % (image_url)
+                    else:
+                        text = msg['content']['data'].get('text/plain', '')
+                elif msg_type == 'stream':
+                    msg_type = msg['content']['name']  # stdout, stderr
+                    text = msg['content']['text']
+                elif msg_type == 'error':
+                    text = self._escape_ansi('\n'.join(
+                        msg['content']['traceback']))
+                    if 'M6_CODE_INTERPRETER_TIMEOUT' in text:
+                        text = 'Timeout: Code execution exceeded the time limit.'
+            except queue.Empty:
+                text = 'Timeout: Code execution exceeded the time limit.'
+                finished = True
+            except Exception:
+                text = 'The code interpreter encountered an unexpected error.'
+                traceback.format_exc()
+                finished = True
+            if text:
+                result += f'\n{text}'
+            if image:
+                result += f'\n\n{image}'
+            if finished:
+                break
+        result = result.lstrip('\n')
+        if not result:
+            result += 'The code executed successfully.'
+        return result
+
+    def _local_call(self, *args, **kwargs):
+        code = self._handle_input_fallback(**kwargs)
+        if not code.strip():
+            return ''
+
+        if self.timeout:
+            code = f'_M6CountdownTimer.start({self.timeout})\n{code}'
+
+        fixed_code = []
+        for line in code.split('\n'):
+            fixed_code.append(line)
+            if line.startswith('sns.set_theme('):
+                fixed_code.append(
+                    'plt.rcParams["font.family"] = _m6_font_prop.get_name()')
+        fixed_code = '\n'.join(fixed_code)
+        result = self._execute_code(self.kc, fixed_code)
+
+        if self.timeout:
+            self._execute_code(self.kc, '_M6CountdownTimer.cancel()')
+
+        return {'result': result}
+
+    def _handle_input_fallback(self, **kwargs):
+        """
+        an alternative method is to parse code in content not from function call
+        such as:
+            text = response['content']
+            code_block = re.search(r'```([\s\S]+)```', text)  # noqa W^05
+            if code_block:
+                result = code_block.group(1)
+                language = result.split('\n')[0]
+                code = '\n'.join(result.split('\n')[1:])
+
+        :param fallback_text:
+        :return: language, cocde
+        """
+
+        code = kwargs.get('code', None)
+        fallback = kwargs.get('fallback', None)
+
+        if code:
+            return code
+        elif fallback:
+            try:
+                text = fallback
+                code_block = re.search(r'```([\s\S]+)```', text)  # noqa W^05
+                if code_block:
+                    result = code_block.group(1)
+                    language = result.split('\n')[0]
+                    if language == 'py' or language == 'python':
+                        # handle py case
+                        # ```py code ```
+                        language = 'python'
+                        code = '\n'.join(result.split('\n')[1:])
+                        return code
+
+                    if language == 'json':
+                        # handle json case
+                        # ```json {language,code}```
+                        parameters = json.loads('\n'.join(
+                            result.split('\n')[1:]).replace('\n', ''))
+                        return parameters['code']
+            except ValueError:
+                return code
+        else:
+            return code

my_modelscope_agent/tools/code_interpreter_utils/__init__.py

@@ -0,0 +1,5 @@
+# all the utility functions under code_interpreter_utils are borrowed from project
+# in order to use python lower than 3.10
+# https://github.com/KillianLucas/open-interpreter
+
+from .base_code_interpreter import BaseCodeInterpreter

my_modelscope_agent/tools/code_interpreter_utils/base_code_interpreter.py

@@ -0,0 +1,13 @@
+class BaseCodeInterpreter:
+    """
+    .run is a generator that yields a dict with attributes: active_line, output
+    """
+
+    def __init__(self):
+        pass
+
+    def run(self, code):
+        pass
+
+    def terminate(self):
+        pass

my_modelscope_agent/tools/code_interpreter_utils/code_interpreter_init_kernel.py

@@ -0,0 +1,50 @@
+import math  # noqa
+import os  # noqa
+import re  # noqa
+import signal
+
+import json  # noqa
+import matplotlib  # noqa
+import matplotlib.pyplot as plt
+import numpy as np  # noqa
+import pandas as pd  # noqa
+import seaborn as sns
+from matplotlib.font_manager import FontProperties
+from sympy import Eq, solve, symbols  # noqa
+
+
+def input(*args, **kwargs):  # noqa
+    raise NotImplementedError('Python input() function is disabled.')
+
+
+def _m6_timout_handler(_signum=None, _frame=None):
+    raise TimeoutError('M6_CODE_INTERPRETER_TIMEOUT')
+
+
+try:
+    signal.signal(signal.SIGALRM, _m6_timout_handler)
+except AttributeError:  # windows
+    pass
+
+
+class _M6CountdownTimer:
+
+    @classmethod
+    def start(cls, timeout: int):
+        try:
+            signal.alarm(timeout)
+        except AttributeError:  # windows
+            pass  # TODO: I haven't found a solution that works with jupyter yet.
+
+    @classmethod
+    def cancel(cls):
+        try:
+            signal.alarm(0)
+        except AttributeError:  # windows
+            pass  # TODO
+
+
+sns.set_theme()
+
+_m6_font_prop = FontProperties(fname='{{M6_FONT_PATH}}')
+plt.rcParams['font.family'] = _m6_font_prop.get_name()

my_modelscope_agent/tools/code_interpreter_utils/create_code_interpreter.py

@@ -0,0 +1,12 @@
+from .language_map import language_map
+
+
+def create_code_interpreter(language):
+    # Case in-sensitive
+    language = language.lower()
+
+    try:
+        CodeInterpreter = language_map[language]
+        return CodeInterpreter()
+    except KeyError:
+        raise ValueError(f'Unknown or unsupported language: {language}')

my_modelscope_agent/tools/code_interpreter_utils/language_map.py

@@ -0,0 +1,19 @@
+from .languages.applescript import AppleScript
+from .languages.html import HTML
+from .languages.javascript import JavaScript
+from .languages.powershell import PowerShell
+from .languages.python import Python
+from .languages.r import R
+from .languages.shell import Shell
+
+language_map = {
+    'python': Python,
+    'bash': Shell,
+    'shell': Shell,
+    'zsh': Shell,
+    'javascript': JavaScript,
+    'html': HTML,
+    'applescript': AppleScript,
+    'r': R,
+    'powershell': PowerShell,
+}

my_modelscope_agent/tools/code_interpreter_utils/languages/applescript.py

@@ -0,0 +1,67 @@
+import os
+
+from ..subprocess_code_interpreter import SubprocessCodeInterpreter
+
+
+class AppleScript(SubprocessCodeInterpreter):
+    file_extension = 'applescript'
+    proper_name = 'AppleScript'
+
+    def __init__(self):
+        super().__init__()
+        self.start_cmd = os.environ.get('SHELL', '/bin/zsh')
+
+    def preprocess_code(self, code):
+        """
+        Inserts an end_of_execution marker and adds active line indicators.
+        """
+        # Add active line indicators to the code
+        code = self.add_active_line_indicators(code)
+
+        # Escape double quotes
+        code = code.replace('"', r"\"")
+
+        # Wrap in double quotes
+        code = '"' + code + '"'
+
+        # Prepend start command for AppleScript
+        code = 'osascript -e ' + code
+
+        # Append end of execution indicator
+        code += '; echo "##end_of_execution##"'
+
+        return code
+
+    def add_active_line_indicators(self, code):
+        """
+        Adds log commands to indicate the active line of execution in the AppleScript.
+        """
+        modified_lines = []
+        lines = code.split('\n')
+
+        for idx, line in enumerate(lines):
+            # Add log command to indicate the line number
+            if line.strip():  # Only add if line is not empty
+                modified_lines.append(f'log "##active_line{idx + 1}##"')
+            modified_lines.append(line)
+
+        return '\n'.join(modified_lines)
+
+    def detect_active_line(self, line):
+        """
+        Detects active line indicator in the output.
+        """
+        prefix = '##active_line'
+        if prefix in line:
+            try:
+                return int(line.split(prefix)[1].split()[0])
+            except Exception as e:
+                print(e)
+                pass
+        return None
+
+    def detect_end_of_execution(self, line):
+        """
+        Detects end of execution marker in the output.
+        """
+        return '##end_of_execution##' in line

my_modelscope_agent/tools/code_interpreter_utils/languages/html.py

@@ -0,0 +1,26 @@
+import os
+import tempfile
+import webbrowser
+
+from ..base_code_interpreter import BaseCodeInterpreter
+
+
+class HTML(BaseCodeInterpreter):
+    file_extension = 'html'
+    proper_name = 'HTML'
+
+    def __init__(self):
+        super().__init__()
+
+    def run(self, code):
+        # Create a temporary HTML file with the content
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.html') as f:
+            f.write(code.encode())
+
+        # Open the HTML file with the default web browser
+        webbrowser.open('file://' + os.path.realpath(f.name))
+
+        yield {
+            'output':
+            f"Saved to {os.path.realpath(f.name)} and opened with the user's default web browser."
+        }

my_modelscope_agent/tools/code_interpreter_utils/languages/javascript.py

@@ -0,0 +1,66 @@
+import re
+
+from ..subprocess_code_interpreter import SubprocessCodeInterpreter
+
+
+class JavaScript(SubprocessCodeInterpreter):
+    file_extension = 'js'
+    proper_name = 'JavaScript'
+
+    def __init__(self):
+        super().__init__()
+        self.start_cmd = 'node -i'
+
+    def preprocess_code(self, code):
+        return preprocess_javascript(code)
+
+    def line_postprocessor(self, line):
+        # Node's interactive REPL outputs a billion things
+        # So we clean it up:
+        if 'Welcome to Node.js' in line:
+            return None
+        if line.strip() in ['undefined', 'Type ".help" for more information.']:
+            return None
+        # Remove trailing ">"s
+        line = re.sub(r'^\s*(>\s*)+', '', line)
+        return line
+
+    def detect_active_line(self, line):
+        if '##active_line' in line:
+            return int(line.split('##active_line')[1].split('##')[0])
+        return None
+
+    def detect_end_of_execution(self, line):
+        return '##end_of_execution##' in line
+
+
+def preprocess_javascript(code):
+    """
+    Add active line markers
+    Wrap in a try catch
+    Add end of execution marker
+    """
+
+    # Split code into lines
+    lines = code.split('\n')
+    processed_lines = []
+
+    for i, line in enumerate(lines, 1):
+        # Add active line print
+        processed_lines.append(f'console.log("##active_line{i}##");')
+        processed_lines.append(line)
+
+    # Join lines to form the processed code
+    processed_code = '\n'.join(processed_lines)
+
+    # Wrap in a try-catch and add end of execution marker
+    processed_code = f"""
+try {{
+{processed_code}
+}} catch (e) {{
+    console.log(e);
+}}
+console.log("##end_of_execution##");
+"""
+
+    return processed_code

my_modelscope_agent/tools/code_interpreter_utils/languages/powershell.py

@@ -0,0 +1,75 @@
+import os
+import platform
+import shutil
+
+from ..subprocess_code_interpreter import SubprocessCodeInterpreter
+
+
+class PowerShell(SubprocessCodeInterpreter):
+    file_extension = 'ps1'
+    proper_name = 'PowerShell'
+
+    def __init__(self):
+        super().__init__()
+
+        # Determine the start command based on the platform (use "powershell" for Windows)
+        if platform.system() == 'Windows':
+            self.start_cmd = 'powershell.exe'
+            # self.start_cmd = os.environ.get('SHELL', 'powershell.exe')
+        else:
+            # On non-Windows platforms, prefer pwsh (PowerShell Core) if available, or fall back to bash
+            self.start_cmd = 'pwsh' if shutil.which('pwsh') else 'bash'
+
+    def preprocess_code(self, code):
+        return preprocess_powershell(code)
+
+    def line_postprocessor(self, line):
+        return line
+
+    def detect_active_line(self, line):
+        if '##active_line' in line:
+            return int(line.split('##active_line')[1].split('##')[0])
+        return None
+
+    def detect_end_of_execution(self, line):
+        return '##end_of_execution##' in line
+
+
+def preprocess_powershell(code):
+    """
+    Add active line markers
+    Wrap in try-catch block
+    Add end of execution marker
+    """
+    # Add commands that tell us what the active line is
+    code = add_active_line_prints(code)
+
+    # Wrap in try-catch block for error handling
+    code = wrap_in_try_catch(code)
+
+    # Add end marker (we'll be listening for this to know when it ends)
+    code += '\nWrite-Output "##end_of_execution##"'
+
+    return code
+
+
+def add_active_line_prints(code):
+    """
+    Add Write-Output statements indicating line numbers to a PowerShell script.
+    """
+    lines = code.split('\n')
+    for index, line in enumerate(lines):
+        # Insert the Write-Output command before the actual line
+        lines[index] = f'Write-Output "##active_line{index + 1}##"\n{line}'
+    return '\n'.join(lines)
+
+
+def wrap_in_try_catch(code):
+    """
+    Wrap PowerShell code in a try-catch block to catch errors and display them.
+    """
+    try_catch_code = """
+try {
+    $ErrorActionPreference = "Stop"
+"""
+    return try_catch_code + code + '\n} catch {\n    Write-Error $_\n}\n'

my_modelscope_agent/tools/code_interpreter_utils/languages/python.py

@@ -0,0 +1,161 @@
+import ast
+import os
+import re
+import shlex
+import sys
+
+from ..subprocess_code_interpreter import SubprocessCodeInterpreter
+
+
+class Python(SubprocessCodeInterpreter):
+    file_extension = 'py'
+    proper_name = 'Python'
+
+    def __init__(self):
+        super().__init__()
+        executable = sys.executable
+        if os.name != 'nt':  # not Windows
+            executable = shlex.quote(executable)
+        self.start_cmd = executable + ' -i -q -u'
+
+    def preprocess_code(self, code):
+        return preprocess_python(code)
+
+    def line_postprocessor(self, line):
+        if re.match(r'^(\s*>>>\s*|\s*\.\.\.\s*)', line):
+            return None
+        return line
+
+    def detect_active_line(self, line):
+        if '##active_line' in line:
+            return int(line.split('##active_line')[1].split('##')[0])
+        return None
+
+    def detect_end_of_execution(self, line):
+        return '##end_of_execution##' in line
+
+
+def preprocess_python(code):
+    """
+    Add active line markers
+    Wrap in a try except
+    Add end of execution marker
+    """
+
+    # Add print commands that tell us what the active line is
+    code = add_active_line_prints(code)
+
+    # Wrap in a try except
+    code = wrap_in_try_except(code)
+
+    # Remove any whitespace lines, as this will break indented blocks
+    # (are we sure about this? test this)
+    code_lines = code.split('\n')
+    code_lines = [c for c in code_lines if c.strip() != '']
+    code = '\n'.join(code_lines)
+
+    # Add end command (we'll be listening for this so we know when it ends)
+    code += '\n\nprint("##end_of_execution##")'
+
+    return code
+
+
+def add_active_line_prints(code):
+    """
+    Add print statements indicating line numbers to a python string.
+    """
+    tree = ast.parse(code)
+    transformer = AddLinePrints()
+    new_tree = transformer.visit(tree)
+    return ast.unparse(new_tree)
+
+
+class AddLinePrints(ast.NodeTransformer):
+    """
+    Transformer to insert print statements indicating the line number
+    before every executable line in the AST.
+    """
+
+    def insert_print_statement(self, line_number):
+        """Inserts a print statement for a given line number."""
+        return ast.Expr(
+            value=ast.Call(
+                func=ast.Name(id='print', ctx=ast.Load()),
+                args=[ast.Constant(value=f'##active_line{line_number}##')],
+                keywords=[],
+            ))
+
+    def process_body(self, body):
+        """Processes a block of statements, adding print calls."""
+        new_body = []
+
+        # In case it's not iterable:
+        if not isinstance(body, list):
+            body = [body]
+
+        for sub_node in body:
+            if hasattr(sub_node, 'lineno'):
+                new_body.append(self.insert_print_statement(sub_node.lineno))
+            new_body.append(sub_node)
+
+        return new_body
+
+    def visit(self, node):
+        """Overridden visit to transform nodes."""
+        new_node = super().visit(node)
+
+        # If node has a body, process it
+        if hasattr(new_node, 'body'):
+            new_node.body = self.process_body(new_node.body)
+
+        # If node has an orelse block (like in for, while, if), process it
+        if hasattr(new_node, 'orelse') and new_node.orelse:
+            new_node.orelse = self.process_body(new_node.orelse)
+
+        # Special case for Try nodes as they have multiple blocks
+        if isinstance(new_node, ast.Try):
+            for handler in new_node.handlers:
+                handler.body = self.process_body(handler.body)
+            if new_node.finalbody:
+                new_node.finalbody = self.process_body(new_node.finalbody)
+
+        return new_node
+
+
+def wrap_in_try_except(code):
+    # Add import traceback
+    code = 'import traceback\n' + code
+
+    # Parse the input code into an AST
+    parsed_code = ast.parse(code)
+
+    # Wrap the entire code's AST in a single try-except block
+    try_except = ast.Try(
+        body=parsed_code.body,
+        handlers=[
+            ast.ExceptHandler(
+                type=ast.Name(id='Exception', ctx=ast.Load()),
+                name=None,
+                body=[
+                    ast.Expr(
+                        value=ast.Call(
+                            func=ast.Attribute(
+                                value=ast.Name(id='traceback', ctx=ast.Load()),
+                                attr='print_exc',
+                                ctx=ast.Load(),
+                            ),
+                            args=[],
+                            keywords=[],
+                        )),
+                ],
+            )
+        ],
+        orelse=[],
+        finalbody=[],
+    )
+
+    # Assign the try-except block as the new body
+    parsed_code.body = [try_except]
+
+    # Convert the modified AST back to source code
+    return ast.unparse(parsed_code)

my_modelscope_agent/tools/code_interpreter_utils/languages/r.py

@@ -0,0 +1,71 @@
+import re
+
+from ..subprocess_code_interpreter import SubprocessCodeInterpreter
+
+
+class R(SubprocessCodeInterpreter):
+    file_extension = 'r'
+    proper_name = 'R'
+
+    def __init__(self):
+        super().__init__()
+        self.start_cmd = 'R -q --vanilla'  # Start R in quiet and vanilla mode
+
+    def preprocess_code(self, code):
+        """
+        Add active line markers
+        Wrap in a tryCatch for better error handling in R
+        Add end of execution marker
+        """
+
+        lines = code.split('\n')
+        processed_lines = []
+
+        for i, line in enumerate(lines, 1):
+            # Add active line print
+            processed_lines.append(f'cat("##active_line{i}##\\n");{line}')
+
+        # Join lines to form the processed code
+        processed_code = '\n'.join(processed_lines)
+
+        # Wrap in a tryCatch for error handling and add end of execution marker
+        processed_code = f"""
+tryCatch({{
+{processed_code}
+}}, error=function(e){{
+    cat("## execution_error ##\\n", conditionMessage(e), "\\n");
+}})
+cat("## end_of_execution ##\\n");
+"""
+        # Count the number of lines of processed_code
+        # (R echoes all code back for some reason, but we can skip it if we track this!)
+        self.code_line_count = len(processed_code.split('\n')) - 1
+
+        return processed_code
+
+    def line_postprocessor(self, line):
+        # If the line count attribute is set and non-zero, decrement and skip the line
+        if hasattr(self, 'code_line_count') and self.code_line_count > 0:
+            self.code_line_count -= 1
+            return None
+
+        if re.match(r'^(\s*>>>\s*|\s*\.\.\.\s*|\s*>\s*|\s*\+\s*|\s*)$', line):
+            return None
+        if 'R version' in line:  # Startup message
+            return None
+        if line.strip().startswith('[1] "') and line.endswith(
+                '"'):  # For strings, trim quotation marks
+            return line[5:-1].strip()
+        if line.strip().startswith(
+                '[1]'):  # Normal R output prefix for non-string outputs
+            return line[4:].strip()
+
+        return line
+
+    def detect_active_line(self, line):
+        if '##active_line' in line:
+            return int(line.split('##active_line')[1].split('##')[0])
+        return None
+
+    def detect_end_of_execution(self, line):
+        return '##end_of_execution##' in line or '## execution_error ##' in line

my_modelscope_agent/tools/code_interpreter_utils/languages/shell.py

@@ -0,0 +1,89 @@
+import os
+import platform
+import re
+
+from ..subprocess_code_interpreter import SubprocessCodeInterpreter
+
+
+class Shell(SubprocessCodeInterpreter):
+    file_extension = 'sh'
+    proper_name = 'Shell'
+
+    def __init__(self):
+        super().__init__()
+
+        # Determine the start command based on the platform
+        if platform.system() == 'Windows':
+            self.start_cmd = 'cmd.exe'
+        else:
+            self.start_cmd = os.environ.get('SHELL', 'bash')
+
+    def preprocess_code(self, code):
+        return preprocess_shell(code)
+
+    def line_postprocessor(self, line):
+        return line
+
+    def detect_active_line(self, line):
+        if '##active_line' in line:
+            return int(line.split('##active_line')[1].split('##')[0])
+        return None
+
+    def detect_end_of_execution(self, line):
+        return '##end_of_execution##' in line
+
+
+def preprocess_shell(code):
+    """
+    Add active line markers
+    Wrap in a try except (trap in shell)
+    Add end of execution marker
+    """
+
+    # Add commands that tell us what the active line is
+    # if it's multiline, just skip this. soon we should make it work with multiline
+    if not has_multiline_commands(code):
+        code = add_active_line_prints(code)
+
+    # Add end command (we'll be listening for this so we know when it ends)
+    code += '\necho "##end_of_execution##"'
+
+    return code
+
+
+def add_active_line_prints(code):
+    """
+    Add echo statements indicating line numbers to a shell string.
+    """
+    lines = code.split('\n')
+    for index, line in enumerate(lines):
+        # Insert the echo command before the actual line
+        lines[index] = f'echo "##active_line{index + 1}##"\n{line}'
+    return '\n'.join(lines)
+
+
+def has_multiline_commands(script_text):
+    # Patterns that indicate a line continues
+    continuation_patterns = [
+        r'\\$',  # Line continuation character at the end of the line
+        r'\|$',  # Pipe character at the end of the line indicating a pipeline continuation
+        r'&&\s*$',  # Logical AND at the end of the line
+        r'\|\|\s*$',  # Logical OR at the end of the line
+        r'<\($',  # Start of process substitution
+        r'\($',  # Start of subshell
+        r'{\s*$',  # Start of a block
+        r'\bif\b',  # Start of an if statement
+        r'\bwhile\b',  # Start of a while loop
+        r'\bfor\b',  # Start of a for loop
+        r'do\s*$',  # 'do' keyword for loops
+        r'then\s*$',  # 'then' keyword for if statements
+    ]
+
+    # Check each line for multiline patterns
+    for line in script_text.splitlines():
+        if any(
+                re.search(pattern, line.rstrip())
+                for pattern in continuation_patterns):
+            return True
+
+    return False

my_modelscope_agent/tools/code_interpreter_utils/subprocess_code_interpreter.py

@@ -0,0 +1,152 @@
+import queue
+import subprocess
+import threading
+import time
+import traceback
+
+from .base_code_interpreter import BaseCodeInterpreter
+
+
+class SubprocessCodeInterpreter(BaseCodeInterpreter):
+
+    def __init__(self):
+        self.start_cmd = ''
+        self.process = None
+        self.debug_mode = False
+        self.output_queue = queue.Queue()
+        self.done = threading.Event()
+
+    def detect_active_line(self, line):
+        return None
+
+    def detect_end_of_execution(self, line):
+        return None
+
+    def line_postprocessor(self, line):
+        return line
+
+    def preprocess_code(self, code):
+        """
+        This needs to insert an end_of_execution marker of some kind,
+        which can be detected by detect_end_of_execution.
+
+        Optionally, add active line markers for detect_active_line.
+        """
+        return code
+
+    def terminate(self):
+        self.process.terminate()
+
+    def start_process(self):
+        if self.process:
+            self.terminate()
+
+        self.process = subprocess.Popen(
+            self.start_cmd.split(),
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            bufsize=0,
+            universal_newlines=True,
+        )
+        threading.Thread(
+            target=self.handle_stream_output,
+            args=(self.process.stdout, False),
+            daemon=True,
+        ).start()
+        threading.Thread(
+            target=self.handle_stream_output,
+            args=(self.process.stderr, True),
+            daemon=True,
+        ).start()
+
+    def run(self, code):
+        retry_count = 0
+        max_retries = 3
+
+        # Setup
+        try:
+            code = self.preprocess_code(code)
+            if not self.process:
+                self.start_process()
+        except Exception as e:
+            print(e)
+            yield {'output': traceback.format_exc()}
+            return
+
+        while retry_count <= max_retries:
+            if self.debug_mode:
+                print(
+                    f'(after processing) Running processed code:\n{code}\n---')
+
+            self.done.clear()
+
+            try:
+                self.process.stdin.write(code + '\n')
+                self.process.stdin.flush()
+                break
+            except Exception as e:
+                print(e)
+                if retry_count != 0:
+                    # For UX, I like to hide this if it happens once. Obviously feels better to not see errors
+                    # Most of the time it doesn't matter, but we should figure out why it happens frequently with:
+                    # applescript
+                    yield {'output': traceback.format_exc()}
+                    yield {
+                        'output': f'Retrying... ({retry_count}/{max_retries})'
+                    }
+                    yield {'output': 'Restarting process.'}
+
+                self.start_process()
+
+                retry_count += 1
+                if retry_count > max_retries:
+                    yield {
+                        'output':
+                        'Maximum retries reached. Could not execute code.'
+                    }
+                    return
+
+        while True:
+            if not self.output_queue.empty():
+                yield self.output_queue.get()
+            else:
+                time.sleep(0.1)
+            try:
+                output = self.output_queue.get(
+                    timeout=0.3)  # Waits for 0.3 seconds
+                yield output
+            except queue.Empty:
+                if self.done.is_set():
+                    # Try to yank 3 more times from it... maybe there's something in there...
+                    # (I don't know if this actually helps. Maybe we just need to yank 1 more time)
+                    for _ in range(3):
+                        if not self.output_queue.empty():
+                            yield self.output_queue.get()
+                        time.sleep(0.2)
+                    break
+
+    def handle_stream_output(self, stream, is_error_stream):
+        for line in iter(stream.readline, ''):
+            if self.debug_mode:
+                print(f'Received output line:\n{line}\n---')
+
+            line = self.line_postprocessor(line)
+
+            if line is None:
+                continue  # `line = None` is the postprocessor's signal to discard completely
+
+            if self.detect_active_line(line):
+                active_line = self.detect_active_line(line)
+                self.output_queue.put({'active_line': active_line})
+            elif self.detect_end_of_execution(line):
+                self.output_queue.put({'active_line': None})
+                time.sleep(0.1)
+                self.done.set()
+            elif is_error_stream and 'KeyboardInterrupt' in line:
+                self.output_queue.put({'output': 'KeyboardInterrupt'})
+                time.sleep(0.1)
+                self.done.set()
+            else:
+                self.output_queue.put({'output': line})

my_modelscope_agent/tools/code_interpreter_utils/truncate_output.py

@@ -0,0 +1,15 @@
+def truncate_output(data, max_output_chars=2000):
+    needs_truncation = False
+
+    message = f'Output truncated. Showing the last {max_output_chars} characters.\n\n'
+
+    # Remove previous truncation message if it exists
+    if data.startswith(message):
+        data = data[len(message):]
+        needs_truncation = True
+
+    # If data exceeds max length, truncate it and add message
+    if len(data) > max_output_chars or needs_truncation:
+        data = message + data[-max_output_chars:]
+
+    return data

my_modelscope_agent/tools/hf_tool.py

@@ -0,0 +1,22 @@
+from typing import Dict, List
+
+from transformers.tools import Tool as HFTool
+
+from .tool import Tool
+
+
+class HFTool(Tool):
+    """Simple wrapper for huggingface transformers tools
+
+    """
+
+    def __init__(self, tool: HFTool, description: str, name: str,
+                 parameters: List[Dict]):
+        self.tool = tool
+        self.description = description
+        self.name = name
+        self.parameters = parameters
+        super().__init__()
+
+    def _local_call(self, *args, **kwargs):
+        return {'result': self.tool(**kwargs)}

my_modelscope_agent/tools/image_chat_tool.py

@@ -0,0 +1,51 @@
+from modelscope.utils.constant import Tasks
+from .pipeline_tool import ModelscopePipelineTool
+
+
+class ImageChatTool(ModelscopePipelineTool):
+    default_model = 'damo/multi-modal_mplug_owl_multimodal-dialogue_7b'
+    description = '图文对话和图像描述服务，针对输入的图片和用户的文本输入，给出文本回复'
+    name = 'modelscope_image-chat'
+    parameters: list = [{
+        'name': 'image',
+        'description': '用户输入的图片',
+        'required': True
+    }, {
+        'name': 'text',
+        'description': '用户输入的文本',
+        'required': True
+    }]
+    task = Tasks.multimodal_dialogue
+
+    def construct_image_chat_input(self, **kwargs):
+        image = kwargs.pop('image', '')
+        text = kwargs.pop('text', '')
+
+        system_prompt_1 = 'The following is a conversation between a curious human and AI assistant.'
+        system_prompt_2 = "The assistant gives helpful, detailed, and polite answers to the user's questions."
+        messages = {
+            'messages': [
+                {
+                    'role': 'system',
+                    'content': system_prompt_1 + ' ' + system_prompt_2
+                },
+                {
+                    'role': 'user',
+                    'content': [{
+                        'image': image
+                    }]
+                },
+                {
+                    'role': 'user',
+                    'content': text
+                },
+            ]
+        }
+        return messages
+
+    def _remote_parse_input(self, *args, **kwargs):
+        messages = self.construct_image_chat_input(**kwargs)
+        return {'input': messages}
+
+    def _local_parse_input(self, *args, **kwargs):
+        return (self.construct_image_chat_input(**kwargs)), {}

my_modelscope_agent/tools/openapi_plugin.py

@@ -0,0 +1,370 @@
+import os
+import re
+from typing import List, Optional
+
+import json
+import requests
+from jsonschema import RefResolver
+from pydantic import BaseModel, ValidationError
+from requests.exceptions import RequestException, Timeout
+
+from .tool import Tool
+
+MAX_RETRY_TIMES = 3
+
+
+class ParametersSchema(BaseModel):
+    name: str
+    description: str
+    required: Optional[bool] = True
+
+
+class ToolSchema(BaseModel):
+    name: str
+    description: str
+    parameters: List[ParametersSchema]
+
+
+class OpenAPIPluginTool(Tool):
+    """
+     openapi schema tool
+    """
+    name: str = 'api tool'
+    description: str = 'This is a api tool that ...'
+    parameters: list = []
+
+    def __init__(self, cfg, name):
+        self.name = name
+        self.cfg = cfg.get(self.name, {})
+        self.is_remote_tool = self.cfg.get('is_remote_tool', False)
+        # remote call
+        self.url = self.cfg.get('url', '')
+        self.token = self.cfg.get('token', '')
+        self.header = self.cfg.get('header', '')
+        self.method = self.cfg.get('method', '')
+        self.parameters = self.cfg.get('parameters', [])
+        self.description = self.cfg.get('description',
+                                        'This is a api tool that ...')
+        self.responses_param = self.cfg.get('responses_param', [])
+        try:
+            all_para = {
+                'name': self.name,
+                'description': self.description,
+                'parameters': self.parameters
+            }
+            self.tool_schema = ToolSchema(**all_para)
+        except ValidationError:
+            raise ValueError(f'Error when parsing parameters of {self.name}')
+        self._str = self.tool_schema.model_dump_json()
+        self._function = self.parse_pydantic_model_to_openai_function(all_para)
+
+    def _remote_call(self, *args, **kwargs):
+        if self.url == '':
+            raise ValueError(
+                f"Could not use remote call for {self.name} since this tool doesn't have a remote endpoint"
+            )
+
+        remote_parsed_input = json.dumps(
+            self._remote_parse_input(*args, **kwargs))
+        origin_result = None
+        if self.method == 'POST':
+            retry_times = MAX_RETRY_TIMES
+            while retry_times:
+                retry_times -= 1
+                try:
+                    print(f'data: {kwargs}')
+                    print(f'header: {self.header}')
+                    response = requests.request(
+                        'POST',
+                        url=self.url,
+                        headers=self.header,
+                        data=remote_parsed_input)
+
+                    if response.status_code != requests.codes.ok:
+                        response.raise_for_status()
+                    origin_result = json.loads(
+                        response.content.decode('utf-8'))
+
+                    final_result = self._parse_output(
+                        origin_result, remote=True)
+                    return final_result
+                except Timeout:
+                    continue
+                except RequestException as e:
+                    raise ValueError(
+                        f'Remote call failed with error code: {e.response.status_code},\
+                        error message: {e.response.content.decode("utf-8")}')
+
+            raise ValueError(
+                'Remote call max retry times exceeded! Please try to use local call.'
+            )
+        elif self.method == 'GET':
+            retry_times = MAX_RETRY_TIMES
+
+            new_url = self.url
+            matches = re.findall(r'\{(.*?)\}', self.url)
+            for match in matches:
+                if match in kwargs:
+                    new_url = new_url.replace('{' + match + '}', kwargs[match])
+                else:
+                    print(
+                        f'The parameter {match} was not generated by the model.'
+                    )
+
+            while retry_times:
+                retry_times -= 1
+                try:
+                    print('GET:', new_url)
+                    print('GET:', self.url)
+
+                    response = requests.request(
+                        'GET',
+                        url=new_url,
+                        headers=self.header,
+                        params=remote_parsed_input)
+                    if response.status_code != requests.codes.ok:
+                        response.raise_for_status()
+
+                    origin_result = json.loads(
+                        response.content.decode('utf-8'))
+
+                    final_result = self._parse_output(
+                        origin_result, remote=True)
+                    return final_result
+                except Timeout:
+                    continue
+                except RequestException as e:
+                    raise ValueError(
+                        f'Remote call failed with error code: {e.response.status_code},\
+                        error message: {e.response.content.decode("utf-8")}')
+
+            raise ValueError(
+                'Remote call max retry times exceeded! Please try to use local call.'
+            )
+        else:
+            raise ValueError(
+                'Remote call method is invalid!We have POST and GET method.')
+
+    def _remote_parse_input(self, *args, **kwargs):
+        restored_dict = {}
+        for key, value in kwargs.items():
+            if '.' in key:
+                # Split keys by "." and create nested dictionary structures
+                keys = key.split('.')
+                temp_dict = restored_dict
+                for k in keys[:-1]:
+                    temp_dict = temp_dict.setdefault(k, {})
+                temp_dict[keys[-1]] = value
+            else:
+                # f the key does not contain ".", directly store the key-value pair into restored_dict
+                restored_dict[key] = value
+            kwargs = restored_dict
+        print('传给tool的参数：', kwargs)
+        return kwargs
+
+
+# openapi_schema_convert,register to tool_config.json
+def extract_references(schema_content):
+    references = []
+    if isinstance(schema_content, dict):
+        if '$ref' in schema_content:
+            references.append(schema_content['$ref'])
+        for key, value in schema_content.items():
+            references.extend(extract_references(value))
+    elif isinstance(schema_content, list):
+        for item in schema_content:
+            references.extend(extract_references(item))
+    return references
+
+
+def parse_nested_parameters(param_name, param_info, parameters_list, content):
+    param_type = param_info['type']
+    param_description = param_info.get('description',
+                                       f'用户输入的{param_name}')  # 按需更改描述
+    param_required = param_name in content['required']
+    try:
+        if param_type == 'object':
+            properties = param_info.get('properties')
+            if properties:
+                # If the argument type is an object and has a non-empty "properties" field,
+                # its internal properties are parsed recursively
+                for inner_param_name, inner_param_info in properties.items():
+                    inner_param_type = inner_param_info['type']
+                    inner_param_description = inner_param_info.get(
+                        'description', f'用户输入的{param_name}.{inner_param_name}')
+                    inner_param_required = param_name.split(
+                        '.')[0] in content['required']
+
+                    # Recursively call the function to handle nested objects
+                    if inner_param_type == 'object':
+                        parse_nested_parameters(
+                            f'{param_name}.{inner_param_name}',
+                            inner_param_info, parameters_list, content)
+                    else:
+                        parameters_list.append({
+                            'name':
+                            f'{param_name}.{inner_param_name}',
+                            'description':
+                            inner_param_description,
+                            'required':
+                            inner_param_required,
+                            'type':
+                            inner_param_type,
+                            'value':
+                            inner_param_info.get('enum', '')
+                        })
+        else:
+            # Non-nested parameters are added directly to the parameter list
+            parameters_list.append({
+                'name': param_name,
+                'description': param_description,
+                'required': param_required,
+                'type': param_type,
+                'value': param_info.get('enum', '')
+            })
+    except Exception as e:
+        raise ValueError(f'{e}:schema结构出错')
+
+
+def parse_responses_parameters(param_name, param_info, parameters_list):
+    param_type = param_info['type']
+    param_description = param_info.get('description',
+                                       f'调用api返回的{param_name}')  # 按需更改描述
+    try:
+        if param_type == 'object':
+            properties = param_info.get('properties')
+            if properties:
+                # If the argument type is an object and has a non-empty "properties"
+                # field, its internal properties are parsed recursively
+
+                for inner_param_name, inner_param_info in properties.items():
+                    param_type = inner_param_info['type']
+                    param_description = inner_param_info.get(
+                        'description',
+                        f'调用api返回的{param_name}.{inner_param_name}')
+                    parameters_list.append({
+                        'name': f'{param_name}.{inner_param_name}',
+                        'description': param_description,
+                        'type': param_type,
+                    })
+        else:
+            # Non-nested parameters are added directly to the parameter list
+            parameters_list.append({
+                'name': param_name,
+                'description': param_description,
+                'type': param_type,
+            })
+    except Exception as e:
+        raise ValueError(f'{e}:schema结构出错')
+
+
+def openapi_schema_convert(schema, auth):
+
+    resolver = RefResolver.from_schema(schema)
+    servers = schema.get('servers', [])
+    if servers:
+        servers_url = servers[0].get('url')
+    else:
+        print('No URL found in the schema.')
+    # Extract endpoints
+    endpoints = schema.get('paths', {})
+    description = schema.get('info', {}).get('description',
+                                             'This is a api tool that ...')
+    config_data = {}
+    # Iterate over each endpoint and its contents
+    for endpoint_path, methods in endpoints.items():
+        for method, details in methods.items():
+            summary = details.get('summary', 'No summary').replace(' ', '_')
+            name = details.get('operationId', 'No operationId')
+            url = f'{servers_url}{endpoint_path}'
+            security = details.get('security', [{}])
+            # Security (Bearer Token)
+            authorization = ''
+            if security:
+                for sec in security:
+                    if 'BearerAuth' in sec:
+                        api_token = auth.get('apikey', os.environ['apikey'])
+                        api_token_type = auth.get('apikey_type',
+                                                  os.environ['apikey_type'])
+                        authorization = f'{api_token_type} {api_token}'
+            if method.upper() == 'POST':
+                requestBody = details.get('requestBody', {})
+                if requestBody:
+                    for content_type, content_details in requestBody.get(
+                            'content', {}).items():
+                        schema_content = content_details.get('schema', {})
+                        references = extract_references(schema_content)
+                        for reference in references:
+                            resolved_schema = resolver.resolve(reference)
+                            content = resolved_schema[1]
+                            parameters_list = []
+                            for param_name, param_info in content[
+                                    'properties'].items():
+                                parse_nested_parameters(
+                                    param_name, param_info, parameters_list,
+                                    content)
+                            X_DashScope_Async = requestBody.get(
+                                'X-DashScope-Async', '')
+                            if X_DashScope_Async == '':
+                                config_entry = {
+                                    'name': name,
+                                    'description': description,
+                                    'is_active': True,
+                                    'is_remote_tool': True,
+                                    'url': url,
+                                    'method': method.upper(),
+                                    'parameters': parameters_list,
+                                    'header': {
+                                        'Content-Type': content_type,
+                                        'Authorization': authorization
+                                    }
+                                }
+                            else:
+                                config_entry = {
+                                    'name': name,
+                                    'description': description,
+                                    'is_active': True,
+                                    'is_remote_tool': True,
+                                    'url': url,
+                                    'method': method.upper(),
+                                    'parameters': parameters_list,
+                                    'header': {
+                                        'Content-Type': content_type,
+                                        'Authorization': authorization,
+                                        'X-DashScope-Async': 'enable'
+                                    }
+                                }
+                else:
+                    config_entry = {
+                        'name': name,
+                        'description': description,
+                        'is_active': True,
+                        'is_remote_tool': True,
+                        'url': url,
+                        'method': method.upper(),
+                        'parameters': [],
+                        'header': {
+                            'Content-Type': 'application/json',
+                            'Authorization': authorization
+                        }
+                    }
+            elif method.upper() == 'GET':
+                parameters_list = []
+                parameters_list = details.get('parameters', [])
+                config_entry = {
+                    'name': name,
+                    'description': description,
+                    'is_active': True,
+                    'is_remote_tool': True,
+                    'url': url,
+                    'method': method.upper(),
+                    'parameters': parameters_list,
+                    'header': {
+                        'Authorization': authorization
+                    }
+                }
+            else:
+                raise 'method is not POST or GET'
+
+            config_data[summary] = config_entry
+    return config_data

my_modelscope_agent/tools/pipeline_tool.py

@@ -0,0 +1,40 @@
+from modelscope.pipelines import pipeline
+from .tool import Tool
+
+
+class ModelscopePipelineTool(Tool):
+
+    default_model: str = ''
+    task: str = ''
+    model_revision = None
+
+    def __init__(self, cfg):
+
+        super().__init__(cfg)
+        self.model = self.cfg.get('model', None) or self.default_model
+        self.model_revision = self.cfg.get('model_revision',
+                                           None) or self.model_revision
+
+        self.pipeline_params = self.cfg.get('pipeline_params', {})
+        self.pipeline = None
+        self.is_initialized = False
+
+    def setup(self):
+
+        # only initialize when this tool is really called to save memory
+        if not self.is_initialized:
+            self.pipeline = pipeline(
+                task=self.task,
+                model=self.model,
+                model_revision=self.model_revision,
+                **self.pipeline_params)
+        self.is_initialized = True
+
+    def _local_call(self, *args, **kwargs):
+
+        self.setup()
+
+        parsed_args, parsed_kwargs = self._local_parse_input(*args, **kwargs)
+        origin_result = self.pipeline(*parsed_args, **parsed_kwargs)
+        final_result = self._parse_output(origin_result, remote=False)
+        return final_result

my_modelscope_agent/tools/plugin_tool.py

@@ -0,0 +1,30 @@
+from copy import deepcopy
+
+from .tool import Tool
+
+
+class LangchainTool(Tool):
+
+    def __init__(self, langchain_tool):
+        from langchain.tools import BaseTool
+
+        if not isinstance(langchain_tool, BaseTool):
+            raise ValueError('langchain_tool should be type of langchain tool')
+        self.langchain_tool = langchain_tool
+        self.parse_langchain_schema()
+        super().__init__()
+
+    def parse_langchain_schema(self):
+        # convert langchain tool schema to modelscope_agent tool schema
+        self.description = self.langchain_tool.description
+        self.name = self.langchain_tool.name
+        self.parameters = []
+        for name, arg in self.langchain_tool.args.items():
+            tool_arg = deepcopy(arg)
+            tool_arg['name'] = name
+            tool_arg['required'] = True
+            tool_arg.pop('title')
+            self.parameters.append(tool_arg)
+
+    def _local_call(self, *args, **kwargs):
+        return {'result': self.langchain_tool.run(kwargs)}

my_modelscope_agent/tools/text_address_tool.py

@@ -0,0 +1,20 @@
+from modelscope.utils.constant import Tasks
+from .pipeline_tool import ModelscopePipelineTool
+
+
+class TextAddressTool(ModelscopePipelineTool):
+    default_model = 'damo/mgeo_geographic_elements_tagging_chinese_base'
+    description = '地址解析服务，针对中文地址信息，识别出里面的元素，包括省、市、区、镇、社区、道路、路号、POI、楼栋号、户室号等'
+    name = 'modelscope_text-address'
+    parameters: list = [{
+        'name': 'input',
+        'description': '用户输入的地址信息',
+        'required': True
+    }]
+    task = Tasks.token_classification
+
+    def _parse_output(self, origin_result, *args, **kwargs):
+        final_result = {}
+        for e in origin_result['output']:
+            final_result[e['type']] = e['span']
+        return {'result': final_result}

my_modelscope_agent/tools/text_ie_tool.py

@@ -0,0 +1,32 @@
+from collections import defaultdict
+
+from modelscope.utils.constant import Tasks
+from .pipeline_tool import ModelscopePipelineTool
+
+
+class TextInfoExtractTool(ModelscopePipelineTool):
+    default_model = 'damo/nlp_structbert_siamese-uie_chinese-base'
+    description = '信息抽取服务，针对中文的文本，根据schema要抽取的内容，找出其中对应信息，并用json格式展示'
+    name = 'modelscope_text-ie'
+    parameters: list = [{
+        'name': 'input',
+        'description': '用户输入的文本',
+        'required': True
+    }, {
+        'name': 'schema',
+        'description': '要抽取信息的json表示',
+        'required': True
+    }]
+    task = Tasks.siamese_uie
+
+    def _remote_parse_input(self, *args, **kwargs):
+        kwargs['parameters'] = {'schema': kwargs['schema']}
+        kwargs.pop('schema')
+        return kwargs
+
+    def _parse_output(self, origin_result, *args, **kwargs):
+        final_result = defaultdict(list)
+        for e in origin_result['output']:
+            final_result[e[0]['type']].append(e[0]['span'])
+
+        return {'result': dict(final_result)}