Upload 237 files

.env

@@ -0,0 +1,58 @@
+# LLM import use litellm lib (https://docs.litellm.ai/docs/)
+# OPENAI | AZURE | OR other LLM PREFIX
+LLM_SOURCE='OPENAI'
+
+LLM_TEMPERATURE='0.1'
+
+# open ai
+OPENAI_API_KEY='sk-666'
+OPENAI_API_BASE='https://api.zhtec.xyz/xht/chatWith16k.php'
+OPENAI_EMBEDDING_MODEL='text-embedding-ada-002'
+OPENAI_LLM_MODEL_NORMAL='gpt-3.5-turbo-16k'
+OPENAI_LLM_MODEL_SMART='gpt-4'
+OPENAI_LLM_MODEL_SMART_LIMIT='8000'
+OPENAI_LLM_MODEL_SMART_LONG='gpt-4-32k'
+OPENAI_LLM_MODEL_SMART_LONG_LIMIT='32000'
+OPENAI_LLM_MODEL_LONG='gpt-4-1106-preview'
+OPENAI_LLM_MODEL_VISION='gpt-4-vision-preview'
+OPENAI_LLM_MODEL_ALL='gpt-4-all'
+
+# azure open ai
+AZURE_API_KEY='xx'
+AZURE_API_BASE='xx'
+AZURE_API_VERSION='2023-06-01-preview'
+AZURE_EMBEDDING_MODEL='azure/ada002'
+AZURE_LLM_MODEL_NORMAL='azure/gpt35t'
+AZURE_LLM_MODEL_SMART='azure/gpt4'
+OPENAI_LLM_MODEL_SMART_LIMIT='8000'
+AZURE_LLM_MODEL_LONG='azure/gpt4'
+
+# replicate
+REPLICATE_API_TOKEN='r8_E2UIynoq1yuY5ErDijgm4uR1C2pW8pG4IPXU7'
+
+## user data directory, store user's applications、functions、application_datas and server datas
+DATA_DIR='/workspace/data/'
+
+# cache llm inference and embedding, useful when develop and debug
+LLM_CACHE='no'
+EMBEDDING_CACHE='yes'
+# CACHE_PATH='./llm_cache.json'
+
+# google search tool at https://google.serper.dev
+SERPER_API_KEY='7a7a62b5f4a991043dd7fe96553802fb7675927c'
+
+# Whether to automatically run python, shell, applescript and other codes
+# Default no: n
+AUTO_RUN='y'
+
+# Logging level
+# Default: INFO, can be DEBUG, INFO, WARNING, ERROR
+LOG_LEVEL='INFO'
+
+## ui builder directory, webui server(webui/server/server/app.py) will automatically set TSX_BUILDER_DIR to webui/server/server/ts_builder
+# TSX_BUILDER_DIR
+
+## local applications code directory. webui server(webui/server/server/app.py) will automatically set LOCAL_APPLICATIONS_DIR to webui/server/server/applicatoins
+# LOCAL_APPLICATIONS_DIR
+
+# export $(grep -v '^#' .env | sed 's/^export //g' | xargs)

.gitattributes

@@ -35,3 +35,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 GeneralAgent/GeneralAgent/skills/merge_video_audio/music.wav filter=lfs diff=lfs merge=lfs -text
 GeneralAgent/test/data/Nougat.pdf filter=lfs diff=lfs merge=lfs -text
+GeneralAgent/skills/merge_video_audio/music.wav filter=lfs diff=lfs merge=lfs -text
+test/data/Nougat.pdf filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,40 @@
+*.pyc
+GeneralAgent/.env
+GeneralAgent/*.json
+
+test/data_0.json
+test/data/test_workspace/*
+test/tetris.py
+test/tetris.py.bak
+GeneralAgent/cache_json
+test/data/plan_memory.json
+test/data/test_interpreter.bin
+test/data/b.txt
+test/data/a.txt
+test/data/hello.pptx
+test/.env
+
+build/*
+dist/*
+GeneralAgent.egg-info*
+test/multi_lines_input/*
+test/multi_lines_input/*
+.env
+examples/memory.json
+test/link_memory.json
+test/memory.json
+test/llm_cache.json
+test/summary_memory.json
+*/llm_cache.json
+test/test_skills/data/*
+test/test_skills/llm_cache.json
+webui/server/server/applications/test_application_id/bot.json
+webui/server/server/applications/test_application_id/main.py
+data/*
+
+.idea/*
+test/test_skills/code/*
+test/data/ui/*
+test/code/*
+test/data/read_interpreter/*
+webui/server/server/ts_builder/src/lib/index.tsx

GeneralAgent/__init__.py

@@ -0,0 +1 @@
+from .skills import skills

GeneralAgent/agent/__init__.py

@@ -0,0 +1,14 @@
+# import 
+from .normal_agent import NormalAgent
+from .stack_agent import StackAgent
+
+class Agent(NormalAgent):
+    pass
+
+import os
+LLM_SOURCE = os.environ.get('LLM_SOURCE', None)
+if LLM_SOURCE is None:
+    print('enviroment variable LLM_SOURCE not available.')
+
+from GeneralAgent.utils import set_logging_level
+set_logging_level()

GeneralAgent/agent/abs_agent.py

@@ -0,0 +1,83 @@
+import abc
+import os
+import asyncio
+from GeneralAgent.utils import default_get_input, default_output_callback
+
+
+class AbsAgent(metaclass=abc.ABCMeta):
+    """
+    Abstract Agent
+    @memory: Memory
+    @interpreters: list, interpreters
+    @model_type: str, 'normal' or 'smart' or 'long'. For OpenAI api, normal=gpt3.5, smart=gpt4, long=gpt3.5-16k
+    @hide_output_parse: bool, hide the llm's output that output interpreters will parse, default True
+    """
+    memory = None
+    interpreters = []
+    model_type = 'normal'
+    hide_output_parse = False
+
+    def add_role_prompt(self, prompt):
+        """
+        add role prompt
+        """
+        if len(self.interpreters) > 0 and self.interpreters[0].__class__.__name__ == 'RoleInterpreter':
+            role_interpreter = self.interpreters[0]
+            if role_interpreter.system_prompt is not None:
+                role_interpreter.system_prompt += '\n' + prompt
+            else:
+                role_interpreter.system_prompt_template += '\n' + prompt
+
+    @abc.abstractmethod
+    async def run(self, input=None, output_callback=default_output_callback, input_for_memory_node_id=-1):
+        """
+        input: str, user's new input, None means continue to run where it stopped
+        input_for_memory_node_id: int, -1 means input is not from memory, None means input new, otherwise input is for memory node
+        output_callback: async function, output_callback(content: str) -> None
+        """
+        pass
+
+    def stop(self):
+        """
+        stop the agent
+        """
+        self.stop_event.set()
+
+
+    def __init__(self, workspace='./'):
+        """
+        @workspace: str, workspace path
+        """
+        if not os.path.exists(workspace):
+            os.makedirs(workspace)
+        self.is_running = False
+        self.stop_event = asyncio.Event()
+        self.workspace = workspace
+
+    @classmethod
+    def empty(cls, workspace='./'):
+        """
+        empty agent, only role interpreter and memory, work like a basic LLM chatbot
+        @workspace: str, workspace path
+        """
+        pass
+
+    @classmethod
+    def default(cls, workspace='./', retrieve_type='embedding'):
+        """
+        default agent, with all interpreters
+        @workspace: str, workspace path
+        @retrieve_type: str, 'embedding' or 'link'
+        """
+        pass
+    
+    @classmethod
+    def with_functions(cls, functions, role_prompt=None, workspace = './', model_type='smart'):
+        """
+        functions: list, [function1, function2, ...]
+        @role_prompt: str, role prompt
+        @workspace: str, workspace path
+        @import_code: str, import code
+        @libs: str, libs
+        """
+        pass

GeneralAgent/agent/normal_agent.py

@@ -0,0 +1,189 @@
+# Agent
+import os, re
+import asyncio
+import logging
+from GeneralAgent.utils import default_get_input, default_output_callback
+from GeneralAgent.memory import NormalMemory
+from GeneralAgent.interpreter import Interpreter
+from GeneralAgent.interpreter import EmbeddingRetrieveInterperter, LinkRetrieveInterperter
+from GeneralAgent.interpreter import RoleInterpreter, PythonInterpreter, ShellInterpreter, AppleScriptInterpreter, FileInterpreter
+from .abs_agent import AbsAgent
+
+
+class NormalAgent(AbsAgent):
+
+    def __init__(self, workspace='./'):
+        super().__init__(workspace)
+        self.memory = NormalMemory(serialize_path=f'{workspace}/normal_memory.json')
+
+    @classmethod
+    def empty(cls, workspace='./'):
+        """
+        empty agent, only role interpreter and memory, work like a basic LLM chatbot
+        """
+        agent = cls(workspace)
+        agent.interpreters = [RoleInterpreter()]
+        return agent
+
+    @classmethod
+    def default(cls, workspace='./', retrieve_type='embedding'):
+        """
+        default agent, with all interpreters
+        @workspace: str, workspace path
+        @retrieve_type: str, 'embedding' or 'link'
+        """
+        agent = cls(workspace)
+        # memory
+        # interpreters
+        role_interpreter = RoleInterpreter()
+        python_interpreter = PythonInterpreter(serialize_path=f'{workspace}/code.bin')
+        if retrieve_type == 'embedding':
+            retrieve_interperter = EmbeddingRetrieveInterperter(serialize_path=f'{workspace}/read_interperter/')
+        else:
+            retrieve_interperter = LinkRetrieveInterperter(python_interpreter)
+        bash_interpreter = ShellInterpreter(workspace)
+        applescript_interpreter = AppleScriptInterpreter()
+        file_interpreter = FileInterpreter()
+        agent.interpreters = [role_interpreter, retrieve_interperter, python_interpreter, bash_interpreter, applescript_interpreter, file_interpreter]
+        return agent
+
+    @classmethod
+    def with_functions(cls, functions, role_prompt=None, workspace = './', model_type='smart'):
+        """
+        functions: list, [function1, function2, ...]
+        @role_prompt: str, role prompt
+        @workspace: str, workspace path
+        @import_code: str, import code
+        @libs: str, libs
+        """
+        agent = cls(workspace)
+        role_interpreter = RoleInterpreter()
+        python_interpreter = PythonInterpreter(serialize_path=f'{workspace}/code.bin')
+        python_interpreter.function_tools = functions
+        agent.interpreters = [role_interpreter, python_interpreter, ShellInterpreter()]
+        agent.model_type = model_type
+        if role_prompt is not None:
+            agent.add_role_prompt(role_prompt)
+        return agent
+
+    
+    async def run(self, input=None, output_callback=default_output_callback, input_for_memory_node_id=-1):
+        """
+        agent run: parse intput -> get llm messages -> run LLM and parse output
+        @input: str, user's new input, None means continue to run where it stopped
+        @input_for_memory_node_id: int, -1 means input is not from memory, None means input new, otherwise input is for memory node
+        @output_callback: async function, output_callback(content: str) -> None
+        """
+        self.is_running = True
+
+        input_stop = await self._parse_input(input, output_callback)
+        if input_stop:
+            self.is_running = False
+            return
+
+        while True:
+            messages = await self._get_llm_messages()
+            output_stop = await self._llm_and_parse_output(messages, output_callback)
+            if output_stop:
+                self.is_running = False
+                return
+            await asyncio.sleep(0)
+            if self.stop_event.is_set():
+                self.is_running = False
+                return
+
+    async def _parse_input(self, input, output_callback):
+        self.memory.add_message('user', input)
+        input_content = input
+        input_stop = False
+        interpreter:Interpreter = None
+        for interpreter in self.interpreters:
+            if interpreter.input_match(input_content):
+                logging.info('interpreter: ' + interpreter.__class__.__name__)
+                parse_output, case_is_stop = await interpreter.input_parse(input_content)
+                if case_is_stop:
+                    await output_callback(parse_output)
+                    input_stop = True
+        return input_stop
+    
+    async def _get_llm_messages(self):
+        from GeneralAgent import skills
+        messages = self.memory.get_messages()
+        token_limit = skills.get_llm_token_limit(self.model_type)
+        messages = skills.cut_messages(messages, int(token_limit*0.8))
+        system_prompt = '\n\n'.join([await interpreter.prompt(messages) for interpreter in self.interpreters])
+        messages = [{'role': 'system', 'content': system_prompt}] + messages
+        return messages
+
+    async def _llm_and_parse_output(self, messages, output_callback):
+        from GeneralAgent import skills
+        try:
+            result = ''
+            is_stop = True
+            is_break = False
+            in_parse_content = False
+            cache_tokens = []
+            response = skills.llm_inference(messages, model_type=self.model_type, stream=True)
+            for token in response:
+                if token is None: break
+                result += token
+                # logging.debug(result)
+                # print(token)
+                if self.hide_output_parse:
+                    if not in_parse_content:
+                        interpreter:Interpreter = None
+                        for interpreter in self.interpreters:
+                            is_start_matched, string_matched = interpreter.output_match_start(result)
+                            if is_start_matched:
+                                in_parse_content = True
+                                # clear cache
+                                cache_tokens.append(token)
+                                left_count = len(string_matched)
+                                while left_count > 0:
+                                    left_count -= len(cache_tokens[-1])
+                                    cache_tokens.remove(cache_tokens[-1])
+                                while len(cache_tokens) > 0:
+                                    pop_token = cache_tokens.pop(0)
+                                    await output_callback(pop_token)
+                        if not in_parse_content:
+                            # cache token
+                            cache_tokens.append(token)
+                            if len(cache_tokens) > 5:
+                                pop_token = cache_tokens.pop(0)
+                                await output_callback(pop_token)
+                else:
+                    await output_callback(token)
+                interpreter:Interpreter = None
+                for interpreter in self.interpreters:
+                    if interpreter.output_match(result):
+                        logging.info('interpreter: ' + interpreter.__class__.__name__)
+                        output, is_stop = await interpreter.output_parse(result)
+                        if interpreter.outptu_parse_done_recall is not None:
+                            await interpreter.outptu_parse_done_recall()
+                        if self.hide_output_parse:
+                            is_matched, string_left = interpreter.output_match_end(result)
+                            await output_callback(string_left)
+                        while len(cache_tokens) > 0:
+                            pop_token = cache_tokens.pop(0)
+                            await output_callback(pop_token)
+                        result += '\n' + output.strip() + '\n'
+                        # logging.debug(result)
+                        if not self.hide_output_parse or is_stop:
+                            await output_callback('\n' + output.strip() + '\n')
+                        is_break = True
+                        in_parse_content = False
+                        break
+                if is_break:
+                    break
+            while len(cache_tokens) > 0:
+                pop_token = cache_tokens.pop(0)
+                await output_callback(pop_token)
+            # append messages
+            # logging.debug(result)
+            self.memory.append_message('assistant', result)
+            return is_stop
+        except Exception as e:
+            # if fail, recover
+            logging.exception(e)
+            await output_callback(str(e))
+            return True

GeneralAgent/agent/stack_agent.py

@@ -0,0 +1,221 @@
+# Agent
+import os, re
+import asyncio
+import logging
+from GeneralAgent.utils import default_get_input, default_output_callback
+from GeneralAgent.memory import StackMemory, StackMemoryNode
+from GeneralAgent.interpreter import Interpreter
+from GeneralAgent.interpreter import PlanInterpreter, EmbeddingRetrieveInterperter, LinkRetrieveInterperter
+from GeneralAgent.interpreter import RoleInterpreter, PythonInterpreter, ShellInterpreter, AppleScriptInterpreter, FileInterpreter
+from .abs_agent import AbsAgent
+
+
+class StackAgent(AbsAgent):
+
+    def __init__(self, workspace='./'):
+        super().__init__(workspace)
+        self.memory = StackMemory(serialize_path=f'{workspace}/memory.json')
+
+    @classmethod
+    def empty(cls, workspace='./'):
+        """
+        empty agent, only role interpreter and memory, work like a basic LLM chatbot
+        """
+        agent = cls(workspace)
+        agent.interpreters = [RoleInterpreter()]
+        return agent
+
+    @classmethod
+    def default(cls, workspace='./', retrieve_type='embedding'):
+        """
+        default agent, with all interpreters
+        @workspace: str, workspace path
+        @retrieve_type: str, 'embedding' or 'link'
+        """
+        agent = cls(workspace)
+        # memory
+        # interpreters
+        role_interpreter = RoleInterpreter()
+        python_interpreter = PythonInterpreter(serialize_path=f'{workspace}/code.bin')
+        plan_interperter = PlanInterpreter(agent.memory)
+        if retrieve_type == 'embedding':
+            retrieve_interperter = EmbeddingRetrieveInterperter(serialize_path=f'{workspace}/read_interperter/')
+        else:
+            retrieve_interperter = LinkRetrieveInterperter(python_interpreter)
+        bash_interpreter = ShellInterpreter(workspace)
+        applescript_interpreter = AppleScriptInterpreter()
+        file_interpreter = FileInterpreter()
+        agent.interpreters = [role_interpreter, plan_interperter, retrieve_interperter, python_interpreter, bash_interpreter, applescript_interpreter, file_interpreter]
+        return agent
+
+    @classmethod
+    def with_functions(cls, functions, role_prompt=None, workspace = './', model_type='smart'):
+        """
+        functions: list, [function1, function2, ...]
+        @role_prompt: str, role prompt
+        @workspace: str, workspace path
+        @import_code: str, import code
+        @libs: str, libs
+        """
+        agent = cls(workspace)
+        role_interpreter = RoleInterpreter(system_prompt=role_prompt)
+        python_interpreter = PythonInterpreter(serialize_path=f'{workspace}/code.bin')
+        python_interpreter.function_tools = functions
+        agent.interpreters = [role_interpreter, python_interpreter, ShellInterpreter()]
+        agent.model_type = model_type
+        return agent
+
+    
+    async def run(self, input=None, output_callback=default_output_callback, input_for_memory_node_id=-1):
+        """
+        agent run: parse intput -> get llm messages -> run LLM and parse output
+        input: str, user's new input, None means continue to run where it stopped
+        input_for_memory_node_id: int, -1 means input is not from memory, None means input new, otherwise input is for memory node
+        output_callback: async function, output_callback(content: str) -> None
+        """
+        self.is_running = True
+        
+        if input_for_memory_node_id == -1:
+            memory_node_id = self.memory.current_node.node_id if self.memory.current_node is not None else None
+        else:
+            memory_node_id = input_for_memory_node_id
+        input_node = self._insert_node(input, memory_node_id) if input is not None else None
+
+        # input interpreter
+        if input_node is not None:
+            input_content = input
+            input_stop = False
+            self.memory.set_current_node(input_node)
+            interpreter:Interpreter = None
+            for interpreter in self.interpreters:
+                if interpreter.input_match(input_content):
+                    logging.info('interpreter: ' + interpreter.__class__.__name__)
+                    # await output_callback('input parsing\n')
+                    input_content, case_is_stop = await interpreter.input_parse(input_content)
+                    if case_is_stop:
+                        await output_callback(input_content)
+                        input_stop = True
+            input_node.content = input_content
+            self.memory.update_node(input_node)
+            if input_stop:
+                self.memory.success_node(input_node)
+                self.is_running = False
+                return input_node.node_id
+
+        # execute todo node from memory
+        todo_node = self.memory.get_todo_node() or input_node
+        logging.debug(self.memory)
+        while todo_node is not None:
+            new_node, is_stop = await self._execute_node(todo_node, output_callback)
+            logging.debug(self.memory)
+            logging.debug(new_node)
+            logging.debug(is_stop)
+            if is_stop:
+                return new_node.node_id
+            todo_node = self.memory.get_todo_node()
+            await asyncio.sleep(0)
+            if self.stop_event.is_set():
+                self.is_running = False
+                return None
+        self.is_running = False
+        return None
+
+    def _insert_node(self, input, memory_node_id=None):
+        node = StackMemoryNode(role='user', action='input', content=input)
+        if memory_node_id is None:
+            logging.debug(self.memory)
+            self.memory.add_node(node)
+        else:
+            for_node = self.memory.get_node(memory_node_id)
+            self.memory.add_node_after(for_node, node)
+            self.memory.success_node(for_node)
+        return node
+    
+    async def _execute_node(self, node, output_callback):
+        # construct system prompt
+        from GeneralAgent import skills
+        messages = self.memory.get_related_messages_for_node(node)
+        messages = skills.cut_messages(messages, 3000)
+        system_prompt = '\n\n'.join([await interpreter.prompt(messages) for interpreter in self.interpreters])
+        messages = [{'role': 'system', 'content': system_prompt}] + messages
+
+        # add answer node and set current node
+        answer_node = StackMemoryNode(role='system', action='answer', content='')
+        self.memory.add_node_after(node, answer_node)
+        self.memory.set_current_node(answer_node)
+
+        if node.action == 'plan':
+            await output_callback(f'\n[{node.content}]\n')
+
+        try:
+            result = ''
+            is_stop = False
+            is_break = False
+            in_parse_content = False
+            cache_tokens = []
+            response = skills.llm_inference(messages, model_type=self.model_type, stream=True)
+            for token in response:
+                if token is None: break
+                result += token
+                # print(token)
+                if self.hide_output_parse:
+                    if not in_parse_content:
+                        interpreter:Interpreter = None
+                        for interpreter in self.interpreters:
+                            is_start_matched, string_matched = interpreter.output_match_start(result)
+                            if is_start_matched:
+                                in_parse_content = True
+                                # clear cache
+                                cache_tokens.append(token)
+                                left_count = len(string_matched)
+                                while left_count > 0:
+                                    left_count -= len(cache_tokens[-1])
+                                    cache_tokens.remove(cache_tokens[-1])
+                                while len(cache_tokens) > 0:
+                                    pop_token = cache_tokens.pop(0)
+                                    await output_callback(pop_token)
+                        if not in_parse_content:
+                            # cache token
+                            cache_tokens.append(token)
+                            if len(cache_tokens) > 5:
+                                pop_token = cache_tokens.pop(0)
+                                await output_callback(pop_token)
+                else:
+                    await output_callback(token)
+                interpreter:Interpreter = None
+                for interpreter in self.interpreters:
+                    if interpreter.output_match(result):
+                        logging.info('interpreter: ' + interpreter.__class__.__name__)
+                        output, is_stop = await interpreter.output_parse(result)
+                        if interpreter.outptu_parse_done_recall is not None:
+                            await interpreter.outptu_parse_done_recall()
+                        if self.hide_output_parse:
+                            is_matched, string_left = interpreter.output_match_end(result)
+                            await output_callback(string_left)
+                        result += '\n' + output.strip() + '\n'
+                        if not self.hide_output_parse or is_stop:
+                            await output_callback('\n' + output.strip() + '\n')
+                        is_break = True
+                        in_parse_content = False
+                        break
+                if is_break:
+                    break
+            while len(cache_tokens) > 0:
+                pop_token = cache_tokens.pop(0)
+                await output_callback(pop_token)
+            # await output_callback('\n')
+            # update current node and answer node
+            answer_node.content = result
+            self.memory.update_node(answer_node)
+            self.memory.success_node(node)
+            # llm run end with any interpreter, success the node
+            if not is_break:
+                self.memory.success_node(answer_node)
+            return answer_node, is_stop
+        except Exception as e:
+            # if fail, recover
+            logging.exception(e)
+            await output_callback(str(e))
+            self.memory.delete_node(answer_node)
+            self.memory.set_current_node(node)
+            return node, True

GeneralAgent/cli.py

@@ -0,0 +1,39 @@
+import os
+import asyncio
+import datetime
+import argparse
+from GeneralAgent.agent import Agent
+from GeneralAgent.utils import default_get_input, set_logging_level
+
+
+async def _main(args):
+    if args.auto_run:
+        os.environ['AUTO_RUN'] = 'y'
+    else:
+        os.environ['AUTO_RUN'] = 'n'
+    set_logging_level(os.environ.get('LOG_LEVEL', 'ERROR'))
+    workspace = args.workspace
+    if args.new:
+        if os.path.exists(workspace):
+            workspace = workspace + '_' + datetime.datetime.now().strftime('%Y%m%d%H%M%S')
+            print('New workspace: ', workspace)
+    if not os.path.exists(workspace):
+        os.mkdir(workspace)
+    
+    agent = Agent.default(workspace=workspace)
+    print('You can input multi lines, enter twice to end')
+    while True:
+        input_content = default_get_input()
+        print('[output]\n', end='', flush=True)
+        await agent.run(input_content)
+
+def main():
+    parser = argparse.ArgumentParser(description='GeneralAgent CLI')
+    parser.add_argument('--workspace', default='./general_agent', help='Set workspace directory')
+    parser.add_argument('--new', action='store_true', help='Enable new workspace')
+    parser.add_argument('--auto_run', action='store_true', help='Auto run code without confirm')
+    args = parser.parse_args()
+    asyncio.run(_main(args))
+
+if __name__ == '__main__':
+    main()

GeneralAgent/interpreter/__init__.py

@@ -0,0 +1,22 @@
+# import 
+from .interpreter import Interpreter
+
+# role
+from .role_interpreter import RoleInterpreter
+
+# input
+from .plan_interpreter import PlanInterpreter
+
+# retrieve
+from .embedding_retrieve_interpreter import EmbeddingRetrieveInterperter
+from .link_retrieve_interpreter import LinkRetrieveInterperter
+
+# output
+from .applescript_interpreter import AppleScriptInterpreter
+from .file_interpreter import FileInterpreter
+from .python_interpreter import SyncPythonInterpreter
+from .python_interpreter import AsyncPythonInterpreter
+from .shell_interpreter import ShellInterpreter
+from .ui_interpreter import UIInterpreter
+class PythonInterpreter(AsyncPythonInterpreter):
+    pass

GeneralAgent/interpreter/applescript_interpreter.py

@@ -0,0 +1,46 @@
+import re
+from .interpreter import Interpreter
+from GeneralAgent.utils import confirm_to_run
+
+applescript_promt = """
+# Run applescript
+* Here are the commands
+```applescript
+<applescript_command>
+```
+* the command will be executed if in macOS computer.
+"""
+
+class AppleScriptInterpreter(Interpreter):
+    
+    output_match_pattern = '```(\n)?applescript(.*?)\n```'
+    output_match_start_pattern = '```(\n)?applescript'
+
+    async def prompt(self, messages) -> str:
+        return applescript_promt
+    
+    async def output_parse(self, string) -> (str, bool):
+        pattern = re.compile(self.output_match_pattern, re.DOTALL)
+        match = pattern.search(string)
+        assert match is not None
+        if confirm_to_run():
+            sys_out = self._run_applescript(match.group(2))
+            return sys_out.strip(), False
+        else:
+            return '', False
+
+    def _run_applescript(self, content):
+        content = content.replace('"', '\\"')
+        sys_out = ''
+        import subprocess
+        try:
+            p = subprocess.Popen('osascript -e "{}"'.format(content), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        except:
+            pass
+        finally:
+            sys_out, err = p.communicate()
+            sys_out = sys_out.decode('utf-8')
+        sys_out = sys_out.strip()
+        if sys_out == '':
+            sys_out = 'run successfully'
+        return sys_out

GeneralAgent/interpreter/embedding_retrieve_interpreter.py

@@ -0,0 +1,77 @@
+# read the document and can retrieve the information
+import re
+from .interpreter import Interpreter
+import chromadb
+import logging
+
+class EmbeddingRetrieveInterperter(Interpreter):
+    """
+    EmbeddingRetrieveInterperter can retrieve the information from the memory by embedding.
+    """
+    
+    input_match_pattern = '```read\n(.*?)\n```'
+
+    def __init__(self, serialize_path='./read_data/', prompt_max_length=1000, useful_msg_count=2) -> None:
+        self.prompt_max_length = prompt_max_length
+        self.useful_msg_count = useful_msg_count
+        self.client = chromadb.PersistentClient(path=serialize_path)
+        self.collection = self.client.get_or_create_collection(name="read", metadata={"hnsw:space": "cosine"})
+
+    async def prompt(self, messages) -> str:
+        from GeneralAgent import skills
+        # when collection is empty, return empty string
+        if self.collection.count() == 0:
+            return ''
+        
+        querys = []
+        for x in messages[-self.useful_msg_count:]:
+            querys += skills.split_text(x['content'], 200)
+        query_embeddings = skills.embedding_batch(querys)
+        result = self.collection.query(
+            query_embeddings=query_embeddings,
+            n_results=2,
+        )
+        # extract distances and documents
+        distances = [x for z in result['distances'] for x in z]
+        documents = [x for z in result['documents'] for x in z]
+
+        # sort distances and documents by distance
+        sorted_docs = sorted(list(zip(distances, documents)), key=lambda x: x[0])
+
+        # filter documents with distance < 100
+        documents = [x for d, x in sorted_docs if d < 100]
+        # texts token count should less than prompt_max_length
+        texts = []
+        texts_token_count = 0
+        for x in documents:
+            if texts_token_count + skills.string_token_count(x) > self.prompt_max_length:
+                break
+            texts.append(x)
+        return '\n'.join(texts)
+    
+    async def input_parse(self, string) -> (str, bool):
+        from GeneralAgent import skills
+        information = []
+        pattern = re.compile(self.input_match_pattern, re.DOTALL)
+        match = pattern.search(string)
+        assert match is not None
+        file_paths = match.group(1).strip().split('\n')
+        for file_path in file_paths:
+            paragraphs = skills.split_text(skills.read_file_content(file_path), max_token=300)
+            if len(paragraphs) > 0:
+                information.append(f'The content of file {file_path} is: ' + '\n'.join(paragraphs)[:100] + '\n......')
+            embeddings = skills.embedding_batch(paragraphs)
+            # logging.debug(paragraphs[:2])
+            # logging.debug(embeddings[:2])
+            self.collection.add(
+                documents=paragraphs,
+                embeddings=embeddings,
+                metadatas=[{'file_path': file_path} for _ in paragraphs],
+                ids=[file_path+str(i) for i in range(len(paragraphs))],
+            )
+        stop = False
+        if string.replace(match.group(0), '').strip() == '':
+            stop = True
+        info = '\n'.join(information)
+        string = f'\n{string}\n```{info}```\n'
+        return string, stop

GeneralAgent/interpreter/file_interpreter.py

@@ -0,0 +1,120 @@
+import re, os
+import logging
+from .interpreter import Interpreter
+
+file_prompt = """
+# For file operations, ALWAYS enclose your commands in triple backticks (```). Here are the commands:
+
+1. Write: 
+```file
+<file_path> write <start_line> <end_line> <<EOF
+<content>
+EOF
+```
+2. Read: 
+```file
+<file_path> read <start_line> <end_line>
+```
+3. Delete:
+```file
+<file_path> delete <start_line> <end_line>
+```
+
+Line numbers start from 0, and -1 is the last line.
+Read will print the content of the file with [line numbers] prefixed.
+"""
+
+
+class FileInterpreter(Interpreter):
+    
+    output_match_pattern = '```(\n)?file(\n| )?(.*?) (write|read|delete) (-?\d+) (-?\d+)(.*?)```'
+
+    async def prompt(self, messages) -> str:
+        return file_prompt
+    
+    def _parse_commands(self, string):
+        match = re.search(self.output_match_pattern, string, re.DOTALL)
+        assert match is not None
+        file_path = match.group(3)
+        operation = match.group(4)
+        start_line = int(match.group(5))
+        end_line = int(match.group(6))
+        content = match.group(7).strip()
+        if content.startswith('<<EOF'):
+            content = content[5:].strip()
+        if content.endswith('EOF'):
+            content = content[:-3].strip()
+        return file_path, operation, start_line, end_line, content
+
+    async def output_parse(self, string) -> (str, bool):
+        logging.debug('FileInterpreter:parse called')
+        file_path, operation, start_line, end_line, content = self._parse_commands(string)
+        is_stop = True
+        if operation == 'write':
+            self._write_file(file_path, content, start_line, end_line)
+            return f'Content write to {file_path} successfully\n', is_stop
+        elif operation == 'delete':
+            self._delete_file(file_path, start_line, end_line)
+            return f'Delete lines of {file_path} successfully\n', is_stop
+        elif operation == 'read':
+            content = self._read_file(file_path, start_line, end_line)
+            return f'Read {file_path} succesfully, the content is below: \n\n```\n{content}\n```\n', is_stop
+    
+    def _write_file(self, file_path, content, start_index, end_index):
+        # if .py file, remove ```python  and ``` pair
+        if file_path.endswith('.py'):
+            content = content.replace('```python', '')
+            content = content.replace('```', '')
+        # file_path = os.path.join(self.workspace, file_path)
+        dir_path = os.path.dirname(file_path)
+        if not os.path.exists(dir_path):
+            os.makedirs(dir_path)
+        if not os.path.exists(file_path):
+            with open(file_path, 'w', encoding='utf-8') as f:
+                f.write('')
+        with open(file_path, 'r', encoding='utf-8') as f:
+            lines = f.readlines()
+        if start_index == -1:
+            start_index = len(lines)
+        if end_index == -1:
+            end_index = len(lines)
+        lines = lines[:start_index] + [content] + lines[end_index+1:]
+        with open(file_path, 'w', encoding='utf-8') as f:
+            f.writelines(lines)
+
+    def _delete_file(self, file_path, start_index, end_index):
+        # file_path = os.path.join(self.workspace, file_path)
+        if not os.path.exists(file_path):
+            with open(file_path, 'w', encoding='utf-8') as f:
+                f.write('')
+        with open(file_path, 'r', encoding='utf-8') as f:
+            lines = f.readlines()
+        if start_index == -1:
+            start_index = len(lines)
+        if end_index == -1:
+            end_index = len(lines)
+        lines = lines[:start_index] + lines[end_index+1:]
+        with open(file_path, 'w', encoding='utf-8') as f:
+            f.writelines(lines)
+
+    def _read_file(self, file_path, start_index, end_index):
+        # file_path = os.path.join(self.workspace, file_path)
+        if not os.path.exists(file_path):
+            with open(file_path, 'w', encoding='utf-8') as f:
+                f.write('')
+        with open(file_path, 'r', encoding='utf-8') as f:
+            lines = f.readlines()
+        if start_index == -1:
+            start_index = len(lines)
+        if end_index == -1:
+            end_index = len(lines)
+        content = ''
+        end_index = min(end_index + 1, len(lines))
+        for index in range(start_index, end_index):
+            new_add = f'[{index}]{lines[index]}'
+            if len(content + new_add) > 2000:
+                left_count = len(lines) - index
+                content += f'...\n[there are {left_count} lines left]'
+                break
+            content += new_add
+        return content.strip()

GeneralAgent/interpreter/interpreter.py

@@ -0,0 +1,80 @@
+# Interpreter
+import abc
+import re
+
+class Interpreter(metaclass=abc.ABCMeta):
+    """
+    Interpreter is the base class for all interpreters.
+    input_match_pattern is the pattern to match the LLM input string. for example ```tsx\n(.*?)\n```
+    output_match_pattern is the pattern to match the LLM ouput string. for example ```tsx\n(.*?)\n```
+    output_match_start_pattern is the pattern to match the start of the LLM output string. for example: ```tsx\n
+    outptu_parse_done_recall is the callback function to recall when the output_parse is done
+    """
+    input_match_pattern = None
+    output_match_pattern = None
+    output_match_start_pattern = None
+    outptu_parse_done_recall = None
+
+    async def prompt(self, messages) -> str:
+        """
+        :param messages: list of messages
+        :return: string
+        """
+        return ''
+    
+    def input_match(self, string) -> bool:
+        if self.input_match_pattern is None:
+            return False
+        match = re.compile(self.input_match_pattern, re.DOTALL).search(string)
+        if match is not None:
+            return True
+        else:
+            return False
+
+    def output_match(self, string) -> bool:
+        if self.output_match_pattern is None:
+            return False
+        match = re.compile(self.output_match_pattern, re.DOTALL).search(string)
+        if match is not None:
+            return True
+        else:
+            return False
+        
+    def output_match_start(self, string) -> (bool, str):
+        """
+        return is_match, string_matched
+        """
+        if self.output_match_start_pattern is None:
+            return False, ''
+        match = re.compile(self.output_match_start_pattern, re.DOTALL).search(string)
+        if match is not None:
+            string_matched = string[match.start():]
+            return True, string_matched
+        else:
+            return False, ''
+        
+    def output_match_end(self, string) -> (bool, str):
+        """
+        return is_match, string_matched
+        """
+        if self.output_match_pattern is None:
+            return False, ''
+        match = re.compile(self.output_match_pattern, re.DOTALL).search(string)
+        if match is not None:
+            string_left = string[match.end():]
+            return True, string_left
+        else:
+            return False, ''
+        
+    async def input_parse(self, string) -> (str, bool):
+        """
+        parse the input、output string, and return the output string and is_stop
+        """
+        return '', False
+
+    async def output_parse(self, string) -> (str, bool):
+        """
+        parse the input、output string, and return the output string and is_stop
+        """
+        return '', False
+    

GeneralAgent/interpreter/link_retrieve_interpreter.py

@@ -0,0 +1,57 @@
+# read the document and can retrieve the information
+import re
+from .interpreter import Interpreter
+from GeneralAgent.memory import LinkMemory
+
+
+class LinkRetrieveInterperter(Interpreter):
+    """
+    LinkRetrieveInterperter store and retrieve the information from the memory by link embed in the document. like I live in <<My Home>>.
+    LinkRetrieveInterperter handle input string like this:
+    ```read
+    path/to/file1.pdf
+    path/to/file2.pdf
+    ```
+    """
+    
+    input_match_pattern = '```read\n(.*?)\n```'
+
+    def __init__(self, python_interpreter=None, sparks_dict_name='sparks'):
+        self.python_intrepreter = python_interpreter
+        self.sparks_dict_name = sparks_dict_name
+        self.link_memory = LinkMemory()
+
+    async def prompt(self, messages) -> str:
+        if self.link_memory.is_empty():
+            return None
+        else:
+            access_prompt = f"""
+In Python, You can access the values of <<key>> in all documents through the dictionary {self.sparks_dict_name}, such as <<Hello world>>:
+```
+print({self.sparks_dict_name}['Hello world'])
+```
+"""
+            return await 'Background Information: \n' + self.link_memory.get_memory(messages) + access_prompt
+    
+    async def input_parse(self, string) -> (str, bool):
+        from GeneralAgent import skills
+        pattern = re.compile(self.input_match_pattern, re.DOTALL)
+        match = pattern.search(string)
+        assert match is not None
+        file_paths = match.group(1).strip().split('\n')
+        result = ''
+        async def output_callback(token):
+            nonlocal result
+            if token is not None:
+                result += token
+        for file_path in file_paths:
+            content = skills.read_file_content(file_path)
+            await self.link_memory.add_memory(content, output_callback=output_callback)
+        self._update_python_variables()
+        return string + '\n' + result, True
+    
+    def _update_python_variables(self):
+        if self.python_intrepreter is not None:
+            nodes = self.link_memory.concepts.values()
+            sparks_dict = dict(zip([node.key for node in nodes], [node.content for node in nodes]))
+            self.python_intrepreter.set_variable(self.sparks_dict_name, sparks_dict)

GeneralAgent/interpreter/plan_interpreter.py

@@ -0,0 +1,59 @@
+import re
+from collections import OrderedDict
+from .interpreter import Interpreter
+from GeneralAgent.memory import StackMemoryNode
+
+
+class PlanInterpreter(Interpreter):
+    """
+    PlanInterperter is used to parse the plan structure.
+    PlanInterpreter handle the input string like this:
+    ```runplan
+    section1
+        section1.1
+    ```
+    Note: this is only for StackAgent and StackMemory.
+    """
+    input_match_pattern = '```runplan(.*?)?\n(.*?)\n```'
+    
+    def __init__(self, memory, max_plan_depth=4) -> None:
+        self.memory = memory
+        self.max_plan_depth = max_plan_depth
+
+    async def prompt(self, messages) -> str:
+        return ''
+    
+    async def input_parse(self, string) -> (str, bool):
+        pattern = re.compile(self.input_match_pattern, re.DOTALL)
+        match = pattern.search(string)
+        assert match is not None
+        prefix = match.group(1).strip()
+        structure_data = match.group(2).strip()
+        plan_dict = self._structure_plan(structure_data)
+        current_node = self.memory.current_node
+        self._add_plans_for_node(current_node, plan_dict, prefix)
+        return string, False
+    
+    def _add_plans_for_node(self, node:StackMemoryNode, plan_dict, prefix):
+        if self.memory.get_node_level(node) >= self.max_plan_depth:
+            return
+        for k, v in plan_dict.items():
+            new_node = StackMemoryNode(role='system', action='plan', content=k.strip(), prefix=prefix)
+            self.memory.add_node_in(node, new_node)
+            if len(v) > 0:
+                self._add_plans_for_node(new_node, v, prefix)
+
+    @classmethod
+    def _structure_plan(cls, data):
+        structured_data = OrderedDict()
+        current_section = [structured_data]
+        for line in data.split('\n'):
+            if not line.strip():
+                continue
+            depth = line.count('    ')
+            section = line.strip()
+            while depth < len(current_section) - 1:
+                current_section.pop()
+            current_section[-1][section] = OrderedDict()
+            current_section.append(current_section[-1][section])
+        return structured_data

GeneralAgent/interpreter/python_interpreter.py

@@ -0,0 +1,326 @@
+import re, io, os, sys
+import pickle
+import logging
+from jinja2 import Template
+from .interpreter import Interpreter
+from GeneralAgent.utils import confirm_to_run
+from GeneralAgent import skills
+import asyncio
+
+default_import_code = """
+import os, sys, math
+from GeneralAgent import skills
+"""
+
+# default_libs = ' '.join(["requests", "tinydb", "openai", "jinja2", "numpy", "bs4", "playwright", "retrying", "pymupdf", "python-pptx", "python-docx", "yfinance"])
+# default_libs = skills.get_current_env_python_libs()
+default_libs = ''
+
+# from GeneralAgent.tools import Tools
+
+class SyncPythonInterpreter(Interpreter):
+    """
+    Sync Python Interpreter: run python code in the interpreter. Not same namespace with the agent & Can Only run synchronous code
+    """
+
+    output_match_start_pattern = '```python\n'
+    output_match_pattern = '```python\n(.*?)\n```'
+
+    python_prompt_template = """
+# Run python
+* format is : ```python\\nthe_code\\n```
+* the code will be executed
+* python version is {{python_version}}
+* only write synchronous code
+* The output display should be limited in length and should be truncated when displaying characters whose length is unknown. for example: print(a[:100])
+* * Pickleable objects can be shared between different codes and variables
+* Available libraries: {{python_libs}}
+* The following functions can be used in code (already implemented and imported for you):
+```
+{{python_funcs}}
+```
+"""
+
+    function_tools = []
+
+    def __init__(self, 
+                 serialize_path:str=None, 
+                 libs: str=default_libs, 
+                 import_code:str=None,
+                 prompt_append='',
+                 stop_wrong_count = 2
+                 ):
+        """
+        Args:
+            serialize_path (str): path to save the global variables, default None, which means not save, like './serialized.bin'
+            libs ([str], optional): libraries can be to used. Defaults to skills.get_current_env_python_libs()
+            import_code (str, optional): code to import. The tools used should be imported. Defaults to default_import_code.
+            prompt_append: append to the prompt, custom prompt can be added here
+            stop_wrong_count: stop running when the code is wrong for stop_wrong_count times
+        """
+        from GeneralAgent import skills
+        self.globals = {}  # global variables shared by all code
+        self.python_libs = libs
+        self.import_code = import_code or default_import_code
+        self.serialize_path = serialize_path
+        self.prompt_append = prompt_append
+        # self.tools = tools or Tools([])
+        self.globals = self.load()
+        # count the number of times the code is wrong, and stop running when it reaches the threshold
+        self.run_wrong_count = 0
+        self.stop_wrong_count = stop_wrong_count
+
+    def load(self):
+        if self.serialize_path is None:
+            return {}
+        if os.path.exists(self.serialize_path):
+            with open(self.serialize_path, 'rb') as f:
+                data = pickle.loads(f.read())
+                return data['globals']
+        return {}
+
+    async def prompt(self, messages) -> str:
+        from GeneralAgent import skills
+        funtions = '\n\n'.join([skills.get_function_signature(x) for x in self.function_tools])
+        variables = {
+            'python_libs': self.python_libs,
+            'python_funcs': funtions,
+            'python_version': skills.get_python_version()
+        }
+        return Template(self.python_prompt_template).render(**variables) + self.prompt_append
+
+    def save(self):
+        if self.serialize_path is None:
+            return
+        self._remove_unpickleable()
+        # save
+        with open(self.serialize_path, 'wb') as f:
+            data = {'globals': self.globals}
+            f.write(pickle.dumps(data))
+
+    def _remove_unpickleable(self):
+        if '__builtins__' in self.globals:
+            self.globals.__delitem__('__builtins__')
+        keys = list(self.globals.keys())
+        for key in keys:
+            try:
+                pickle.dumps(self.globals[key])
+            except Exception as e:
+                self.globals.__delitem__(key)
+
+    async def output_parse(self, string) -> (str, bool):
+        sys_out = ''
+        pattern = re.compile(self.output_match_pattern, re.DOTALL)
+        match = pattern.search(string)
+        assert match is not None
+        if confirm_to_run():
+            sys_out, stop = await self.run_code(match.group(1))
+            result = 'python runs result:\n' + sys_out.strip()
+            return result, stop
+        else:
+            return '', False
+
+    async def run_code(self, code):
+        stop = False
+        code = self.add_print(code)
+        code = self.import_code + '\n' + code
+        globals_backup = self.load()
+        logging.debug(code)
+        sys_stdout = ''
+        output = io.StringIO()
+        sys.stdout = output
+        success = False
+        try:
+            exec(code, self.globals)
+            success = True
+            self.run_wrong_count = 0
+        except Exception as e:
+            import traceback
+            sys_stdout += traceback.format_exc()
+            self.globals = globals_backup
+            self.run_wrong_count += 1
+            if self.run_wrong_count >= self.stop_wrong_count:
+                stop = True
+        finally:
+            sys_stdout += output.getvalue()
+            sys.stdout = sys.__stdout__
+        if success:
+            self.save()
+        sys_stdout = sys_stdout.strip()
+        if sys_stdout == '':
+            sys_stdout = 'run successfully'
+        return sys_stdout, stop
+
+    def get_variable(self, name):
+        if name in self.globals:
+            return self.globals[name]
+        else:
+            logging.warning(f"Variable {name} not found")
+            return None
+
+    def set_variable(self, name, value):
+        self.globals[name] = value
+        self.save()
+
+    @classmethod
+    def add_print_old(cls, code_string):
+        pattern = r'^(\s*)(\w+)(\s*)$'
+        lines = code_string.split('\n')
+        for i, line in enumerate(lines):
+            keywords = ['False' 'None', 'True', 'and', 'as', 'assert', 'async', 'await', 'break', 'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'finally', 'for', 'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'nonlocal', 'not', 'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield']
+            if line.strip() in keywords:
+                continue
+            match = re.match(pattern, line)
+            if match:
+                lines[i] = f'{match.group(1)}print({match.group(2)}){match.group(3)}'
+        return '\n'.join(lines)
+    
+    @classmethod
+    def add_print(cls, code_string):
+        from GeneralAgent import skills
+        code = code_string.strip()
+        lines = code.split('\n')
+        if len(lines) > 0:
+            last_line = lines[-1]
+            if skills.python_line_is_variable_expression(last_line):
+                last_line = f'print({last_line})'
+                lines[-1] = last_line
+        return '\n'.join(lines)
+
+def _remove_unpickleable(namespace):
+    import pickle
+    if '__builtins__' in namespace:
+        namespace.__delitem__('__builtins__')
+    keys = list(namespace.keys())
+    for key in keys:
+        try:
+            pickle.dumps(namespace[key])
+        except Exception as e:
+            namespace.__delitem__(key)
+    return namespace
+
+
+def code_wrap(code, namespace):
+    lines = code.split('\n')
+    code = '\n    '.join(lines)
+    variables = '\n    '.join([f'{name} = globals()[\'{name}\']' for name, value in namespace.items()])
+    content = f"""
+import asyncio
+
+def _remove_unpickleable(namespace):
+    import pickle
+    if '__builtins__' in namespace:
+        namespace.__delitem__('__builtins__')
+    keys = list(namespace.keys())
+    for key in keys:
+        try:
+            pickle.dumps(namespace[key])
+        except Exception as e:
+            namespace.__delitem__(key)
+    for name in ['__name', '__value', '__namespace']:
+        if name in namespace:
+            namespace.__delitem__(name)
+    return namespace
+
+__namespace = None
+
+async def __main():
+    {variables}
+    {code}
+    global __namespace
+    __namespace = _remove_unpickleable(locals().copy())
+"""
+    # print('----------<code>--------')
+    # print(content)
+    # print('----------</code>--------')
+    return content
+
+
+class AsyncPythonInterpreter(SyncPythonInterpreter):
+    """
+    Sync Python Interpreter: run python code in the interpreter. Same namespace with the agent & Can run async code
+    """
+
+    python_prompt_template = """
+# Run python
+- format is : ```python\\nthe_code\\n```
+- the code will be executed
+- python version is {{python_version}}
+- Pickleable objects can be shared between different codes and variables
+- The output display should be limited in length and should be truncated when displaying characters whose length is unknown. for example: print(a[:100])
+- Available libraries: {{python_libs}}
+Complete the entire process in one code instead of writing multiple codes to run step by step. For example, the following code is allowed:
+```python
+# step 1
+a = fun1(xx)
+# step 2
+c = fun2(a)
+# step 3
+d = fun3(c)
+...
+```
+- The following functions can be used in code (already implemented and imported for you):
+```
+{{python_funcs}}
+```
+"""
+
+    async def run_code(self, code):
+        stop = False
+        code = self.add_print(code)
+        code = code_wrap(code, self.globals)
+        code = self.import_code + '\n' + code
+        # print('hello')
+        # print(code)
+        # print(self.function_tools)
+        globals_backup = self.load()
+        logging.debug(code)
+        sys_stdout = ''
+        output = io.StringIO()
+        sys.stdout = output
+        success = False
+        try:
+            # exec(code, self.globals)
+            # run async python code
+            local_vars = self.globals
+            # register functions
+            for fun in self.function_tools:
+                local_vars[fun.__name__] = fun
+            # print(local_vars)
+            exec(code, local_vars, local_vars)
+            main_function = local_vars['__main']
+            await asyncio.create_task(main_function())
+            local_vars = _remove_unpickleable(local_vars)
+            local_vars = local_vars['__namespace']
+            # remove functions
+            for fun in self.function_tools:
+                if fun.__name__ in local_vars:
+                    local_vars.__delitem__(fun.__name__)
+            self.globals = local_vars
+
+            success = True
+            self.run_wrong_count = 0
+        except Exception as e:
+            import traceback
+            sys_stdout += traceback.format_exc()
+            self.globals = globals_backup
+            logging.exception((e))
+            self.run_wrong_count += 1
+            if self.run_wrong_count >= self.stop_wrong_count:
+                stop = True
+        finally:
+            sys_stdout += output.getvalue()
+            sys.stdout = sys.__stdout__
+        if success:
+            self.save()
+        sys_stdout = sys_stdout.strip()
+        if sys_stdout == '':
+            sys_stdout = 'run successfully'
+        return sys_stdout, stop
+    
+
+# class PythonInterpreter(AsyncPythonInterpreter):
+#     """
+#     Sync Python Interpreter: run python code in the interpreter. Same namespace with the agent & Can run async code
+#     """
+#     pass

GeneralAgent/interpreter/role_interpreter.py

@@ -0,0 +1,40 @@
+import os
+import datetime
+import platform
+from jinja2 import Template
+from .interpreter import Interpreter
+
+
+class RoleInterpreter(Interpreter):
+    """
+    RoleInterpreter, a interpreter that can change the role of the agent.
+    Note: This should be the first interpreter in the agent.
+    """
+    
+    system_prompt_template = \
+"""
+Now: {{now}}
+You are GeneralAgent, a agent on the {{os_version}} computer to help the user solve the problem.
+Remember, you can control the computer and access the internet.
+Reponse message in markdown format to user. for example file a.txt, you should reponse [title](a.txt)
+You can use the following skills (start with # ) to help you solve the problem directly without explain, without ask for permission.
+"""
+
+    def __init__(self, system_prompt=None) -> None:
+        from GeneralAgent import skills
+        self.os_version = skills.get_os_version()
+        self.system_prompt = system_prompt
+
+    async def prompt(self, messages) -> str:
+        if self.system_prompt is not None:
+            return self.system_prompt
+        if os.environ.get('LLM_CACHE', 'no') in ['yes', 'y', 'YES']:
+            now = '2023-09-27 00:00:00'
+        else:    
+            now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        data = {
+            'now': now,
+            'os_version': self.os_version
+        }
+        the_prompt = Template(self.system_prompt_template).render(**data)
+        return the_prompt

GeneralAgent/interpreter/shell_interpreter.py

@@ -0,0 +1,44 @@
+import re
+from .interpreter import Interpreter
+from GeneralAgent.utils import confirm_to_run
+
+shell_prompt = """
+# Run shell
+* format is : ```shell\\nthe_command\\n```
+* the command will be executed
+"""
+
+class ShellInterpreter(Interpreter):
+
+    output_match_start_pattern = '```shell\n'
+    output_match_pattern = '```shell\n(.*?)\n```'
+    
+    def __init__(self, workspace='./') -> None:
+        self.workspace = workspace
+
+    async def prompt(self, messages) -> str:
+        return shell_prompt
+
+    async def output_parse(self, string) -> (str, bool):
+        pattern = re.compile(self.output_match_pattern, re.DOTALL)
+        match = pattern.search(string)
+        assert match is not None
+        if confirm_to_run():
+            output = self._run_bash(match.group(1))
+            return output.strip(), False
+        else:
+            return '', False
+
+    def _run_bash(self, content):
+        sys_out = ''
+        import subprocess
+        if 'python ' in content:
+            content = content.replace('python ', 'python3 ')
+        try:
+            p = subprocess.Popen(content, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        except:
+            pass
+        finally:
+            sys_out, err = p.communicate()
+            sys_out = sys_out.decode('utf-8')
+        return sys_out

GeneralAgent/interpreter/ui_interpreter.py

@@ -0,0 +1,51 @@
+import re
+from .interpreter import Interpreter
+import asyncio
+
+class UIInterpreter(Interpreter):
+    
+    output_match_start_pattern = '```tsx\n'
+    output_match_pattern = '```tsx\n(.*?)\n```'
+
+    ui_prompt = """
+# Send UI to user
+Use the following tsx architecture to write a React component. The component will be compiled into a UI and sent to the user. The user's input can be sent to you through the save_data function.
+```tsx
+const React = (window as any).React;
+const antd = (window as any).antd;
+
+const [Form, Input, Button] = [antd.Form, antd.Input, antd.Button];
+
+const LibTemplate = ({save_data}: {save_data: (data:any)=>void}) => {
+  // use save_data to save the data
+}
+
+export default LibTemplate;
+```
+"""
+
+    def __init__(self, send_ui, output_callback, workspace=None) -> None:
+        """
+        :param send_ui: the async function to send ui to user
+        :param workspace: workspace for the interpreter
+        """
+        self.send_ui = send_ui
+        self.output_callback = output_callback
+        self.workspace = workspace
+
+    async def prompt(self, messages) -> str:
+        return self.ui_prompt
+
+    async def output_parse(self, string) -> (str, bool):
+        from GeneralAgent import skills
+        pattern = re.compile(self.output_match_pattern, re.DOTALL)
+        match = pattern.search(string)
+        assert match is not None
+        code = match.group(1)
+        lib_name, js_path = skills.parse_tsx_to_ui(code, save_dir=self.workspace)
+        # Terminate the output callback
+        await self.output_callback(None)
+        # Send UI to user
+        await self.send_ui(lib_name, js_path)
+        print('Send UI to user successfuly.')
+        return '', True

GeneralAgent/memory/__init__.py

@@ -0,0 +1,4 @@
+# import 
+from .normal_memory import NormalMemory
+from .stack_memory import StackMemory, StackMemoryNode
+from .link_memory import LinkMemory, LinkMemoryNode

GeneralAgent/memory/link_memory.py

@@ -0,0 +1,125 @@
+from dataclasses import dataclass
+from typing import List
+from tinydb import TinyDB, Query
+import asyncio
+
+@dataclass
+class LinkMemoryNode:
+    key: str
+    content: str
+    childrens: List[str] = None
+    parents: List[str] = None
+
+    def __post_init__(self):
+        self.childrens = self.childrens if self.childrens else []
+        self.parents = self.parents if self.parents else []
+
+    def __str__(self):
+        return f'<<{self.key}>>\n{self.content}'
+    
+    def __repr__(self):
+        return str(self)
+
+
+async def summarize_and_segment(text, output_callback=None):
+    from GeneralAgent import skills
+    summary = await skills.summarize_text(text)
+    if output_callback is not None:
+        await output_callback(f'Summary: {summary}\n')
+    segments = await skills.segment_text(text)
+    if output_callback is not None:
+        for key in segments:
+            await output_callback(f'<<{key}>>\n')
+    return summary, segments
+
+
+class LinkMemory():
+    def __init__(self, serialize_path='./link_memory.json', short_memory_limit=2000) -> None:
+        self.serialize_path = serialize_path
+        self.short_memory_limit = short_memory_limit
+        self.db = TinyDB(serialize_path)
+        nodes = [LinkMemoryNode(**x) for x in self.db.all()]
+        self.concepts = dict(zip([node.key for node in nodes], nodes))
+        self.short_memory = ''
+        self._load_short_memory()
+
+    def is_empty(self):
+        return len(self.concepts) == 0
+
+    async def add_memory(self, content, output_callback=None):
+        from GeneralAgent import skills
+        # await self._oncurrent_summarize_content(content, output_callback)
+        await self._summarize_content(content, output_callback)
+        while skills.string_token_count(self.short_memory) > self.short_memory_limit:
+            content = self.short_memory
+            self.short_memory = ''
+            # await self._oncurrent_summarize_content(content, output_callback)
+            await self._summarize_content(content, output_callback)
+    
+    async def get_memory(self, messages=None):
+        from GeneralAgent import skills
+        if len(self.concepts) == 0:
+            return ''
+        if messages is None:
+            return self.short_memory
+        else:
+            messages = skills.cut_messages(messages, 1000)
+            xx = self.short_memory.split('\n')
+            background = '\n'.join([f'#{line} {xx[line]}' for line in range(len(xx))])
+            task = '\n'.join([f'{x["role"]}: {x["content"]}' for x in messages])
+            info = await skills.extract_info(background, task)
+            line_numbers, keys = skills.parse_extract_info(info)
+            result = []
+            for line_number in line_numbers:
+                if line_number < len(xx) and line_number >= 0:
+                    result.append(xx[line_number])
+            for key in keys:
+                if key in self.concepts:
+                    result.append(f'{key}\n{self.concepts[key]}\n')
+            return '\n'.join(result)
+
+    def _load_short_memory(self):
+        short_memorys = self.db.table('short_memory').all()
+        self.short_memory = '' if len(short_memorys) == 0 else short_memorys[0]['content']
+
+    def _save_short_memory(self):
+        self.db.table('short_memory').truncate()
+        self.db.table('short_memory').insert({'content': self.short_memory})
+    
+    async def _oncurrent_summarize_content(self, input):
+        from GeneralAgent import skills
+        inputs = skills.split_text(input, max_token=3000)
+        print('splited count: ', len(inputs))
+        coroutines = [summarize_and_segment(x) for x in inputs]
+        results = await asyncio.gather(*coroutines)
+        for summary, nodes in results:
+            new_nodes = {}
+            for key in nodes:
+                new_key = self._add_node(key, nodes[key])
+                new_nodes[new_key] = nodes[key]
+            self.short_memory += '\n' + summary + ' Detail in ' + ', '.join([f'<<{key}>>' for key in new_nodes])
+        self.short_memory = self.short_memory.strip()
+        self._save_short_memory()
+
+    async def _summarize_content(self, input, output_callback=None):
+        from GeneralAgent import skills
+        inputs = skills.split_text(input, max_token=3000)
+        for text in inputs:
+            summary, nodes = await summarize_and_segment(text, output_callback)
+            new_nodes = {}
+            for key in nodes:
+                new_key = self._add_node(key, nodes[key])
+                new_nodes[new_key] = nodes[key]
+            self.short_memory += '\n' + summary + ' Detail in ' + ', '.join([f'<<{key}>>' for key in new_nodes])
+        self.short_memory = self.short_memory.strip()
+        self._save_short_memory()
+
+    def _add_node(self, key, value):
+        index = 0
+        new_key = key
+        while new_key in self.concepts:
+            index += 1
+            new_key = key + str(index)
+        self.concepts[new_key] = LinkMemoryNode(key=new_key, content=value)
+        self.db.upsert(self.concepts[key].__dict__, Query().key == new_key)
+        return new_key

GeneralAgent/memory/normal_memory.py

@@ -0,0 +1,42 @@
+# Memeory
+import json
+import os
+
+class NormalMemory:
+    def __init__(self, serialize_path='./memory.json'):
+        self.messages = []
+        self.serialize_path = serialize_path
+        if os.path.exists(serialize_path):
+            with open(serialize_path, 'r', encoding='utf-8') as f:
+                self.messages = json.load(f)
+
+    def save(self):
+        with open(self.serialize_path, 'w', encoding='utf-8') as f:
+            json.dump(self.messages, f)
+
+    def add_message(self, role, content):
+        assert role in ['user', 'assistant']
+        self.messages.append({'role': role, 'content': content})
+        self.save()
+
+    def append_message(self, role, content):
+        assert role in ['user', 'assistant']
+        if len(self.messages) > 0 and self.messages[-1]['role'] == role:
+            self.messages[-1]['content'] += '\n' + content
+        else:
+            self.messages.append({'role': role, 'content': content})
+        self.save()
+
+    def get_messages(self):
+        return self.messages
+    
+
+def test_NormalMemory():
+    serialize_path = './memory.json'
+    mem = NormalMemory(serialize_path=serialize_path)
+    mem.add_message('user', 'hello')
+    mem.add_message('assistant', 'hi')
+    mem = NormalMemory(serialize_path=serialize_path)
+    assert len(mem.get_messages()) == 2
+    mem.append_message('assistant', 'hi')
+    assert len(mem.get_messages()) == 2

GeneralAgent/memory/stack_memory.py

@@ -0,0 +1,223 @@
+# Memeory
+from dataclasses import dataclass
+from typing import List
+from tinydb import TinyDB, Query
+
+
+@dataclass
+class StackMemoryNode:
+    role: str
+    action: str
+    state: str = 'ready'
+    content: str = None
+    prefix: str = None
+    node_id: int = None
+    parent: int = None
+    childrens: List[int] = None
+
+    def __post_init__(self):
+        assert self.role in ['user', 'system', 'root'], self.role
+        assert self.action in ['input', 'answer', 'plan'], self.action
+        assert self.state in ['ready', 'success', 'fail'], self.state
+        self.childrens = self.childrens if self.childrens else []
+
+    def __str__(self):
+        return f'<{self.role}><{self.action}><{self.state}>: {self.content}'
+    
+    def __repr__(self):
+        return str(self)
+
+    def success_work(self):
+        self.state = 'success'
+
+    def fail_work(self):
+        self.state = 'fail'
+
+    def is_root(self):
+        return self.role == 'root'
+    
+    def get_level(self):
+        if self.is_root():
+            return 0
+        else:
+            return self.get_parent().get_level() + 1
+    
+    @classmethod
+    def new_root(cls):
+        return cls(node_id=0, role='root', action='input', state='success', content='root', parent=None, childrens=[])
+
+
+class StackMemory:
+    def __init__(self, serialize_path='./memory.json'):
+        self.db = TinyDB(serialize_path)
+        nodes = [StackMemoryNode(**node) for node in self.db.all()]
+        self.spark_nodes = dict(zip([node.node_id for node in nodes], nodes))
+        # add root node
+        if len(self.spark_nodes) == 0:
+            root_node = StackMemoryNode.new_root()
+            self.spark_nodes[root_node.node_id] = root_node
+            self.db.insert(root_node.__dict__)
+        # load current_node
+        current_nodes = self.db.table('current_node').all()
+        if len(current_nodes) > 0:
+            node_id = current_nodes[0]['id']
+            # print(node_id)
+            # print(self)
+            self.current_node = self.get_node(node_id)
+        else:
+            self.current_node = None
+
+    def set_current_node(self, current_node):
+        self.current_node = current_node
+        # save current node
+        self.db.table('current_node').truncate()
+        self.db.table('current_node').insert({'id': current_node.node_id})
+
+    def new_node_id(self):
+        return max(self.spark_nodes.keys()) + 1
+    
+    def node_count(self):
+        # ignore root node
+        return len(self.spark_nodes.keys()) - 1
+        
+    def is_all_children_success(self, node):
+        # check if all childrens of node are success
+        childrens = [self.get_node(node_id) for node_id in node.childrens]
+        return all([children.state == 'success' for children in childrens])
+
+    def add_node(self, node):
+        # put in root node
+        root_node = self.get_node(0)
+        node.node_id = self.new_node_id()
+        node.parent = root_node.node_id
+        root_node.childrens.append(node.node_id)
+        # save node
+        self.update_node(root_node)
+        self.db.insert(node.__dict__)
+        self.spark_nodes[node.node_id] = node
+
+    def delete_node(self, node):
+        # delete node and all its childrens
+        for children_id in node.childrens:
+            children = self.get_node(children_id)
+            self.delete_node(children)
+        parent = self.get_node_parent(node)
+        if parent:
+            parent.childrens.remove(node.node_id)
+            self.update_node(parent)
+        self.db.remove(Query().node_id == node.node_id)
+        del self.spark_nodes[node.node_id]
+
+    
+    def add_node_after(self, last_node, node):
+        # add node after last_node
+        node.node_id = self.new_node_id()
+        node.parent = last_node.parent
+        parent = self.get_node_parent(node)
+        if parent:
+            parent.childrens.insert(parent.childrens.index(last_node.node_id)+1, node.node_id)
+            self.update_node(parent)
+        # move childrens of last_node to node
+        node.childrens = last_node.childrens
+        last_node.childrens = []
+        self.update_node(last_node)
+        for children_id in node.childrens:
+            children = self.get_node(children_id)
+            children.parent = node.node_id
+            self.update_node(children)
+        # save node
+        self.db.insert(node.__dict__)
+        self.spark_nodes[node.node_id] = node
+        return node
+    
+    def add_node_in(self, parent_node, node, put_first=False):
+        # add node in parent_node
+        node.node_id = self.new_node_id()
+        node.parent = parent_node.node_id
+        if put_first:
+            parent_node.childrens.insert(0, node.node_id)
+        else:
+            parent_node.childrens.append(node.node_id)
+        self.update_node(parent_node)
+        # save node
+        self.db.insert(node.__dict__)
+        self.spark_nodes[node.node_id] = node
+        return node
+    
+    def get_node(self, node_id):
+        return self.spark_nodes[node_id]
+    
+    def get_node_level(self, node:StackMemoryNode):
+        if node.is_root():
+            return 0
+        else:
+            return self.get_node_level(self.get_node_parent(node)) + 1
+    
+    def get_node_parent(self, node):
+        if node.parent is None:
+            return None
+        else:
+            return self.get_node(node.parent)
+    
+    def update_node(self, node):
+        self.db.update(node.__dict__, Query().node_id == node.node_id)
+    
+    def get_related_nodes_for_node(self, node):
+        # ancestors + left_brothers + self
+        parent = self.get_node_parent(node)
+        brothers = [self.get_node(node_id) for node_id in parent.childrens]
+        left_brothers = [('brother', x) for x in brothers[:brothers.index(node)]]
+        ancestors = self.get_related_nodes_for_node(parent) if not parent.is_root() else []
+        return ancestors + left_brothers + [('direct', node)]
+    
+    def get_related_messages_for_node(self, node: StackMemoryNode):
+        def _get_message(node, position='direct'):
+            content = node.content if node.prefix is None else node.prefix + ' ' + node.content
+            if position == 'brother' and node.action == 'plan' and len(node.childrens) > 0:
+                content = node.content + ' [detail ...]'
+            return {'role': node.role, 'content': content}
+        nodes_with_position = self.get_related_nodes_for_node(node)
+        messages = [_get_message(node, position) for position, node in nodes_with_position]
+        # if node.action == 'plan':
+        #     messages[-1]['content'] = 'Improve the details of this topic:: ' + messages[-1]['content']
+        return messages
+    
+    def get_all_description_of_node(self, node, intend_char='    ', depth=0):
+        lines = []
+        description = intend_char * depth + str(node)
+        if not node.is_root():
+            lines += [description]
+        for children_id in node.childrens:
+            children = self.get_node(children_id)
+            lines += self.get_all_description_of_node(children, intend_char, depth+1)
+        return lines
+    
+    def __str__(self) -> str:
+        lines = self.get_all_description_of_node(self.get_node(0), depth=-1)
+        return '\n'.join(lines)
+    
+    def success_node(self, node):
+        node.success_work()
+        self.update_node(node)
+    
+    def _get_todo_node(self, node=None):
+        # get the first ready node in the tree of node
+        if node is None:
+            node = self.get_node(0)
+        for node_id in node.childrens:
+            child = self._get_todo_node(self.get_node(node_id))
+            if child is not None:
+                return child
+        if node.is_root():
+            return None
+        if node.state in ['ready']:
+            return node
+        return None
+    
+    def get_todo_node(self):
+        todo_node = self._get_todo_node()
+        # if all childrens of todo_node are success, success todo_node
+        if todo_node is not None and len(todo_node.childrens) > 0 and self.is_all_children_success(todo_node):
+            self.success_node(todo_node)
+            return self.get_todo_node()
+        return todo_node

GeneralAgent/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+python_files = *.py

GeneralAgent/requirements.txt

@@ -0,0 +1,30 @@
+fastapi==0.104.1
+requests==2.31.0
+tinydb==4.8.0
+openai==0.27.6
+jinja2==3.1.2
+numpy==1.24.4
+bs4==0.0.1
+playwright==1.39.0
+retrying==1.3.4
+pymupdf==1.23.6
+python-pptx==0.6.23
+python-docx==1.1.0
+yfinance==0.2.31
+beautifulsoup4==4.12.2
+python-dotenv==1.0.0
+uvicorn==0.24.0.post1
+tiktoken==0.5.1
+httpx==0.25.1
+pulsar-client==3.3.0
+pymongo==4.6.0
+websocket-client-py3==0.15.0
+websockets==12.0
+pypdf==3.17.1
+replicate==0.18.1
+edge-tts==6.1.9
+pydub==0.25.1
+chromadb==0.4.17
+python-multipart==0.0.6
+pytest==7.4.3
+pytest-asyncio==0.21.1

GeneralAgent/skills/__init__.py

@@ -0,0 +1,121 @@
+# 单列
+import os
+
+def load_functions_with_path(python_code_path):
+    try:
+        import importlib.util
+        import inspect
+
+        # 指定要加载的文件路径和文件名
+        module_name = 'skills'
+        module_file = python_code_path
+
+        # 使用importlib加载文件
+        spec = importlib.util.spec_from_file_location(module_name, module_file)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+
+        # 获取文件中的所有函数
+        functions = inspect.getmembers(module, inspect.isfunction)
+
+        # 过滤functions中以下划线开头的函数、test_开头的函数
+        functions = filter(lambda f: not f[0].startswith('_'), functions)
+        # functions = filter(lambda f: not f[0].startswith('test_'), functions)
+
+        return [f[1] for f in functions]
+    except Exception as e:
+        # 代码可能有错误，加载不起来
+        import logging
+        logging.exception(e)
+        return []
+
+class Skills:
+    __instance = None
+
+    @classmethod
+    def __getInstance(cls):
+        return cls.__instance
+
+    @classmethod
+    def _instance(cls, *args, **kwargs):
+        if not Skills.__instance:
+            Skills.__instance = Skills(*args, **kwargs)
+        return Skills.__instance
+    
+    def __setattr__(self, name, value):
+        if name.startswith('_'):
+            object.__setattr__(self, name, value)
+        else:
+            self._local_funs[name] = value
+
+    def __getattr__(self, name):
+        if name.startswith('_'):
+            return object.__getattr__(self, name)
+        else:
+            return self._get_func(name)
+        
+    def _get_func(self, name):
+        fun = self._local_funs.get(name, None)
+        if fun is not None:
+            return fun
+        fun = self._remote_funs.get(name, None)
+        if fun is not None:
+            return fun
+        self._load_remote_funs()
+        fun = self._remote_funs.get(name, None)
+        if fun is not None:
+            return fun
+        print('Function {} not found'.format(name))
+        return None
+    
+    def __init__(self):
+        self._local_funs = {}
+        self._remote_funs = {}
+        self._load_local_funs()
+        self._load_remote_funs()
+
+    def _load_funcs(self, the_dir):
+        total_funs = []
+        for file in os.listdir(the_dir):
+            # 如果file是文件夹
+            if os.path.isdir(os.path.join(the_dir, file)):
+                total_funs += self._load_funcs(os.path.join(the_dir, file))
+            else:
+                # 如果file是文件
+                if file.endswith('.py') and (not file.startswith('__init__') and not file.startswith('_') and not file == 'main.py'):
+                    funcs = load_functions_with_path(os.path.join(the_dir, file))
+                    total_funs += funcs
+        return total_funs
+
+    def _load_local_funs(self):
+        self._local_funs = {}
+        funcs = self._load_funcs(os.path.dirname(__file__))
+        for fun in funcs:
+            self._local_funs[fun.__name__] = fun
+
+    def _load_remote_funs(self):
+        from GeneralAgent.utils import get_functions_dir
+        self._remote_funs = {}
+        funcs = self._load_funcs(get_functions_dir())
+        for fun in funcs:
+            self._remote_funs[fun.__name__] = fun
+
+    def _search_functions(self, task_description, return_list=False):
+        """
+        Search functions that may help to solve the task.
+        """
+        from .llm_inference import search_similar_texts
+        signatures = self._all_function_signatures()
+        results = search_similar_texts(task_description, signatures, top_k=5)
+        if return_list:
+            return results
+        else:
+            return '\n'.join(results)
+    
+    def _all_function_signatures(self):
+        from .python_envs import get_function_signature
+        locals = [get_function_signature(fun, 'skills') for fun in self._local_funs.values() if not fun.__name__.startswith('test_')]
+        remotes = [get_function_signature(fun, 'skills') for fun in self._remote_funs.values() if not fun.__name__.startswith('test_')]
+        return locals + remotes
+
+skills = Skills._instance()

GeneralAgent/skills/agent_builder_2.py

@@ -0,0 +1,245 @@
+def _llm_write_ui_lib(lib_name:str, task:str) -> str:
+    """
+    Write a UI library for a given task description.
+    @param lib_name: The name of the UI library.
+    @param task: The task description.
+    @return: The UI library code.
+    """
+    from GeneralAgent import skills
+    prompt_template = """
+You are a React and Typescript expert.
+
+# Task
+Create a React function component named LibTemplate in tsx language. 
+The component should have the following functionality:
+{{task}}
+Note: 
+1. The component only save data to backend, no need to display data, or result of task.
+2. When uploaded file, should show the file path
+3. No need to ask user set result file name, backend server will use a unique name to save the file.
+
+# Import
+Use the following import syntax:
+```
+const React = (window as any).React;
+const antd = (window as any).antd;
+```
+No other import methods are allowed.
+
+# DEMO
+
+```tsx
+const React = (window as any).React;
+const antd = (window as any).antd;
+
+interface Props {
+  save_data: (user_data:any)=>void,
+  FileUploadConponent: (props: {onUploadSuccess: (file_path: string) => void, title?: string}) => React.ReactElement
+}
+
+// user save_data to save the user_data
+// user_data will be processed into a string through save_data using JSON.stringify({'data': user_data}) and sent to the backend
+// use FileUploadConponent to upload file (<props.FileUploadConponent onUploadSuccess={handleUploadSuccess} title=''/>) and add file_path to data before save
+
+const LibTemplate = (props: Props) => {
+
+  const handleCommit = () => {
+    #   props.save_data(all_data_should_save)
+  };
+
+  return (<>{xxx}</>);
+};
+
+export default LibTemplate;
+```
+
+Please reponse the component code which finish the task without any explaination.
+"""
+
+    from jinja2 import Template
+    prompt = Template(prompt_template).render(task=task)
+    messages = [{'role': 'system', 'content': prompt}]
+    response = skills.llm_inference(messages, model_type="normal", stream=True)
+    result = ''
+    for token in response:
+        result += token
+    result = result.replace('LibTemplate', lib_name)
+    return result
+
+
+def create_application_ui(task: str, component_name: str = None) -> (str, str):
+    """
+    Convert a given task description into UI components. 
+    In the code, user_data will be processed into a string through save_data using JSON.stringify({'data': user_data}) and sent to the backend
+    @param task: Task description, should include all details related to creating the UI
+    @param component_name: The name of the UI library.
+    @return: The name of the UI library,  the path of the UI library, the code of the UI library. None if failed. 
+    Example:
+        create_application_ui('A task description with all the necessary details')
+    """
+    import os
+    import uuid
+    from GeneralAgent import skills
+    lib_name = component_name
+    if lib_name is None:
+        lib_name = 'Lib' + str(uuid.uuid1())[:4]
+    target_dir = os.path.join(skills.get_code_dir(), lib_name)
+    content = _llm_write_ui_lib(lib_name, task)
+    code = skills.extract_tsx_code(content)
+    success = skills.compile_tsx(lib_name, code, target_dir)
+    if success:
+        js_path = os.path.join(lib_name, 'index.js')
+        print(f'UI library created successfully.\n js_component_name: {lib_name}\n js_path: {js_path}\n code: \n```tsx\n{code}\n```')
+        return lib_name, js_path, code
+    else:
+        return None
+    
+
+def update_application_meta_2(
+        application_id:str=None,
+        type:str=None,
+        application_name:str=None,
+        description:str=None,
+        js_component_name:str=None,
+        js_path:str=None,
+        agent_can_upload_file=None,
+        ) -> None:
+    """
+    Update application meta data. When type is application, you should provide js_component_name and js_path. When type is agent, you should ignore js_component_name and js_path.
+    @param application_id: application id, You should name it, example: translat_text, ai_draw
+    @param type: application type, one of ['application', 'agent']. application is a normal application with ui, agent is a agent application with chat interface.
+    @param application_name: application name
+    @param description: application description
+    @param js_component_name: js component name
+    @param js_path: js file path
+    @param agent_can_upload_file: agent can upload file or not, default is False, if True, agent can upload file to the application
+    @return: None
+    """
+    import os, json
+    from GeneralAgent import skills
+    bot_json_path = os.path.join(skills.get_code_dir(), 'bot.json')
+    if os.path.exists(bot_json_path):
+        with open(bot_json_path, 'r', encoding='utf-8') as f:
+            app_json = json.loads(f.read())
+    else:
+        app_json = {}
+    if application_id is not None:
+        from GeneralAgent import skills
+        bots = skills.load_applications()
+        if application_id in [x['id'] for x in bots]:
+            print(f'application_id ({application_id}) exists. ignore If you are just edit the exist application, or you should change the application_id')
+        app_json['id'] = application_id
+    if type is not None:
+        app_json['type'] = type
+    if application_name is not None:
+        app_json['name'] = application_name
+    if description is not None:
+        app_json['description'] = description
+    app_json['upload_file'] = 'yes'
+    if js_component_name is not None:
+        app_json['js_name'] = js_component_name
+    if js_path is not None:
+        app_json['js_path'] = js_path
+    if agent_can_upload_file is not None:
+        app_json['upload_file'] = 'yes'
+    if os.path.exists(os.path.join(skills.get_code_dir(), 'icon.jpg')):
+        app_json['icon'] = 'icon.jpg'
+    else:
+        del app_json['icon']
+    with open(bot_json_path, 'w', encoding='utf-8') as f:
+        f.write(json.dumps(app_json, indent=4))
+
+
+def edit_application_code_2(task_description:str) -> str:
+    """
+    edit_application_code_2 is an Agent. You just tell it what will be done and vailable functions, it will generate a python function to complete the task. the code will be saved in main.py, which will be used to create a normal application or agent application.
+    @param task_description: task description, should be a string and include the detail of task, and what functions can be used. when building a normal application, task_desciption should have the detail of data format that ui save to backend, example: "Create a image creation application. Available functions:\n\nskills.image_generation(prompt) generate a image with prompt (in english), return a image url\n\nskills.translate_text(content, target_language), data format is {'data': {'prompt': 'xxxxx'}}". when building a agent application, ignore the detail of data format, example: "Create a agent. role prompt is : You are a image creator, transfer users's need to create a image.  Available functions:\n\n xxxx"
+    @return: python code for the task
+    """
+    import os
+    from GeneralAgent import skills
+    code_path = os.path.join(skills.get_code_dir(),  'main.py')
+    old_code = None
+    if os.path.exists(code_path):
+        with open(code_path, 'r', encoding='utf-8') as f:
+            old_code = f.read()
+    code = _generate_agent_code(task_description, default_code=old_code)
+    with open(code_path, 'w', encoding='utf-8') as f:
+        f.write(code)
+    return code
+
+
+def _generate_agent_code(task_description, default_code=None):
+    """Return the python code text that completes the task to build a chat bot, when default_code is not None, update default_code by task"""
+    from GeneralAgent import skills
+    python_version = skills.get_python_version()
+    requirements = skills.get_current_env_python_libs()
+    prompt = f"""
+You are a python expert, write a python function to complete user's task.
+The function in code will be used to create a chat bot, like slack, discord.
+
+# Function signature
+```
+async def main(messages, input, output_callback):
+    # messages is a list of dict, like [{{"role": "user", "content": "hello"}}, {{"role": "system", "content": "hi"}}]
+    # input is a string, user's input in agent application, or json string by save_data in UI in normal application.
+    # output_callback is a async function, output_callback(content: str) -> None. output_callback will send content to user. the content should be markdown format. file should be like [title](sandbox:file_path)
+```
+
+# Python Version: {python_version}
+
+# Python Libs installed
+{requirements}
+
+# CONSTRAINTS:
+- Do not import the lib that the function not use.
+- Import the lib in the function
+- In the code, Intermediate files are written directly to the current directory (./)
+- Give the function a name that describe the task
+- The docstring of the function should be as concise as possible without losing key information, only one line, and output in English
+
+# DEMO 1 : normal application, write user's input to a file and return
+```python
+async def main(messages, input, output_callback):
+    from GeneralAgent import skills
+    import json
+    data = json.loads(input)['data']
+    # file_path should be a unique name, because the file will not be deleted, and the application will run many times.
+    file_path = skills.unique_name() + '.json
+    with open(file_path, 'w', encoding='utf-8') as f:
+        f.write(json.dumps(data))
+    await output_callback(f'file saved: [user_data.json](sandbox:{{file_path}})')
+```
+
+# DEMO 2 : agent application, Agent with functions
+```python
+async def main(messages, input, output_callback):
+    from GeneralAgent.agent import Agent
+    role_prompt = \"\"\"
+You are a translation agent.
+You complete user requirements by writing python code to call the predefined functions.
+\"\"\"
+    functions = [
+        skills.translate_text
+    ]
+    agent = Agent.with_functions(functions)
+    agent.add_role_prompt(role_prompt)
+    await agent.run(input, output_callback=output_callback)
+```python
+
+# There are two function types:
+1. Application: like DEMO1, The application process is fixed and less flexible, but the function will be more stable
+2. Agent: like DEMO2, Agent is a chat bot that can use functions to complete user's task. The agent will automatic handle user's input and output
+You can choose one of them to complete the task.
+
+Please think step by step carefully, consider any possible situation, and write a complete code like DEMO
+Just reponse the python code, no any explain, no start with ```python, no end with ```, no any other text.
+"""
+
+    messages = [{"role": "system", "content": prompt}]
+    if default_code is not None:
+        messages += [{"role": "system", "content": "user's code: " + default_code}]
+    messages += [{"role": "system", "content": f"user's task: {task_description}"}]
+    code = skills.llm_inference(messages, model_type='smart')
+    code = skills.get_python_code(code)
+    return code

GeneralAgent/skills/agents.py

@@ -0,0 +1,105 @@
+
+async def get_function_builder_agent():
+    """
+    Get a function builder agent
+    """
+    from GeneralAgent.agent import Agent
+    from GeneralAgent import skills
+    from GeneralAgent.interpreter import RoleInterpreter, PythonInterpreter, FileInterpreter, ShellInterpreter
+    from GeneralAgent.utils import get_functions_dir
+    function_dir = get_functions_dir()
+    role_prompt = f"""
+You are an agent who writes python function and the test function of it, into files according to user needs.
+You can control your computer and access the Internet.
+
+# make a directory to store function file and test dataset
+```shell
+mkdir -p {function_dir}/function_folder_name
+```
+
+# copy files uploaded by users to the function file directory
+```shell
+cp yy.zz {function_dir}/function_folder_name/yy.zz
+```
+
+# When writing a function, you can first search for available functions. For Example
+```python
+result = search_functions('scrape web page')
+print(result)
+```
+
+# The function should be written in the folder {function_dir}/functoin_folder_name, and the file name should be the function name
+# The content of the file is the function and the test function of the function (starting with test_)
+# every created file should have a unique name, which can be generated by skills.unique_name()
+# Import code should be placed inside the function
+
+# DEMO
+```file 
+{function_dir}/image_generation/image_generation.py write 0 -1 <<EOF
+def image_generation(prompt) -> str:
+    \"\"\"
+    Generate an image according to the prompt and return the image path. For example, when the prompt is "apple" you will get an image of an apple. Note: The prompt should describe objective things in detail, not abstract concepts. For example, if you want to draw a picture of Chengdu, the prompt should be "Picture of Chengdu, with giant pandas playing in the bamboo forest, people eating hot pot around, and a Jinsha Sunbird totem next to it" instead of "Draw a picture of Chengdu" "painting"
+    @param prompt: The prompt should be detailed enough to describe the image. Tips can be in any type of language, but English is recommended.
+    \"\"\"
+    import replicate
+    from GeneralAgent import skills
+    output = replicate.run(
+        "stability-ai/sdxl:2f779eb9b23b34fe171f8eaa021b8261566f0d2c10cd2674063e7dbcd351509e",
+        input={{"prompt": prompt}}
+    )
+    image_url = output
+    if not skills.text_is_english(prompt):
+        prompt = skills.translate_text(prompt, 'english')
+    image_url = _replicate_image_generation(prompt)
+    image_path  = skills.try_download_file(image_url)
+    print(f'image created at {{image_path}}')
+    return image_path
+
+def test_image_generation():
+    import os
+    # load test file
+    file_path = os.path.join(os.path.dirname(__file__), 'yy.zz')
+    prompt = 'xxx'
+    image_path = image_generation(prompt)
+    assert os.path.exists(image_path)
+EOF
+```
+
+# The written functions can be accessed through GeneralAgent's skills library, such as:
+
+```python
+from GeneralAgent import skills
+result = skills.function_name()
+skills.test_function_name()
+```
+
+# Note:
+- Don't make up functions that don't exist
+
+# General process for write function
+* Fully communicate needs with users
+* search available functions (by search_functions in python, optional)
+* edit functions (by file operation)
+* test functions (by python)
+* ask for test files if needed, for example test data, test code, etc.
+"""
+    functoins = [
+        skills.search_functions,
+        # skills.scrape_dynamic_web
+    ]
+    workspace = './'
+    agent = Agent(workspace)
+    role_interpreter = RoleInterpreter(system_prompt=role_prompt)
+    python_interpreter = PythonInterpreter(serialize_path=f'{workspace}/code.bin')
+    python_interpreter.function_tools = functoins
+    
+    # when file operation(python file), reload functions
+    file_interpreter = FileInterpreter()
+    async def reload_funs():
+        skills._load_remote_funs()
+    
+    file_interpreter.outptu_parse_done_recall = reload_funs
+    agent.interpreters = [role_interpreter, python_interpreter, FileInterpreter(), ShellInterpreter()]
+    agent.model_type = 'smart'
+    agent.hide_output_parse = False
+    return agent

GeneralAgent/skills/ai_draw_prompt_gen.py

@@ -0,0 +1,11 @@
+
+def ai_draw_prompt_gen(command):
+    """The prompt word generator for AI painting inputs the user's needs and outputs a description word for a picture."""
+    from GeneralAgent import skills
+    system_prompt = "You are a prompt word engineer for AI painting. Your task is to describe the user's needs into descriptors for a picture, and require the content of the picture to meet the user's needs as much as possible. You can add some of your own creativity and understanding. Directly returns the description of the image without explanation or suffixes."
+    messages = [
+        {'role': 'system', 'content': system_prompt},
+        {"role": "user", "content": command}
+    ]
+    image_prompt = skills.llm_inference(messages)
+    return image_prompt

GeneralAgent/skills/application_builder.py

@@ -0,0 +1,598 @@
+
+CODE_DIR = './code'
+
+def get_code_dir():
+    global CODE_DIR
+    import os
+    if not os.path.exists(CODE_DIR):
+        os.makedirs(CODE_DIR)
+    return CODE_DIR
+
+def _set_code_dir(code_dir):
+    global CODE_DIR
+    CODE_DIR = code_dir
+
+
+def search_functions(task_description:str) -> str:
+    """
+    print function signatures that may help to solve the task, and return the function signatures
+    """
+    from GeneralAgent import skills
+    functions = skills._search_functions(task_description)
+    print(functions)
+    return functions
+    from jinja2 import Template
+#     # print(functions)
+#     prompt_template = """
+# # Functions
+# {{functions}}
+
+# # Task
+# {{task}}
+
+# Please return the function signatures that can solve the task.
+
+# """
+#     prompt = Template(prompt_template).render(task=task_description, functions=functions)
+#     functions = skills.llm_inference([{'role': 'system', 'content': 'You are a helpful assistant'}, {'role': 'user', 'content': prompt}])
+#     print(functions)
+#     return functions
+
+
+def edit_normal_function(function_name:str, task_description:str) -> None:
+    """
+    Edit normal function code by task_description
+    @param function_name: The name of the function to be generated.
+    @param task_description (str): A description of the task that the generated function should perform and what functions can be used.
+    @return: The generated Python function signature as a string.
+    """
+    return _edit_function(function_name, task_description, _generate_function_code)
+
+
+def edit_llm_function(function_name: str, task_description:str) -> str:
+    """
+    This function generates a Python function to perform a specific task using a large language model (LLM), such as translation, planning, answering general knowledge questions and so on.
+    @param function_name: The name of the function to be generated.
+    @param task_description (str): A description of the task that the generated function should perform.
+    @return: The generated Python function signature as a string.
+    """
+    return _edit_function(function_name, task_description, _generate_llm_task_function)
+
+
+def _edit_function(function_name: str, task_description:str, code_fun) -> str:
+    import os
+    from GeneralAgent import skills
+    from GeneralAgent.utils import get_functions_dir
+    file_path = os.path.join(get_functions_dir(), function_name + '.py')
+    code = None
+    if os.path.exists(file_path):
+        with open(file_path, 'r', encoding='utf-8') as f:
+            code = f.read()
+    task_description += f'\n# Function name: {function_name}'
+    code = code_fun(task_description, default_code=code)
+    with open(file_path, 'w', encoding='utf-8') as f:
+        f.write(code)
+    funcs, error = skills.load_functions_with_path(file_path)
+    funcs = [x for x in funcs if x.__name__ == function_name]
+    if len(funcs) <= 0:
+        print(error)
+        return error
+    else:
+        signature = skills.get_function_signature(funcs[0], 'skills')
+        skills._load_remote_funs()
+        print(signature)
+        return signature
+
+def delete_function(func_name:str) -> None:
+    """
+    Delete a function by name
+    """
+    import os
+    from GeneralAgent.utils import get_functions_dir
+    file_path = os.path.join(get_functions_dir(), func_name + '.py')
+    if os.path.exists(file_path):
+        os.remove(file_path)
+
+def create_application_icon(application_description:str) -> None:
+    """
+    Create a application icon by application description. The application description is application_description
+    """
+    import os
+    from GeneralAgent import skills
+    prompt = skills.ai_draw_prompt_gen("Create an application icon. The application's description is below: \n" + application_description)
+    image_url = skills.image_generation(prompt)
+    file_path = skills.try_download_file(image_url)
+    target_path = os.path.join(get_code_dir(), 'icon.jpg')
+    os.system(f"mv {file_path} {target_path}")
+    
+
+def edit_application_code(task_description:str) -> None:
+    """
+    edit_application_code is an Agent. You just tell it what will be done and vailable functions, it will generate a python function to complete the task.
+    Edit agent code by task_description. task description should be a string and include the detail of task, and what functions can be used. 
+    task_description example: "Create a image creation agent. Available functions:\n\nskills.image_generation(prompt) generate a image with prompt (in english), return a image url\n\nskills.translate_text(content, target_language)"
+    """
+    import os
+    code_path = os.path.join(get_code_dir(),  'main.py')
+    old_code = None
+    if os.path.exists(code_path):
+        with open(code_path, 'r', encoding='utf-8') as f:
+            old_code = f.read()
+    from GeneralAgent import skills
+    code = _generate_agent_code(task_description, default_code=old_code)
+    with open(code_path, 'w', encoding='utf-8') as f:
+        f.write(code)
+
+
+def delete_application():
+    """
+    Delete application code
+    """
+    import os
+    code_path = os.path.join(get_code_dir(),  'main.py')
+    if os.path.exists(code_path):
+        os.remove(code_path)
+
+
+def update_application_meta(application_id:str=None, application_name:str=None, description:str=None, upload_file:str=None) -> None:
+    """
+    Update application meta data
+    application_id: application id, You should name it, example: translat_text, ai_draw
+    application_name: application name
+    description: application description
+    upload_file: 'yes' or 'no', when upload_file is 'yes', the application can upload file, when upload_file is 'no', the application can not upload file
+    """
+    import os, json
+    bot_json_path = os.path.join(get_code_dir(), 'bot.json')
+    if os.path.exists(bot_json_path):
+        with open(bot_json_path, 'r', encoding='utf-8') as f:
+            app_json = json.loads(f.read())
+    else:
+        app_json = {}
+    if application_id is not None:
+        from GeneralAgent import skills
+        bots = skills.load_applications()
+        if application_id in [x['id'] for x in bots]:
+            print(f'application_id ({application_id}) exists. ignore If you are just edit the exist application, or you should change the application_id')
+        app_json['id'] = application_id
+    if application_name is not None:
+        app_json['name'] = application_name
+    if description is not None:
+        app_json['description'] = description
+    if upload_file is not None:
+        app_json['upload_file'] = upload_file
+    if os.path.exists(os.path.join(get_code_dir(), 'icon.jpg')):
+        app_json['icon'] = 'icon.jpg'
+    else:
+        del app_json['icon']
+    with open(bot_json_path, 'w', encoding='utf-8') as f:
+        f.write(json.dumps(app_json, indent=4))
+
+
+def install_application() -> None:
+    """
+    Install application to chat bot
+    """
+    # TODO: check function_id and application_name
+    import os, json
+    bot_json_path = os.path.join(get_code_dir(), 'bot.json')
+    if os.path.exists(bot_json_path):
+        with open(bot_json_path, 'r', encoding='utf-8') as f:
+            app_json = json.loads(f.read())
+    else:
+        print('applicatoin meta not exists')
+        return
+    application_id = app_json['id']
+    # move code to bot
+    from GeneralAgent.utils import get_applications_dir
+    target_dir = os.path.join(get_applications_dir(), application_id)
+    if os.path.exists(target_dir):
+        import shutil
+        shutil.rmtree(target_dir)
+    os.makedirs(target_dir)
+    # print(target_dir)
+    os.system(f"cp -r {get_code_dir()}/* {target_dir}")
+
+
+def uninstall_application() -> None:
+    """
+    Uninstall application from chat bot
+    """
+    import os, json
+    bot_json_path = os.path.join(get_code_dir(), 'bot.json')
+    if os.path.exists(bot_json_path):
+        with open(bot_json_path, 'r', encoding='utf-8') as f:
+            app_json = json.loads(f.read())
+    else:
+        print('applicatoin meta not exists')
+        return
+    application_id = app_json['id']
+    # move code to bot
+    from GeneralAgent.utils import get_applications_dir
+    target_dir = os.path.join(get_applications_dir(), application_id)
+    if os.path.exists(target_dir):
+        import shutil
+        shutil.rmtree(target_dir)
+
+
+
+def _generate_function_code(task:str, default_code=None, search_functions=False):
+    """Return the python function code text that completes the task to be used by other function or application, when default_code is not None, update default_code by task"""
+    
+    """
+    Return the python function code text that completes the task(a string)
+    task: 文字描述的任务
+    default_code: 如果不为None，按task修改默认代码，否则按task生成代码
+    return: 一个python代码字符串，主要包含了一个函数
+    """
+    # global skills
+    import os
+    from GeneralAgent import skills
+    python_version = skills.get_python_version()
+    requirements = skills.get_current_env_python_libs()
+    the_skills_can_use = skills._search_functions(task) if search_functions else ''
+    prompt = f"""
+You are a python expert, write a function to complete user's task
+
+# Python Version
+{python_version}
+
+# Python Libs installed
+{requirements}
+
+# You can use skills lib(from GeneralAgent import skills), the function in the lib are:
+{the_skills_can_use}
+
+
+# CONSTRAINTS:
+- Do not import the lib that the function not use.
+- Import the lib in the function, any import statement must be placed in the function
+- docstring the function simplely
+- Do not use other libraries
+- In the code, Intermediate files are written directly to the current directory (./)
+- Give the function a name that describle the task
+- The docstring of the function should be as concise as possible without losing key information, only one line, and output in English
+- The code should be as simple as possible and the operation complexity should be low
+
+# Demo:
+```python
+def translate(text:str, language:str) -> str:
+    \"\"\"
+    translate, return the translated text
+    Parameters: text -- user text, string
+    Returns: the translated text, string
+    \"\"\"
+    from GeneralAgent import skills
+    contents = text.split('.')
+    translated = []
+    for x in contents:
+        prompt = "Translate the following text to " + language + "\n" + x
+        translated += [skills.llm_inference([{{'role': 'system', 'content': prompt}}])
+    return '. '.join(translated)
+```
+
+Please think step by step carefully, consider any possible situation, and write a complete function.
+Just reponse the python code, no any explain, no start with ```python, no end with ```, no any other text.
+"""
+    messages = [{"role": "system", "content": prompt}]
+    if default_code is not None:
+        messages += [{"role": "system", "content": "user's code: " + default_code}]
+    messages += [{"role": "system", "content": f"user's task: {task}"}]
+    code = skills.llm_inference(messages, model_type='smart')
+    code = skills.get_python_code(code)
+    return code
+
+
+def application_code_generation(task, default_code=None):
+    """Return the python code text that completes the task to build a chat bot, when default_code is not None, update default_code by task"""
+    from GeneralAgent import skills
+    python_version = skills.get_python_version()
+    requirements = skills.get_current_env_python_libs()
+    the_skills_can_use = skills._search_functions(task)
+
+    prompt = f"""
+You are a python expert, write a python function to complete user's task.
+The function in code will be used to create a chat bot, like slack, discord.
+
+# Function signature
+```
+async def main(chat_history, input, file_path, output_callback, file_callback, ui_callback):
+    # chat_history is a list of dict, like [{{"role": "user", "content": "hello"}}, {{"role": "system", "content": "hi"}}]
+    # input is a string, user's input
+    # file_path is a string, user's file path
+    # output_callback is a async function, output_callback(content: str) -> None
+    # file_callback is a async function, file_callback(file_path: str) -> None
+    # ui_callback is a async function, ui_callback(name:str, js_path:str, data={{}}) -> None
+```
+
+# Python Version: {python_version}
+
+# Python Libs installed
+{requirements}
+
+# You can use skills lib(from GeneralAgent import skills), the function in the lib are:
+{the_skills_can_use}
+
+# CONSTRAINTS:
+- Do not import the lib that the function not use.
+- Import the lib in the function
+- In the code, Intermediate files are written directly to the current directory (./)
+- Give the function a name that describe the task
+- The docstring of the function should be as concise as possible without losing key information, only one line, and output in English
+
+# DEMO 1 : Chat with A large language model
+```python
+async def main(chat_history, input, file_path, output_callback, file_callback, ui_callback):
+    from GeneralAgent import skills
+    chat_history = skills.cut_messages(chat_history, 4000)
+    messages = [{{"role": "system", "content": "You are a helpful assistant."}}] + chat_history
+    response = skills.llm_inference(messages, stream=True)
+    for token in response:
+        await output_callback(token)
+    await output_callback(None)
+```
+
+# DEMO 2 : Create a image by user's prompt
+```python
+async def main(chat_history, input, file_path, output_callback, file_callback, ui_callback):
+    from GeneralAgent import skills
+    prompt = input
+    if not skills.text_is_english(prompt):
+        prompt = skills.translate_text(prompt, 'english')
+    image_url = skills.image_generation(prompt)
+    await file_callback(image_url)
+```
+
+Please think step by step carefully, consider any possible situation, and write a complete code like DEMO
+Just reponse the python code, no any explain, no start with ```python, no end with ```, no any other text.
+"""
+
+    messages = [{"role": "system", "content": prompt}]
+    if default_code is not None:
+        messages += [{"role": "system", "content": "user's code: " + default_code}]
+    messages += [{"role": "system", "content": f"user's task: {task}"}]
+    code = skills.llm_inference(messages, model_type='smart')
+    code = skills.get_python_code(code)
+    return code
+
+
+def _generate_agent_code(task_description, default_code=None):
+    """Return the python code text that completes the task to build a chat bot, when default_code is not None, update default_code by task"""
+    from GeneralAgent import skills
+    python_version = skills.get_python_version()
+    requirements = skills.get_current_env_python_libs()
+    prompt = f"""
+You are a python expert, write a python function to complete user's task.
+The function in code will be used to create a chat bot, like slack, discord.
+
+# Function signature
+```
+async def main(chat_history, input, file_path, output_callback, file_callback, ui_callback):
+    # chat_history is a list of dict, like [{{"role": "user", "content": "hello"}}, {{"role": "system", "content": "hi"}}]
+    # input is a string, user's input
+    # file_path is a string, user's file path
+    # output_callback is a async function, output_callback(content: str) -> None
+    # file_callback is a async function, file_callback(file_path: str) -> None
+    # ui_callback is a async function, ui_callback(name:str, js_path:str, data={{}}) -> None
+```
+
+# Python Version: {python_version}
+
+# Python Libs installed
+{requirements}
+
+# CONSTRAINTS:
+- Do not import the lib that the function not use.
+- Import the lib in the function
+- In the code, Intermediate files are written directly to the current directory (./)
+- Give the function a name that describe the task
+- The docstring of the function should be as concise as possible without losing key information, only one line, and output in English
+- Every created file should have a unique name, which can be generated by skills.unique_name()
+
+# DEMO 1 : write user's input to a file and return
+```python
+async def main(chat_history, input, file_path, output_callback, file_callback, ui_callback):
+    from GeneralAgent import skills
+    
+```
+
+# DEMO 2 : Agent with functions
+```python
+async def main(chat_history, input, file_path, output_callback, file_callback, ui_callback):
+    from GeneralAgent.agent import Agent
+    role_prompt = \"\"\"
+You are a translation agent.
+You complete user requirements by writing python code to call the predefined functions.
+\"\"\"
+    functions = [
+        skills.translate_text
+    ]
+    agent = Agent.with_functions(functions, role_prompt)
+    await agent.run(input, output_callback=output_callback)
+```python
+
+# There are two function types:
+1. Application: like DEMO1, The application process is fixed and less flexible, but the function will be more stable
+2. Agent: like DEMO2, Agent is a chat bot that can use functions to complete user's task. The agent will automatic handle user's input and output
+You can choose one of them to complete the task.
+
+Please think step by step carefully, consider any possible situation, and write a complete code like DEMO
+Just reponse the python code, no any explain, no start with ```python, no end with ```, no any other text.
+"""
+
+    messages = [{"role": "system", "content": prompt}]
+    if default_code is not None:
+        messages += [{"role": "system", "content": "user's code: " + default_code}]
+    messages += [{"role": "system", "content": f"user's task: {task_description}"}]
+    code = skills.llm_inference(messages, model_type='smart')
+    code = skills.get_python_code(code)
+    return code
+
+
+def _generate_llm_task_function(task_description, default_code=None):
+    """
+    This function generates a Python function to perform a specific task using a large language model (LLM), such as translation, planning, answering general knowledge questions.
+    
+    Parameters:
+    task_description (str): A description of the task that the generated function should perform.
+
+    Returns:
+    str: The generated Python function code as a string.
+    """
+    import os
+    from GeneralAgent import skills
+    python_version = skills.get_python_version()
+    requirements = skills.get_current_env_python_libs()
+    the_skills_can_use = skills._search_functions(task_description) if search_functions else ''
+    prompt = f"""
+You are a python expert, write a function to complete user's task
+
+# Python Version
+{python_version}
+
+# Python Libs installed
+{requirements}
+
+# You can use skills lib(from GeneralAgent import skills), the function in the lib are:
+
+def skills.simple_llm_inference(messages, json_schema):
+     Run LLM (large language model) inference on the provided messages, The total number of tokens in the messages and the returned string must be less than 8000.
+     @param messages: Input messages for the model, like [{{'role': 'system', 'content': 'You are a helpful assistant'}}, {{'role': 'user', 'content': 'translate blow to english:\nxxxx'}}]
+     @param json_schema: the json schema of return dictionary, like {{"type": "object", "properties": {{"name": {{"type": "string"}}, "age": {{"type": "integer" }} }} }}
+     @return returned as a dictionary According to the provided JSON schema.
+
+{the_skills_can_use}
+
+# CONSTRAINTS:
+- Do not import the lib that the function not use.
+- Import the lib in the function, any import statement must be placed in the function
+- docstring the function simplely
+- Do not use other libraries
+- In the code, Intermediate files are written directly to the current directory (./)
+- Give the function a name that describle the task
+- The docstring of the function should be as concise as possible without losing key information, only one line, and output in English
+- The code should be as simple as possible and the operation complexity should be low
+- Every created file should have a unique name, which can be generated by skills.unique_name()
+
+# Demo:
+```python
+def translate(text:str, language:str) -> str:
+    \"\"\"
+    translate, return the translated text
+    Parameters: text -- user text, string
+    Returns: the translated text, string
+    \"\"\"
+    from GeneralAgent import skills
+    contents = text.split('.')
+    translated = []
+    for x in contents:
+        prompt = "Translate the following text to " + language + "\n" + x
+        translated += [skills.llm_inference([{{'role': 'system', 'content': prompt}}])
+    return '. '.join(translated)
+```
+
+Please think step by step carefully, consider any possible situation, and write a complete function.
+Just reponse the python code, no any explain, no start with ```python, no end with ```, no any other text.
+"""
+    messages = [{"role": "system", "content": prompt}]
+    if default_code is not None:
+        messages += [{"role": "system", "content": "user's code: " + default_code}]
+    messages += [{"role": "system", "content": f"user's task: {task_description}"}]
+    code = skills.llm_inference(messages, model_type='smart')
+    code = skills.get_python_code(code)
+    return code
+
+#     from GeneralAgent import skills
+#     from jinja2 import Template
+#     prompt_template = """
+# 你是一个python专家。
+
+
+# Your job is to have a large language model (LLM) perform specific tasks, such as translation, planning, answering general knowledge questions, etc.
+# Your job is to have a large language model (LLM) perform specific tasks, such as translation, planning, answering general knowledge questions, etc.
+# Large language model calling function:
+
+# ```python
+# def xxx(xxx):
+#      \"\"\"
+#      xxx
+#      \"\"\"
+#      from GeneralAgent import skills
+#      # skills.simple_llm_inference
+# ```
+
+# # Task
+
+# {{task}}
+
+# # Note:
+
+# - All imports should be placed inside the function.
+# - While creating your function, consider the all edge cases.
+# - Do not use any other libraries except simple_llm_inference and installed libraries.
+# - The simple_llm_inference function requires that the input messages are less than 8000, and the output length is less than 8000. - 
+# - When the task cannot be completed through one simple_llm_inference, you should consider task disassembly.
+# """
+#     prompt = Template(prompt_template).render({'task': task_description})
+#     if default_code is not None:
+#         prompt += '\n' + 'user\'s code: ' + default_code + '\nUpdate the code to complete the task'
+#     result = skills.llm_inference([{'role': 'system', 'content': prompt}, {'role': 'user', 'content': 'You are a python expert.'}, {'role': 'user', 'content': prompt}], model_type="smart")
+#     code = skills.get_python_code(result)
+#     return code
+
+
+# def create_function(func_name:str, task_description:str):
+#     """
+#     create a function by task description. Where task_description can include functions in GeneralAgent.skills
+#     """
+#     # from GeneralAgent import skills
+#     import os
+#     code = _generate_function_code(task_description)
+#     file_path = os.path.join(get_code_dir(), func_name + '.py')
+#     with open(file_path, 'w', encoding='utf-8') as f:
+#         f.write(code)
+
+# def delete_function(func_name:str) -> None:
+#     """
+#     Delete a function by name
+#     """
+#     import os
+#     file_path = os.path.join(get_code_dir(), func_name + '.py')
+#     if os.path.exists(file_path):
+#         os.remove(file_path)
+
+# def list_functions() -> [str]:
+#     """
+#     list all function names
+#     """
+#     # TODO function description
+#     import os
+#     files = os.listdir(get_code_dir())
+#     functions = [x.split('.')[0] for x in files]
+#     return functions
+
+# def show_function(func_name:str) -> str:
+#     """
+#     Show a function code by name
+#     """
+#     import os
+#     file_path = os.path.join(get_code_dir(), func_name + '.py')
+#     if os.path.exists(file_path):
+#         with open(file_path, 'r', encoding='utf-8') as f:
+#             code = f.read()
+#         return code
+#     else:
+#         return None
+
+
+# def create_application(task_description:str) -> None:
+#     """
+#     Create a application by task_description description. The application name is application_name, the task_description is task_description(string)
+#     """
+#     import os
+#     from GeneralAgent import skills
+#     # code = application_code_generation(task_description)
+#     code = _generate_agent_code(task_description)
+#     code_path = os.path.join(get_code_dir(),  'main.py')
+#     with open(code_path, 'w', encoding='utf-8') as f:
+#         f.write(code)

GeneralAgent/skills/applications.py

@@ -0,0 +1,85 @@
+
+def load_applications():
+    """
+    load all applications(bots) metadata
+    """
+    from GeneralAgent.utils import get_applications_dir
+    local_bots = _load_bots(_get_local_applications_dir())
+    remote_bots=  _load_bots(get_applications_dir())
+    for bot in remote_bots:
+        bot['nickname'] = '# ' + bot['nickname']
+    return local_bots + remote_bots
+
+
+def _get_local_applications_dir():
+    from GeneralAgent.utils import get_local_applications_dir
+    return get_local_applications_dir()
+
+
+def _load_bots(code_dir):
+    import os, json
+    result = []
+    for bot_name in os.listdir(code_dir):
+        bot_dir = os.path.join(code_dir, bot_name)
+        if os.path.isdir(bot_dir):
+            result.append(_load_bot_metadata(bot_dir))
+    return result
+
+
+def _load_bot_metadata(bot_dir):
+    import os, json
+    bot_json_path = os.path.join(bot_dir, 'bot.json')
+    if os.path.exists(bot_json_path):
+        with open(bot_json_path, 'r', encoding='utf-8') as f:
+            bot_json = json.load(f)
+            if 'icon' in bot_json:
+                bot_json['icon_url'] = os.path.join(bot_dir, bot_json['icon'])
+            if 'js_path' in bot_json:
+                bot_json['js_path'] = os.path.join(bot_dir, bot_json['js_path'])
+            bot_json['nickname'] = bot_json['name']
+            return bot_json
+    return None
+
+
+def load_bot_metadata_by_id(bot_id):
+    """
+    load bot metadata by bot id
+    """
+    import os
+    local_dir = _get_local_applications_dir()
+    from GeneralAgent.utils import get_applications_dir
+    remote_dir = get_applications_dir()
+    bot_dir = os.path.join(local_dir, bot_id)
+    if not os.path.exists(bot_dir):
+        bot_dir = os.path.join(remote_dir, bot_id)
+    return _load_bot_metadata(bot_dir)
+
+
+def get_application_module(bot_id):
+    """
+    get application module by bot id
+    """
+    import os
+    from GeneralAgent.utils import get_applications_dir
+    local_dir = _get_local_applications_dir()
+    remote_dir = get_applications_dir()
+    code_path = os.path.join(local_dir, f"{bot_id}/main.py")
+    if not os.path.exists(code_path):
+        code_path = os.path.join(remote_dir, f"{bot_id}/main.py")
+    return _load_application(code_path)
+
+
+def _load_application(code_path):
+    """
+    load application with code path
+    """
+    import os, importlib.util, logging
+    application = None
+    try:
+        spec = importlib.util.spec_from_file_location("main", code_path)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        application = module
+    except Exception as e:
+        logging.exception(e)
+    return application

GeneralAgent/skills/build_web.py

@@ -0,0 +1,149 @@
+from ast import Tuple
+import os
+
+def compile_tsx(lib_name:str, code:str, target_dir:str):
+    """
+    Compile tsx code into a UI library.
+    @lib_name: the name of the UI library
+    @code: the tsx code
+    @target_dir: the directory to save the UI library
+    """
+    # 目标目录不存在就创建
+    if os.path.exists(target_dir):
+        import shutil
+        shutil.rmtree(target_dir)
+    os.makedirs(target_dir)
+
+    from GeneralAgent.utils import get_tsx_builder_dir
+    ts_builder_dir = get_tsx_builder_dir()
+
+    code_path = os.path.join(ts_builder_dir, 'src/lib/index.tsx')
+    with open(code_path, 'w', encoding='utf-8') as f:
+        f.write(code)
+    # 写入lib名称
+    webpack_template_path = os.path.join(ts_builder_dir, 'webpack.config.template.js')
+    webpack_template = ''
+    with open(webpack_template_path, 'r', encoding='utf-8') as f:
+        webpack_template = f.read()
+    webpack_template = webpack_template.replace('LibTemplate', lib_name)
+    webpack_path = os.path.join(ts_builder_dir, 'webpack.config.js')
+    with open(webpack_path, 'w', encoding='utf-8') as f:
+        f.write(webpack_template)
+    # 获取编译的标准输出
+    output = os.popen(f"cd {ts_builder_dir} && npm run build").read()
+    if 'successfully' not in output:
+        print(output)
+        return False
+    # 将编译后的文件移动到目标目录
+    os.system(f"mv {ts_builder_dir}/build/* {target_dir}")
+    return True
+
+
+def _llm_write_ui_lib(lib_name, task):
+    from GeneralAgent import skills
+    prompt_template = """
+You are a React and Typescript expert.
+
+# Task
+Create a React function component named LibTemplate in tsx language. 
+The component should have the following functionality:
+{{task}}
+
+# Import
+Use the following import syntax:
+```
+const React = (window as any).React;
+const antd = (window as any).antd;
+```
+No other import methods are allowed.
+
+# DEMO
+
+```tsx
+const React = (window as any).React;
+const antd = (window as any).antd;
+
+const [Form, Input, Button] = [antd.Form, antd.Input, antd.Button];
+
+const LibTemplate = ({save_data}: {save_data: (data:any)=>void}) => {
+  // use save_data to save the data
+}
+
+export default LibTemplate;
+```
+
+Please reponse the component code which finish the task without any explaination.
+"""
+
+    from jinja2 import Template
+    prompt = Template(prompt_template).render(task=task)
+    messages = [{'role': 'system', 'content': prompt}]
+    response = skills.llm_inference(messages, model_type="normal", stream=True)
+    result = ''
+    for token in response:
+        # print(token, end='', flush=True)
+        result += token
+    result = result.replace('LibTemplate', lib_name)
+    return result
+
+
+def extract_tsx_code(content):
+    """
+    Extract tsx code from markdown content.
+    """
+    import re
+    # 兼容tsx和ts
+    pattern = re.compile(r'```tsx?\n([\s\S]*)\n```')
+    match = pattern.search(content)
+    if match:
+        return match.group(1)
+    else:
+        return content
+
+# def create_ui(task: str, ui_dir: str = './ui', component_name: str = None) -> (str, str):
+#     """
+#     Convert task into UI components. Return (component_name, js_path) tuple.
+#     """
+def create_ui(task: str, ui_dir: str = './ui', component_name: str = None) -> (str, str):
+    """
+    Convert a given task description into UI components.
+
+    Args:
+        task: A string representing the task description with all the necessary details.
+
+    Returns:
+        A tuple containing the name of the UI component and the path to the JavaScript file.
+
+    Example:
+        create_ui('A task description with all the necessary details')
+    """
+    import uuid
+    import os
+    lib_name = component_name
+    if lib_name is None:
+        lib_name = 'Lib' + str(uuid.uuid1())[:4]
+    if not os.path.exists(ui_dir):
+        os.makedirs(ui_dir)
+    target_dir = os.path.join(ui_dir, lib_name)
+    for _ in range(2):
+        content = _llm_write_ui_lib(lib_name, task)
+        code = extract_tsx_code(content)
+        success = compile_tsx(lib_name, code, target_dir)
+        if success:
+            return lib_name, os.path.join(target_dir, 'index.js')
+    return None
+
+def parse_tsx_to_ui(code, save_dir=None):
+    import uuid
+    lib_name = 'Lib' + str(uuid.uuid1())[:4]
+    if save_dir is None:
+        from GeneralAgent import skills
+        save_dir = skills.get_code_dir()
+    if not os.path.exists(save_dir):
+        os.makedirs(save_dir)
+    target_dir = os.path.join(save_dir, lib_name)
+    success = compile_tsx(lib_name, code, target_dir)
+    if success:
+        return lib_name, os.path.join(target_dir, 'index.js')
+    else:
+        return None

GeneralAgent/skills/concatenate_videos/concatenate_videos.py

@@ -0,0 +1,27 @@
+
+def concatenate_videos(video_list: list) -> str:
+    """
+    Concatenate a list of videos into one video.
+    @param video_list: A list of video file paths.
+    @return: The file path of the concatenated video.
+    """
+    import uuid
+    from moviepy.editor import concatenate_videoclips, VideoFileClip
+    clips = [VideoFileClip(video) for video in video_list]
+    final_clip = concatenate_videoclips(clips)
+    output_path = f"{uuid.uuid4().hex}.mp4"  # Generate a unique output file name
+    final_clip.write_videofile(output_path)
+    return output_path
+
+def test_concatenate_videos():
+    """
+    Test the concatenate_videos function.
+    """
+    import os
+    file_path = os.path.join(os.path.dirname(__file__), "f63bfaae7b0e.mp4")
+    video_list = [file_path, file_path, file_path]  # Use the provided video file for testing
+    output_path = concatenate_videos(video_list)
+    assert os.path.exists(output_path)
+
+if __name__ == '__main__':
+    test_concatenate_videos()

GeneralAgent/skills/download_file.py

@@ -0,0 +1,50 @@
+
+def download_file(file_url, save_path):
+    """download file to save_path, return True if success, else False"""
+    """实现一个下载文件的函数，返回文件路径"""
+    import requests
+    import logging
+    try_count = 3
+    while try_count > 0:
+        try:
+            response = requests.get(file_url)
+            with open(save_path, "wb") as f:
+                f.write(response.content)
+            # print("Success: 文件(%s)已下载至 %s" % (file_url, save_path))
+            return True
+        except Exception as e:
+            # print("Error: 文件下载失败:", e)
+            logging.error(e)
+            import time
+            time.sleep(5)
+            try_count -= 1
+            continue
+    return False
+
+
+def try_download_file(file_path):
+    import os
+    from GeneralAgent import skills
+    from PIL import Image
+    """Try to download file if it is a url, else return file_path"""
+    if file_path.startswith("http://") or file_path.startswith("https://"):
+        save_path = skills.unique_name() + '.' + file_path.split('.')[-1]
+        success = skills.download_file(file_path, save_path)
+        if success:
+            if save_path.endswith('.png') or save_path.endswith('.PNG'):
+                # 转成jpg
+                png_image = Image.open(save_path)
+                jpg_save_path = skills.unique_name() + '.jpg'
+                png_image.save(jpg_save_path, 'JPEG')
+                os.remove(save_path)
+                return jpg_save_path
+            else:
+                return save_path
+        else:
+            return file_path
+    else:
+        return file_path
+
+if __name__ == '__main__':
+    # download_file('https://ai.tongtianta.site/file-upload/gvsAc4cEm543iaX5x/5.pdf', '1.pdf')
+    download_file('https://ai.tongtianta.site/file-upload/27XEb3WgyDru5eFFe/9.pdf', '3.pdf')

GeneralAgent/skills/file_operation.py

@@ -0,0 +1,51 @@
+    
+def read_pdf_pages(file_path):
+    """Read the pdf file and return a list of strings on each page of the pdf"""
+    """读取pdf文件，返回pdf每页字符串的列表"""
+    import fitz
+    doc = fitz.open(file_path)
+    documents = []
+    for page in doc:
+        documents.append(page.get_text())
+    return documents
+    
+def read_word_pages(file_path):
+    """Read the word file and return a list of word paragraph strings"""
+    """读取word文件，返回word段落字符串的列表"""
+    # https://zhuanlan.zhihu.com/p/146363527
+    from docx import Document
+    # 打开文档
+    document = Document(file_path)
+    # 读取标题、段落、列表内容
+    ps = [ paragraph.text for paragraph in document.paragraphs]
+    return ps
+
+def read_ppt(file_path):
+    import pptx
+    prs = pptx.Presentation(file_path)
+    documents = []
+    for slide in prs.slides:
+        for shape in slide.shapes:
+            if hasattr(shape, "text"):
+                documents.append(shape.text)
+    return '\n'.join(documents)
+
+
+def read_file_content(file_path):
+    """return content of txt, md, pdf, docx file"""
+    # 支持file_path的类型包括txt、md、pdf、docx
+    if file_path.endswith('.pdf'):
+        return ' '.join(read_pdf_pages(file_path))
+    elif file_path.endswith('.docx'):
+        return ' '.join(read_word_pages(file_path))
+    elif file_path.endswith('.ppt') or file_path.endswith('.pptx'):
+        return read_ppt(file_path)
+    else:
+        # 默认当做文本文件
+        with open(file_path, 'r', encoding='utf-8') as f:
+            return '\n'.join(f.readlines())
+
+def write_file_content(file_path, content):
+    """write content to txt, md"""
+    with open(file_path, 'w', encoding='utf-8') as f:
+        f.write(content)

GeneralAgent/skills/llm_inference.py

@@ -0,0 +1,458 @@
+# from retrying import retry as _retry
+# from tenacity import retry, stop_after_attempt, wait_fixed
+
+class TinyDBCache():
+    def __init__(self):
+        from tinydb import TinyDB
+        import os, json
+        llm_path = os.environ.get('CACHE_PATH', None)
+        if llm_path is None:
+            from GeneralAgent.utils import get_server_dir
+            llm_path = os.path.join(get_server_dir(), 'cache.json')
+        self.db = TinyDB(llm_path)
+
+    @property
+    def cache_llm(self):
+        import os
+        return os.environ.get('LLM_CACHE', 'no') in ['yes', 'y', 'YES']
+    
+    @property
+    def cache_embedding(self):
+        import os
+        return os.environ.get('EMBEDDING_CACHE', 'no') in ['yes', 'y', 'YES']
+
+    def get(self, table, key):
+        from tinydb import Query
+        result = self.db.table(table).get(Query().key == key)
+        if result is not None:
+            return result['value']
+        else:
+            return None
+
+    def set(self, table, key, value):
+        from tinydb import Query
+        self.db.table(table).upsert({'key': key, 'value': value}, Query().key == key)
+
+    def set_llm_cache(self, key, value):
+        if self.cache_llm:
+            self.set('llm', key, value)
+
+    def get_llm_cache(self, key):
+        if self.cache_llm:
+            return self.get('llm', key)
+    
+    def set_embedding_cache(self, key, value):
+        if self.cache_embedding:
+            self.set('embedding', key, value)
+
+    def get_embedding_cache(self, key):
+        if self.cache_embedding:
+            return self.get('embedding', key)
+
+
+global_cache = TinyDBCache()
+
+
+def embedding_single(text) -> [float]:
+    """
+    embedding the text and return a embedding (list of float) for the string
+    """
+    global global_cache
+    key = _md5(text)
+    embedding = global_cache.get_embedding_cache(key)
+    if embedding is not None:
+        # print('embedding cache hitted')
+        return embedding
+    texts = [text]
+    from litellm import embedding
+    resp = embedding(model = _get_embedding_model(),
+                     input=texts
+                     )
+    result = [x['embedding'] for x in resp['data']]
+    embedding = result[0]
+    global_cache.set_embedding_cache(key, embedding)
+    return embedding
+
+
+def test_embedding_single():
+    size = None
+    from GeneralAgent.utils import EnvironmentVariableManager
+    with EnvironmentVariableManager('EMBEDDING_CACHE', 'no'):
+        for source_type in ['OPENAI', 'AZURE']:
+            with EnvironmentVariableManager('LLM_SOURCE', source_type):
+                result = embedding_single("Say this is a test")
+                # print(source_type, result)
+                if size is not None:
+                    assert len(result) == size
+                size = len(result)
+
+def test_embedding_batch():
+    size = None
+    from GeneralAgent.utils import EnvironmentVariableManager
+    with EnvironmentVariableManager('EMBEDDING_CACHE', 'no'):
+        for source_type in ['OPENAI', 'AZURE']:
+            with EnvironmentVariableManager('LLM_SOURCE', source_type):
+                result = embedding_batch(["Say this is a test"])
+                if size is not None:
+                    assert len(result[0]) == size
+                size = len(result[0])
+
+
+def embedding_batch(texts) -> [[float]]:
+    """
+    embedding the texts(list of string), and return a list of embedding (list of float) for every string
+    """
+    global global_cache
+    embeddings = {}
+    remain_texts = []
+    for text in texts:
+        key = _md5(text)
+        embedding = global_cache.get_embedding_cache(key)
+        if embedding is not None:
+            embeddings[text] = embedding
+        else:
+            remain_texts.append(text)
+    if len(remain_texts) > 0:
+        result = _embedding_many(remain_texts)
+        for text, embedding in zip(remain_texts, result):
+            key = _md5(text)
+            global_cache.set_embedding_cache(key, embedding)
+            embeddings[text] = embedding
+    return [embeddings[text] for text in texts]
+
+
+def _embedding_many(texts) -> [[float]]:
+    from litellm import embedding
+    # 每次最多embedding 16个
+    max_batch_size = 16
+    result = []
+    for i in range(0, len(texts), max_batch_size):
+        resp = embedding(model=_get_embedding_model(),input=texts[i:i+max_batch_size])
+        result += [x['embedding'] for x in resp['data']]
+    return result
+
+
+def cos_sim(a, b):
+    import numpy as np
+    from numpy.linalg import norm
+    # This function calculates the cosine similarity (scalar value) between two input vectors 'a' and 'b' (1-D array object), and return the similarity.
+    a = a if isinstance(a, np.ndarray) else np.array(a)
+    b = b if isinstance(b, np.ndarray) else np.array(b)
+    return np.dot(a, b)/(norm(a)*norm(b))
+
+
+def search_similar_texts(focal:str, texts:[str], top_k=5):
+    """
+    search the most similar texts in texts, and return the top_k similar texts
+    """
+    embeddings = embedding_batch([focal] + texts)
+    focal_embedding = embeddings[0]
+    texts_embeddings = embeddings[1:]
+    import numpy as np
+    similarities = np.dot(texts_embeddings, focal_embedding)
+    sorted_indices = np.argsort(similarities)
+    sorted_indices = sorted_indices[::-1]
+    return [texts[i] for i in sorted_indices[:top_k]]
+
+
+def _md5(obj):
+    import hashlib, json
+    if isinstance(obj, str):
+        return hashlib.md5(obj.encode('utf-8')).hexdigest()
+    else:
+        return hashlib.md5(json.dumps(obj).encode('utf-8')).hexdigest()
+
+
+def _get_llm_model(messages, model_type):
+    import os
+    from GeneralAgent import skills
+    assert model_type in ['normal', 'smart', 'long']
+    if model_type == 'normal' and skills.messages_token_count(messages) > 3000:
+        model_type = 'long'
+    api_type = os.environ.get('LLM_SOURCE', 'OPENAI')
+    model_key = f'{api_type}_LLM_MODEL_{model_type.upper()}'
+    model = os.environ.get(model_key, None)
+    if model is not None:
+        return model
+    model_key = f'{api_type}_LLM_MODEL_NORMAL'
+    return os.environ.get(model_key, 'gpt-3.5-turbo')
+
+
+def get_llm_token_limit(model_type):
+    """
+    return the token limit for the model
+    """
+    import os
+    assert model_type in ['normal', 'smart', 'long']
+    api_type = os.environ.get('LLM_SOURCE', 'OPENAI')
+    # OPENAI_LLM_MODEL_SMART_LIMIT
+    limit_key = f'{api_type}_LLM_MODEL_{model_type.upper()}_LIMIT'
+    limit = os.environ.get(limit_key, None)
+    if limit is not None:
+        return int(limit)
+    limit_key = f'{api_type}_LLM_MODEL_NORMAL_LIMIT'
+    return int(os.environ.get(limit_key, 4000))
+
+
+def test_get_llm_token_limit():
+    from GeneralAgent.utils import EnvironmentVariableManager
+    with EnvironmentVariableManager('LLM_SOURCE', 'OPENAI'):
+        with EnvironmentVariableManager('OPENAI_LLM_MODEL_SMART_LIMIT', '8000'):
+            assert get_llm_token_limit('smart') == 8000
+
+
+def _get_embedding_model():
+    import os
+    api_type = os.environ.get('LLM_SOURCE', 'OPENAI')
+    embedding_model = os.environ.get(f'{api_type}_EMBEDDING_MODEL', 'text-embedding-ada-002')
+    return embedding_model
+
+def _get_temperature():
+    import os
+    temperature = float(os.environ.get('LLM_TEMPERATURE', 0.5))
+    return temperature
+
+
+def llm_inference(messages, model_type='normal', stream=False, json_schema=None):
+    """
+    Run LLM (large language model) inference on the provided messages using the specified model.
+    
+    Parameters:
+    messages: Input messages for the model, like [{'role': 'system', 'content': 'You are a helpful assistant'}, {'role': 'user', 'content': 'What is your name?'}]
+    model_type: Type of model to use. Options are 'normal', 'smart', 'long'
+    use_stream: Boolean indicating if the function should use streaming inference
+    json_format: Optional JSON schema string
+
+    Returns:
+    If use_stream is True, returns a generator that yields the inference results as they become available.
+    If use_stream is False, returns a string containing the inference result.
+    If json_format is provided, the inference result is parsed according to the provided JSON schema and returned as a dictionary.
+
+    Note:
+    The total number of tokens in the messages and the returned string must be less than 4000 when model_variant is 'normal', and less than 16000 when model_variant is 'long'.
+    """
+    import logging
+    if stream:
+        return _llm_inference_with_stream(messages, model_type)
+    else:
+        if json_schema is None:
+            return _llm_inference_without_stream(messages, model_type)    
+        else:
+            import json
+            if not isinstance(json_schema, str):
+                json_schema = json.dumps(json_schema)
+            messages[-1]['content'] += '\n' + return_json_prompt + json_schema
+            # messages += [{'role': 'user', 'content': return_json_prompt + json_schema}]
+            logging.debug(messages)
+            result = _llm_inference_without_stream(messages, model_type)
+            logging.debug(result)
+            return json.loads(fix_llm_json_str(result))
+
+
+def test_llm_inference():
+    from GeneralAgent.utils import EnvironmentVariableManager
+    with EnvironmentVariableManager('LLM_CACHE', 'no'):
+        for source_type in ['OPENAI', 'AZURE']:
+            with EnvironmentVariableManager('LLM_SOURCE', source_type):
+                model_types = ['normal', 'smart', 'long']
+                for model_type in model_types:
+                    result = llm_inference([{"role": "user","content": "Say this is a test",}], model_type=model_type, stream=False)
+                    print(source_type, model_type, result)
+                    assert 'test' in result
+                    result = ''
+                    response = llm_inference([{"role": "user","content": "Say this is a test",}], model_type=model_type, stream=True)
+                    for token in response:
+                        result += token
+                    print(source_type, model_type, result)
+                    assert 'test' in result
+
+
+def simple_llm_inference(messages, json_schema=None):
+    """
+    Run LLM (large language model) inference on the provided messages
+    
+    Parameters:
+    messages: Input messages for the model, like [{'role': 'system', 'content': 'You are a helpful assistant'}, {'role': 'user', 'content': 'What is your name?'}]
+    json_format: Optional JSON schema string
+
+    Returns:
+    If json_format is provided, the inference result is parsed according to the provided JSON schema and returned as a dictionary.
+    Else return a string
+
+    Note:
+    The total number of tokens in the messages and the returned string must be less than 16000.
+    """
+    from GeneralAgent import skills
+    return skills.llm_inference(messages, json_schema=json_schema)
+
+
+# @retry(stop=stop_after_attempt(3), wait=wait_fixed(3))
+async def async_llm_inference(messages, model_type='normal'):
+    from litellm import acompletion
+    import logging
+    global global_cache
+    logging.debug(messages)
+    key = _md5(messages)
+    result = global_cache.get_llm_cache(key)
+    if result is not None:
+        return result
+    model = _get_llm_model(messages, model_type)
+    temperature = _get_temperature()
+    try_count = 3
+    while try_count > 0:
+        try:
+            response = await acompletion(model=model, messages=messages, temperature=temperature)
+            result = response['choices'][0]['message']['content']
+            global_cache.set_llm_cache(key, result)
+            return result
+        except Exception as e:
+            try_count -= 1
+            import asyncio
+            await asyncio.sleep(3)
+        if try_count == 0:
+            raise ValueError('LLM(Large Languate Model) error, Please check your key or base_url, or network')
+    return ''
+
+
+# @retry(stop=stop_after_attempt(3), wait=wait_fixed(3))
+def _llm_inference_with_stream(messages, model_type='normal'):
+    """
+    messages: llm messages, model_type: normal, smart, long
+    """
+    from litellm import completion
+    import logging
+    # from GeneralAgent import skills
+    model = _get_llm_model(messages, model_type)
+    logging.debug(messages)
+    global global_cache
+    key = _md5(messages)
+    result = global_cache.get_llm_cache(key)
+    if result is not None:
+        # print('llm_inference cache hitted')
+        for x in result.split(' '):
+            yield x + ' '
+        yield '\n'
+        # yield None
+    else:
+        temperature = _get_temperature()
+        try_count = 3
+        while try_count > 0:
+            try:
+                response = completion(model=model, messages=messages, stream=True, temperature=temperature)
+                result = ''
+                for chunk in response:
+                    # print(chunk)
+                    if chunk['choices'][0]['finish_reason'] is None:
+                        token = chunk['choices'][0]['delta']['content']
+                        if token is None:
+                            continue
+                        result += token
+                        global_cache.set_llm_cache(key, result)
+                        yield token
+                break
+            except Exception as e:
+                try_count -= 1
+                import time
+                time.sleep(3)
+            if try_count == 0:
+                raise ValueError('LLM(Large Languate Model) error, Please check your key or base_url, or network')
+        # logging.info(result)
+        # yield None
+
+# if we choose to use local llm for inferce, we can use the following completion function.
+#def compeltion(model,messages,temperature):
+#    pass
+
+
+# @retry(stop=stop_after_attempt(3), wait=wait_fixed(3))
+def _llm_inference_without_stream(messages, model_type='normal'):
+    from litellm import completion
+    import logging
+    global global_cache
+    logging.debug(messages)
+    # print(messages)
+    key = _md5(messages)
+    result = global_cache.get_llm_cache(key)
+    if result is not None:
+        return result
+    model = _get_llm_model(messages, model_type)
+    temperature = _get_temperature()
+    try_count = 3
+    while try_count > 0:
+        try:
+            response = completion(model=model, messages=messages, temperature=temperature)
+            result = response['choices'][0]['message']['content']
+            global_cache.set_llm_cache(key, result)
+            return result
+        except Exception as e:
+            try_count -= 1
+            import time
+            time.sleep(3)
+        if try_count == 0:
+                raise ValueError('LLM(Large Languate Model) error, Please check your key or base_url, or network')
+    return ''
+
+
+def fix_llm_json_str(string):
+    import json
+    import re
+    new_string = string.strip()
+    if new_string.startswith('```json'):
+        new_string = new_string[7:]
+        if new_string.endswith('```'):
+            new_string = new_string[:-3]
+    try:
+        json.loads(new_string)
+        return new_string
+    except Exception as e:
+        print("fix_llm_json_str failed 1:", e)
+        try:
+            pattern = r'```json(.*?)```'
+            match = re.findall(pattern, new_string, re.DOTALL)
+            if match:
+                new_string = match[-1]
+            
+            json.loads(new_string)
+            return new_string
+        except Exception as e:
+            print("fix_llm_json_str failed 2:", e)
+            try:
+                new_string = new_string.replace("\n", "\\n")
+                json.loads(new_string)
+                return new_string
+            except Exception as e:
+                print("fix_llm_json_str failed 3:", e)
+                
+                messages = [{
+                    "role": "system",
+                    "content": """Do not change the specific content, fix the json, directly return the repaired JSON, without any explanation and dialogue.
+                    ```
+                    """+new_string+"""
+                    ```"""
+                }]
+
+                message = llm_inference(messages)
+                pattern = r'```json(.*?)```'
+                match = re.findall(pattern, message, re.DOTALL)
+                if match:
+                    return match[-1]
+
+                return message
+
+return_json_prompt = """\n\nYou should only directly respond in JSON format without explian as described below, that must be parsed by Python json.loads.
+Response JSON schema: \n"""
+
+
+# def prompt_call(prompt_template, variables, json_schema=None):
+#     from jinja2 import Template
+#     import json
+#     prompt = Template(prompt_template).render(**variables)
+#     if json_schema is not None:
+#         prompt += return_json_prompt + json_schema
+#         result = llm_inference([{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'system', 'content': prompt}], model_type='smart')
+#         return json.loads(fix_llm_json_str(result))
+#     else:
+#         result = llm_inference([{'role': 'system', 'content': prompt}], model_type='smart')
+
+if __name__ == '__main__':
+    test_embedding_single()

GeneralAgent/skills/memory_utils.py

@@ -0,0 +1,153 @@
+
+
+def _parse_segment_llm_result(text):
+    import logging
+    lines = text.strip().split('\n')
+    key = None
+    nodes = {}
+    # print(lines)
+    for line in lines:
+        # print(line)
+        line = line.strip()
+        if len(line) == 0:
+            continue
+        if line.startswith('<<') and line.endswith('>>'):
+            key = line[2:-2]
+        else:
+            if key is None:
+                logging.warning(f'key is None, line: {line}')
+                continue
+            blocks = line.split(':')
+            if len(blocks) >= 2:
+                start = int(blocks[0])
+                end = int(blocks[1])
+                nodes[key] = (start, end)
+    return nodes
+
+
+async def segment_text(text):
+    """
+    将文本进行语义分段，返回分段后的文本和key组成的字典nodes
+    """
+    from GeneralAgent import skills
+    from jinja2 import Template
+    segment_prompt = """
+---------
+{{text}}
+---------
+
+For the text enclosed by ---------, the number following # is the line number.
+Your task is to divide the text into segments (up to 6), each represented by the start and end line numbers. Additionally, assign a brief title (not exceeding 10 words) to each segment.
+The output format is as follows:
+```
+<<Title for Segment>>
+Start_line: End_line
+
+<<Title for Segment>>
+Start_line: End_line
+```
+For instance:
+```
+<<Hello>>
+0:12
+
+<<World>>
+13:20
+```
+Please note, each title should not exceed 10 words. Titles exceeding this limit will be considered invalid. Strive to keep your titles concise yet reflective of the main content in the segment.
+"""
+    lines = text.strip().split('\n')
+    new_lines = []
+    for index in range(len(lines)):
+        new_lines.append(f'#{index} {lines[index]}')
+    new_text = '\n'.join(new_lines)
+    prompt = Template(segment_prompt).render({'text': new_text})
+    messages = [
+        {'role': 'system','content': 'You are a helpful assistant'},
+        {'role': 'user','content': prompt}
+        ]
+    model_type='normal'
+    if skills.messages_token_count(messages) > 3500:
+        model_type = 'long'
+    result = await skills.async_llm_inference(messages, model_type)
+    # print(result)
+    nodes = _parse_segment_llm_result(result)
+    for key in nodes:
+        start, end = nodes[key]
+        nodes[key] = '\n'.join(lines[start:end])
+    return nodes
+
+
+async def summarize_text(text):
+    from GeneralAgent import skills
+    prompt = "Please distill the content between --------- into a concise phrase or sentence that captures the essence without any introductory phrases."
+    # prompt = "请将---------之间的内容提炼成一个简洁的短语或句子，抓住要点，无需任何介绍性短语。"
+    messages = [
+        {'role': 'system','content': 'You are a helpful assistant'},
+        {'role': 'user','content': f'{prompt}.\n---------\n{text}\n---------'}
+        ]
+    result = await skills.async_llm_inference(messages)
+    return result
+
+async def extract_info(background, task):
+    prompt_template = """
+Background (line number is indicated by #number, and <<title>> is a link to the details):
+---------
+{{background}}
+---------
+
+Task
+---------
+{{task}}
+---------
+
+Please provide the line numbers in the background that contain information relevant to solving the task.
+Then, provide the <<titles>> that provide further details related to the background information.
+The expected output format is as follows:
+```
+#Line Number 1
+#Line Number 2
+...
+<<title 1>>
+<<title 2>>
+...
+```
+If no relevant information is found, please output "[Nothing]".
+```
+[Nothing]
+```
+"""
+
+    from GeneralAgent import skills
+    from jinja2 import Template
+    prompt = Template(prompt_template).render({'background': background, 'task': task})
+    messages = [
+        {'role': 'system','content': 'You are a helpful assistant'},
+        {'role': 'user','content': prompt}
+        ]
+    result = await skills.async_llm_inference(messages)
+    return result
+
+
+def parse_extract_info(text):
+    import re
+    numbers = re.findall(r'#(\d+)', text)
+    numbers = [int(x) for x in numbers]
+    titles = re.findall(r'<<([^>>]+)>>', text)
+    return numbers, titles
+
+
+def extract_title(text):
+    """
+    extract title from text
+    """
+    if len(text) > 500:
+        text = text[:500]
+    prompt = "Please distill the content between --------- into a concise title of the content, less than five words.\n---------\n" + text + "\n---------"
+    from GeneralAgent import skills
+    messages = [
+        {'role': 'system','content': 'You are a helpful assistant'},
+        {'role': 'user','content': prompt}
+        ]
+    result = skills.llm_inference(messages)
+    return result

GeneralAgent/skills/merge_video_audio/merge_video_audio.py

@@ -0,0 +1,67 @@
+def merge_video_audio(video_path:str, narration_path:str=None, music_path:str=None) -> str:
+    """
+    Merge video, narration, and background music into a final video based on the shortest length among all elements.
+    Parameters: video_path -- path of the video file, string
+                narration_path -- path of the narration audio file, string, can be None
+                music_path -- path of the background music file, string, can be None
+    Returns: the path of the final video file, string
+    """
+    from moviepy.editor import VideoFileClip, AudioFileClip, CompositeAudioClip
+
+    # Load video file
+    video = VideoFileClip(video_path)
+
+    # Load audio files if they exist
+    audio_clips = []
+    durations = [video.duration]
+    if narration_path is not None:
+        narration = AudioFileClip(narration_path)
+        audio_clips.append(narration)
+        durations.append(narration.duration)
+    if music_path is not None:
+        music = AudioFileClip(music_path)
+        audio_clips.append(music)
+        durations.append(music.duration)
+
+    # Determine base length
+    base_length = min(durations)
+
+    # Adjust lengths of elements
+    audio_clips = [clip.subclip(0, base_length) for clip in audio_clips]
+
+    # Merge audio files
+    if len(audio_clips) == 1:
+        final_audio = audio_clips[0]
+    elif audio_clips:
+        final_audio = CompositeAudioClip(audio_clips)
+    else:
+        final_audio = None
+
+    # Set audio of video file to final audio
+    final_video = video.subclip(0, base_length)
+    if final_audio is not None:
+        final_video = final_video.set_audio(final_audio)
+
+    # Save final video file
+    from GeneralAgent import skills
+    final_video_path = skills.unique_name() + ".mp4"
+    final_video.write_videofile(final_video_path)
+
+    return final_video_path
+
+
+def test_merge_video_audio():
+    """
+    Test merge_video_audio function
+    """
+    import os
+    video_path = os.path.join(os.path.dirname(__file__), "video.mp4")
+    narration_path = os.path.join(os.path.dirname(__file__), "narration.mp3")
+    music_path = os.path.join(os.path.dirname(__file__), "music.wav")
+    final_video_path = merge_video_audio(video_path, narration_path, music_path)
+    assert os.path.exists(final_video_path)
+    os.remove(final_video_path)
+    # os.remove("./final_audio.mp3")
+
+if __name__ == '__main__':
+    test_merge_video_audio()

GeneralAgent/skills/merge_video_audio/music.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27e302b1c36a4feb0a89f31f1cc7981d20356305396edf08fada7c36d57ed3a7
+size 3840078

GeneralAgent/skills/musicgen/generate_music.py

@@ -0,0 +1,32 @@
+def generate_music(prompt: str, model_version: str = 'stereo-melody-large', duration: int = 8, output_format: str = 'wav') -> str:
+    """
+    Generate music according to the prompt and return the music file path. prompt should be english.
+    @param prompt: A description of the music you want to generate.
+    @param model_version: Model to use for generation. Default is 'stereo-melody-large'.
+    @param duration: Duration of the generated audio in seconds. Default is 8.
+    @param output_format: Output format for generated audio. Default is 'wav'.
+    """
+    import replicate
+    from GeneralAgent import skills
+
+    output = replicate.run(
+        "meta/musicgen:7be0f12c54a8d033a0fbd14418c9af98962da9a86f5ff7811f9b3423a1f0b7d7",
+        input={
+            "model_version": model_version,
+            "prompt": prompt,
+            "duration": duration,
+            "output_format": output_format
+        }
+    )
+    music_url = output
+    music_path = skills.try_download_file(music_url)
+    print(f'Music created at {music_path}')
+    return music_path
+
+def test_generate_music():
+    import os
+    music_path = generate_music('Happy birthday song', 'stereo-melody-large', 10, 'mp3')
+    assert os.path.exists(music_path), "Music file does not exist."
+
+if __name__ == '__main__':
+    test_generate_music()

GeneralAgent/skills/python_envs.py

@@ -0,0 +1,130 @@
+
+def get_current_env_python_libs() -> str:
+    """
+    Return the python libs that installed in current env
+    """
+    import os
+    requirements_path = os.path.join(os.path.dirname(__file__), '../requirements.txt')
+    with open(requirements_path, 'r', encoding='utf-8') as f:
+        requirements = f.read()
+        requirements = requirements.replace('\n', ' ')
+        return requirements.strip()
+    
+
+def get_python_version() -> str:
+    """
+    Return the python version, like "3.9.12"
+    """
+    import platform
+    python_version = platform.python_version()
+    return python_version
+
+def get_os_version() -> str:
+    import platform
+    system = platform.system()
+    if system == 'Windows':
+        version = platform.version()
+        return f"Windows version: {version}"
+    elif system == 'Darwin':
+        version = platform.mac_ver()[0]
+        return f"macOS version: {version}"
+    elif system == 'Linux':
+        version = platform.platform()
+        return f"Linux version: {version}"
+    else:
+        return "Unknown system"
+
+def get_python_code(content:str) -> str:
+    """
+    Return the python code text from content
+    """
+    template = '```python\n(.*?)\n```'
+    import re
+    code = re.findall(template, content, re.S)
+    if len(code) > 0:
+        return code[0]
+    else:
+        return content
+    
+def test_get_python_code():
+    content = """
+```python
+import os
+print(os.getcwd())
+```
+"""
+    assert get_python_code(content) == 'import os\nprint(os.getcwd())'
+
+
+def load_functions_with_path(python_code_path):
+    try:
+        import importlib.util
+        import inspect
+
+        # 指定要加载的文件路径和文件名
+        module_name = 'skills'
+        module_file = python_code_path
+
+        # 使用importlib加载文件
+        spec = importlib.util.spec_from_file_location(module_name, module_file)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+
+        # 获取文件中的所有函数
+        functions = inspect.getmembers(module, inspect.isfunction)
+
+        # 过滤functions中以下划线开头的函数
+        functions = filter(lambda f: not f[0].startswith('_'), functions)
+
+        return [f[1] for f in functions], None
+    except Exception as e:
+        # 代码可能有错误，加载不起来
+        import logging
+        logging.exception(e)
+        return [], str(e)
+    
+def get_function_signature(func, module:str=None):
+    """Returns a description string of function"""
+    import inspect
+    sig = inspect.signature(func)
+    sig_str = str(sig)
+    desc = f"{func.__name__}{sig_str}"
+    if func.__doc__:
+        desc += ': ' + func.__doc__.strip()
+    if module is not None:
+        desc = f'{module}.{desc}'
+    if inspect.iscoroutinefunction(func):
+        desc = "async " + desc
+    return desc
+
+
+def python_line_is_variable_expression(line):
+    """
+    Return True if line is a variable expression, else False
+    """
+    import ast
+    try:
+        tree = ast.parse(line)
+    except SyntaxError:
+        return False
+
+    if len(tree.body) != 1 or not isinstance(tree.body[0], ast.Expr):
+        return False
+
+    expr = tree.body[0].value
+    if isinstance(expr, ast.Call):
+        return False
+
+    return True
+
+
+def test_python_line_is_variable_expression():
+    assert python_line_is_variable_expression('a')       
+    assert python_line_is_variable_expression('a, b')       
+    assert python_line_is_variable_expression('a + b')   
+    assert python_line_is_variable_expression('vars[0]') 
+    assert python_line_is_variable_expression('scrape_web("https://www.baidu.com")[0]') 
+
+    assert python_line_is_variable_expression(' vars[0]') is False 
+    assert python_line_is_variable_expression('print(a)') is False 
+    assert python_line_is_variable_expression('x = a + b') is False 

GeneralAgent/skills/replicate_api.py

@@ -0,0 +1,75 @@
+# replicate api
+
+def _replicate_image_generation(prompt):
+    """generate a image with prompt (in english), return a image url"""
+
+    # import replicate
+    # output = replicate.run(
+    #     "cjwbw/taiyi-stable-diffusion-1b-chinese-v0.1:36c580142e9fbbd52e5e678e30541c0da6f2021c9d2039a5c00be192a010e8c5",
+    #     input={"prompt": prompt}
+    # )
+    # print(output)
+
+    # 图片生成切换成为sdxl的version 1 版本
+    import replicate
+
+    output = replicate.run(
+        # "stability-ai/stable-diffusion:ac732df83cea7fff18b8472768c88ad041fa750ff7682a21affe81863cbe77e4",
+        "stability-ai/sdxl:2f779eb9b23b34fe171f8eaa021b8261566f0d2c10cd2674063e7dbcd351509e",
+        input={"prompt": prompt}
+    )
+    image_url = output[0]
+    # print(image_url)
+
+    return image_url
+
+
+def image_generation(prompt) -> str:
+    """
+    Generate an image according to the prompt and return the image path. For example, when the prompt is "apple" you will get an image of an apple. Note: The prompt should describe objective things in detail, not abstract concepts. For example, if you want to draw a picture of Chengdu, the prompt should be "Picture of Chengdu, with giant pandas playing in the bamboo forest, people eating hot pot around, and a Jinsha Sunbird totem next to it" instead of "Draw a picture of Chengdu" "painting"
+    @param prompt: The prompt should be detailed enough to describe the image. Tips can be in any type of language, but English is recommended.
+    """
+    from GeneralAgent import skills
+    if not skills.text_is_english(prompt):
+        prompt = skills.translate_text(prompt, 'english')
+    image_url = _replicate_image_generation(prompt)
+    image_path  = skills.try_download_file(image_url)
+    print(f'image created at ![{image_path}]({image_path})')
+    return image_path
+
+def face_restoration(image_path):
+    """ Practical face restoration algorithm for old photos or AI-generated faces. input image path, and return the new image path"""
+    import replicate
+    from GeneralAgent import skills
+    image_url = replicate.run(
+        "tencentarc/gfpgan:9283608cc6b7be6b65a8e44983db012355fde4132009bf99d976b2f0896856a3",
+        input={"img": open(image_path, "rb")}
+    )
+    new_image_path  = skills.try_download_file(image_url)
+    return new_image_path
+
+def qrcode_stable_diffusion(prompt, qr_code_content):
+    """generate a qrcode image with prompt, return a image url"""
+    import replicate
+    output = replicate.run(
+        "nateraw/qrcode-stable-diffusion:9cdabf8f8a991351960c7ce2105de2909514b40bd27ac202dba57935b07d29d4",
+        input={"prompt": prompt, 'qr_code_content': qr_code_content}
+    )
+    return output[0]
+
+def speech_to_text(audio_file_path):
+    """Convert speech in audio to text, return a text and the language of the text"""
+    import replicate
+    output = replicate.run(
+        "openai/whisper:91ee9c0c3df30478510ff8c8a3a545add1ad0259ad3a9f78fba57fbc05ee64f7",
+        input={"audio": open(audio_file_path, "rb")}
+    )
+    print(output)
+    language = output['detected_language']
+    text = output['transcription']
+
+    return text, language
+
+
+if __name__ == '__main__':
+    image_generation('a cat')

GeneralAgent/skills/scrape_dynamic_web.py

@@ -0,0 +1,70 @@
+def scrape_dynamic_web(url):
+    """
+    This function takes a url and returns the text content of the web page.
+    It uses selenium to load the dynamic content of the page, and BeautifulSoup to parse the HTML and extract the text.
+    It also replaces <span> tags with their text content, and <br> tags with newline characters.
+    """
+    from selenium import webdriver
+    from selenium.webdriver.chrome.service import Service
+    from webdriver_manager.chrome import ChromeDriverManager
+    from selenium.webdriver.common.by import By
+    from selenium.webdriver.chrome.options import Options
+    from selenium.webdriver.common.action_chains import ActionChains
+    from selenium.webdriver.support.ui import WebDriverWait
+    from selenium.webdriver.support import expected_conditions as EC
+    from selenium.webdriver.common.by import By
+    from bs4 import BeautifulSoup
+    import time
+    import re
+    from urllib.parse import urljoin
+
+    # Setup chrome options
+    chrome_options = Options()
+    chrome_options.add_argument("--headless") # Ensure GUI is off
+    chrome_options.add_argument("--no-sandbox")
+    chrome_options.add_argument("--disable-dev-shm-usage")
+
+    # Set path to chromedriver as per your configuration
+    webdriver_service = Service(ChromeDriverManager().install())
+
+    # Choose Chrome Browser
+    driver = webdriver.Chrome(service=webdriver_service, options=chrome_options)
+    driver.get(url)
+
+    # Wait for the dynamic content to load
+    WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.TAG_NAME, 'body')))
+
+    html = driver.page_source
+    driver.quit()
+
+    # Parse html content
+    soup = BeautifulSoup(html, "html.parser")
+    for span in soup.find_all("span"):
+        span.replace_with(span.text)
+    for a in soup.find_all("a"):
+        href = urljoin(url, a.get('href'))
+        a.replace_with(f"[{a.text}]({href})")
+    for br in soup.find_all("br"):
+        br.replace_with("\n")
+    text = soup.get_text(separator="\n")
+
+    # Replace multiple newlines and spaces around them with a single newline
+    text = re.sub('\s*\n\s*', '\n', text)
+
+    # Collapse whitespace
+    text = ' '.join(text.split())
+
+    return text
+
+def test_scrape_dynamic_web():
+    """
+    This function tests the scrape_dynamic_web function.
+    It asserts that the returned text contains the string 'replicate'.
+    """
+    url = "https://replicate.com/stability-ai/stable-video-diffusion/api?tab=python"
+    text = scrape_dynamic_web(url)
+    assert 'replicate' in text
+    print(text)
+
+if __name__ == '__main__':
+    test_scrape_dynamic_web()

GeneralAgent/skills/split_text.py

@@ -0,0 +1,28 @@
+
+
+def split_text(text, max_token=3000, separators='\n'):
+    """
+    Split the text into paragraphs, each paragraph has less than max_token tokens.
+    """
+    import re
+    from GeneralAgent import skills
+    pattern = "[" + re.escape(separators) + "]"
+    paragraphs = list(re.split(pattern, text))
+    print(len(paragraphs))
+    result = []
+    current = ''
+    for paragraph in paragraphs:
+        if skills.string_token_count(current) + skills.string_token_count(paragraph) > max_token:
+            result.append(current)
+            current = ''
+        current += paragraph + '\n'
+    if len(current) > 0:
+        result.append(current)
+    new_result = []
+    for x in result:
+        if skills.string_token_count(x) > max_token:
+            new_result.extend(split_text(x, max_token=max_token, separators="，。,.;；"))
+        else:
+            new_result.append(x)
+    new_result = [x.strip() for x in new_result if len(x.strip()) > 0]
+    return new_result