update

README.md

@@ -1,13 +1,24 @@
 ---
-title: Rubra V0.1
-emoji: 💬
-colorFrom: yellow
-colorTo: purple
+title: Rubra v0.1
+emoji: 🦙
+colorFrom: indigo
+colorTo: pink
 sdk: gradio
-sdk_version: 4.36.1
-app_file: app.py
+sdk_version: 4.37.1
 pinned: false
 license: apache-2.0
+thumbnail: https://rubra.ai/
+suggested_hardware: a10g-large
+# preload_from_hub:
+# - rubra-ai/Meta-Llama-3-8B-Instruct
+# - rubra-ai/Phi-3-mini-128k-instruct
+# - rubra-ai/Mistral-7B-Instruct-v0.3
+# - rubra-ai/Mistral-7B-Instruct-v0.2
+# - rubra-ai/gemma-1.1-2b-it
+# - rubra-ai/Qwen2-7B-Instruct
 ---
 
-An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.22.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).
+# Rubra v0.1 - A Collection of Tool (Function) Calling LLMs
+
+This Space demonstrates Rubra tool calling models. Please, check the original model cards for details.
+

app.py

@@ -1,63 +1,345 @@
+import os
+from threading import Thread
+from typing import Iterator
+import json
+
 import gradio as gr
-from huggingface_hub import InferenceClient
+import spaces
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+import subprocess
+import copy
+
+import subprocess
+import sys
+
+def run_command(command):
+    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+    output, error = process.communicate()
+    if process.returncode != 0:
+        print(f"Error executing command: {command}")
+        print(f"Error message: {error.decode('utf-8')}")
+        sys.exit(1)
+    return output.decode('utf-8')
+
+MAX_MAX_NEW_TOKENS = 2048
+DEFAULT_MAX_NEW_TOKENS = 1024
+MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "8000"))
+model_choices = [
+    "rubra-ai/Meta-Llama-3-8B-Instruct",
+    "rubra-ai/Qwen2-7B-Instruct",
+    "rubra-ai/Phi-3-mini-128k-instruct",
+    "rubra-ai/Mistral-7B-Instruct-v0.3",
+    "rubra-ai/Mistral-7B-Instruct-v0.2",
+    "rubra-ai/gemma-1.1-2b-it"
+]
+
+DESCRIPTION = """\
+# Rubra v0.1 - Top LLMs enhanced with function (tool) calling
+
+This is a demo of the Rubra collection of models. You can use the models for general conversation, 
+task completion, and function calling with the provided tools input.
 
 """
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
+
+LICENSE = """
+<p/>
+
+---
+Rubra code is licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Rubra models are licensed under the parent model's license. See the parent model card for more information.
 """
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
 
+if not torch.cuda.is_available():
+    DESCRIPTION += "\n<p>Running on CPU 🥶 This demo does not work on CPU.</p>"
+
+if torch.cuda.is_available():
+    model_id = "sanjay920/Llama-3-8b-function-calling-alpha-v1"  # Default model
+    model = None
+    tokenizer = None
+
+    def load_model(model_name):
+        global model, tokenizer
+        model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", load_in_4bit=False)
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        tokenizer.use_default_system_prompt = False
+        model.generation_config.pad_token_id = tokenizer.pad_token_id
+
+    load_model(model_id)  # Load the default model
+
+def is_valid_json(tools: str) -> bool:
+    try:
+        json.loads(tools)
+        return True
+    except ValueError:
+        return False
+
+def validate_tools(tools):
+    if tools.strip() == "" or is_valid_json(tools):
+        return gr.update(visible=False)
+    else:
+        return gr.update(visible=True)
 
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
+def json_to_markdown(json_obj):
+    """Convert a JSON object to a formatted markdown string."""
+    markdown = ""
+    for item in json_obj:
+        if item.get("type") == "text":
+            # For text items, just add the text content
+            markdown += item.get("text", "") + "\n\n"
+        elif item.get("type") == "function":
+            # For function calls, format as JSON
+            markdown += "```json\n"
+            # markdown += json.dumps(item.get("function", {}), indent=2)
+            markdown += json.dumps(item, indent=2)
+            markdown += "\n```\n\n"
+    return markdown.strip()
 
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
+def user(user_message, history):
+    return "", history + [[user_message, None]]
 
-    messages.append({"role": "user", "content": message})
+def bot(history, system_prompt, tools, role, max_new_tokens, temperature):
+    user_message = history[-1][0]
+    if history[-1][1] is None:
+        history[-1][1] = ""  # Ensure it's never None
 
-    response = ""
+    ui_history = list(history)  # Clone the history for UI updates
+    all_tool_outputs = []  # Store all processed outputs for final aggregation
+    output_accumulated = ""  # To accumulate outputs before processing
 
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
+    for chunk in generate(user_message, history[:-1], system_prompt, tools, role, max_new_tokens, temperature):
+        history[-1][1] += chunk
+        print(history[-1][1])
+        
+        if "endtoolcall" in history[-1][1]:
+            process_output = postprocess_output(history[-1][1])
+            print("process output:\n", process_output)
+            if process_output:
+                temp_history = copy.deepcopy(history)  # Use deepcopy here
+                if isinstance(process_output, list) and len(process_output) > 0 and isinstance(process_output[0], dict):
+                    markdown_output = json_to_markdown(process_output)
+                    temp_history[-1][1] = markdown_output
+                else:
+                    temp_history[-1][1] = str(process_output)
+                print(temp_history[-1][1])
+                print("--------------------------")
+                yield temp_history
+            else:
+                print(history[-1][1])
+                print("--------------------------")
+                yield history
+        else:
+            print(history[-1][1])
+            print("--------------------------")
+            yield history
+
+@spaces.GPU
+def generate(
+    message: str,
+    chat_history: list[tuple[str, str]],
+    system_prompt: str,
+    tools: str,
+    role: str,
+    max_new_tokens: int = 1024,
+    temperature: float = 0.6,
+) -> Iterator[str]:
+    global model, tokenizer
+    conversation = []
+    if system_prompt:
+        conversation.append({"role": "system", "content": system_prompt})
+    for user, assistant in chat_history:
+        conversation.extend([{"role": "user", "content": user}, {"role": "assistant", "content": assistant}])
+    conversation.append({"role": role, "content": message})
+    
+    if tools:
+        if not is_valid_json(tools):
+            yield "Invalid JSON in tools. Please correct it."
+            return
+        tools = json.loads(tools)
+        formatted_msgs = preprocess_input(msgs=conversation, tools=tools)
+    else:
+        formatted_msgs = conversation
+
+    input_ids = tokenizer.apply_chat_template(formatted_msgs, return_tensors="pt")
+    if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
+        input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
+        gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
+    input_ids = input_ids.to(model.device)
+
+    streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
+    generate_kwargs = dict(
+        input_ids=input_ids,
+        streamer=streamer,
+        max_new_tokens=max_new_tokens,
+        do_sample=True,
+        top_p=0.95,
         temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
+        num_beams=1,
+        repetition_penalty=1.2,
+    )
+    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t.start()
 
-        response += token
-        yield response
+    for text in streamer:
+        # print("Generated text:", text)
+        yield text
 
+bot_message = """Hello! How can I assist you today? If you have any questions or need information on a specific topic, feel free to ask. I can also utilize `tools` that you input to help you better. For example:
+```
+[
+    {
+        "type": "function",
+        "function": {
+            "name": "get_current_weather",
+            "description": "Get the current weather in a given location",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "location": {
+                        "type": "string",
+                        "description": "Must include the city AND state, e.g. 'San Francisco, CA'"
+                    },
+                    "unit": {
+                        "type": "string",
+                        "enum":
+                        ["celsius", "fahrenheit"]
+                    }
+                },
+                "required": ["location"]
+            }
+        }
+    }
+]
+```
+
+You can also define `functions` (deprecated in favor of `tools` in OpenAI):
+```
+[
+    {
+        "name": "get_current_date",
+        "description": "Gets the current date at the given location. Results are in ISO 8601 date format; e.g. 2024-04-25",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "location": {
+                    "type": "string",
+                    "description": "The city and state to get the current date at, e.g. San Francisco, CA"
+                }
+            },
+            "required":["location"]
+        }
+    }
+]
+```
 """
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
-)
 
+def create_chat_interface():
+    with gr.Blocks(css="style.css") as demo:
+        gr.Markdown(DESCRIPTION)
+        
+        with gr.Row(equal_height=True, elem_id="main-row"):
+            with gr.Column(scale=3, min_width=500):
+                # Initialize the chatbot with the welcome message
+                chatbot = gr.Chatbot(
+                    value=[("Hi", bot_message)],
+                    show_copy_button=True,
+                    elem_id="chatbot",
+                    show_label=False,
+                    render_markdown=True,
+                    height="100%",
+                    layout='bubble',
+                    avatar_images=("human.png", "bot.png")
+                )
+                
+                error_box = gr.Markdown(visible=False, elem_id="error-box")
+
+            with gr.Column(scale=2, min_width=300):
+                model_dropdown = gr.Dropdown(
+                    choices=model_choices,
+                    label="Select Model",
+                    value="sanjay920/Llama-3-8b-function-calling-alpha-v1"
+                )
+                model_dropdown.change(load_model, inputs=[model_dropdown])
+                
+                with gr.Accordion("Settings", open=False):
+                    max_new_tokens = gr.Slider(
+                        label="Max new tokens",
+                        minimum=1,
+                        maximum=MAX_MAX_NEW_TOKENS,
+                        step=1,
+                        value=DEFAULT_MAX_NEW_TOKENS,
+                    )
+                    temperature = gr.Slider(
+                        label="Temperature",
+                        minimum=0.0,
+                        maximum=1.2,
+                        step=0.01,
+                        value=0.01,
+                    )
+
+                with gr.Row():
+                    role = gr.Dropdown(choices=["user", "observation"], value="user", label="Role", scale=4)
+                system_prompt = gr.Textbox(label="System Prompt", lines=1, info="Optional")
+                tools = gr.Textbox(label="Tools", lines=1, placeholder="Enter tools in JSON format", info="Optional")
+                
+                
+                
+                with gr.Row():
+                    user_input = gr.Textbox(
+                        label="User Input", 
+                        placeholder="Type your message here...",
+                        show_label=True,
+                        scale=8
+                    )
+                submit_btn = gr.Button("Submit", variant="primary", elem_id="submit-button")
+                clear_btn = gr.Button("Clear Conversation", elem_id="clear-button")
+                
+
+        tools.change(validate_tools, tools, error_box)
+
+        submit_btn.click(
+            user,
+            [user_input, chatbot],
+            [user_input, chatbot],
+            queue=False
+        ).then(
+            bot,
+            [chatbot, system_prompt, tools, role, max_new_tokens, temperature],
+            chatbot
+        )
+
+        clear_btn.click(lambda: ([], None), outputs=[chatbot, error_box])
+
+        gr.Markdown(LICENSE)
+
+    return demo
 
 if __name__ == "__main__":
-    demo.launch()
+    # Initialize npm project if package.json doesn't exist
+    if not os.path.exists('package.json'):
+        print("Initializing npm project...")
+        run_command("npm init -y")
+
+    # Install jsonrepair locally
+    print("Installing jsonrepair...")
+    run_command("npm install jsonrepair")
+
+    # Verify installation
+    print("Verifying jsonrepair installation:")
+    run_command("npm list jsonrepair")
+
+    # Add node_modules/.bin to PATH
+    os.environ['PATH'] = f"{os.path.join(os.getcwd(), 'node_modules', '.bin')}:{os.environ['PATH']}"
+    from preprocess import preprocess_input
+    from postprocess import postprocess_output
+    demo = create_chat_interface()
+    demo.queue(max_size=20).launch()

install_node.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# installs nvm (Node Version Manager)
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
+
+source ~/.nvm/nvm.sh
+
+# download and install Node.js (you may need to restart the terminal)
+nvm install 20
+
+# verifies the right Node.js version is in the environment
+~/.nvm/versions/node/v20.15.0/bin/node -v # should print `v20.15.0`
+
+# verifies the right NPM version is in the environment
+~/.nvm/versions/node/v20.15.0/bin/npm -v # should print `10.7.0`
+
+~/.nvm/versions/node/v20.15.0/bin/npm install jsonrepair -g

packages.txt

@@ -0,0 +1,2 @@
+nodejs
+npm

postprocess.py

@@ -0,0 +1,156 @@
+import json
+import uuid
+import re
+from typing import List
+import subprocess
+import sys
+
+def install(package):
+    subprocess.check_call([sys.executable, "-m", "pip", "install", package])
+
+try:
+    import pythonmonkey
+except ImportError:
+    install('pythonmonkey')
+    import pythonmonkey
+
+# Your code using pythonmonkey
+
+# Assuming jsonrepair is accessible
+jsonrepair = pythonmonkey.require('jsonrepair').jsonrepair
+
+def clean_command_string(command_str):
+    cleaned_command = re.sub(r'\\(?!["\\/bfnrt]|u[a-fA-F0-9]{4})', '', command_str)
+    cleaned_command = cleaned_command.replace('\\"', '"')
+    if cleaned_command.startswith('"') and cleaned_command.endswith('"'):
+        cleaned_command = cleaned_command[1:-1]
+    return cleaned_command
+
+def parse_json_safely(json_str):
+    try:
+        return json.loads(json_str)
+    except json.JSONDecodeError:
+        try:
+            repaired = jsonrepair(json_str)
+            return json.loads(repaired)
+        except Exception:
+            return json_str
+
+def clean_json_object(obj):
+    if isinstance(obj, dict):
+        return {k: clean_json_object(v) for k, v in obj.items()}
+    elif isinstance(obj, list):
+        return [clean_json_object(item) for item in obj]
+    elif isinstance(obj, str):
+        cleaned = clean_command_string(obj)
+        return parse_json_safely(cleaned) if cleaned.startswith('{') or cleaned.startswith('[') else cleaned
+    else:
+        return obj
+
+def extract_tool_calls(output_str):
+    # Pattern to capture everything after 'starttoolcall' until 'endtoolcall' or end of string if 'endtoolcall' isn't present
+    pattern = r'starttoolcall(.*?)(?:endtoolcall|$)'
+    matches = [match for match in re.findall(pattern, output_str, re.DOTALL)]
+    return matches
+
+def extract_tool_calls_and_text(output_str):
+    # Initialize an empty list to collect all segments
+    segments = []
+
+    # Last index processed in the string
+    last_end = 0
+
+    # Pattern to capture everything after 'starttoolcall' until 'endtoolcall' or end of string if 'endtoolcall' isn't present
+    pattern = r'(starttoolcall(.*?)(?:endtoolcall|$))'
+    for match in re.finditer(pattern, output_str, re.DOTALL):
+        start, end = match.span(1)
+        
+        # Capture any text between the end of the last tool call and the start of the current one
+        if start > last_end:
+            text_between = output_str[last_end:start].strip()
+            if text_between:
+                segments.append({"text": text_between, "type": "text"})
+        
+        # Append the current tool call to the list
+        tool_call_content = match.group(2).strip()
+        segments.append({"tool_call": tool_call_content, "type": "function"})
+
+        # Update the last processed index
+        last_end = end
+
+    # Check if there is any remaining text after the last tool call
+    if last_end < len(output_str):
+        remaining_text = output_str[last_end:].strip()
+        if remaining_text:
+            segments.append({"text": remaining_text, "type": "text"})
+
+    return segments
+
+def postprocess_output(output_str: str):
+    segments = extract_tool_calls_and_text(output_str)
+    results = []
+
+    for segment in segments:
+        print("processing segment")
+        print(segment)
+        if segment['type'] == 'function':
+            call = segment['tool_call']
+            try:
+                parsed_call = parse_json_safely(call)
+                cleaned_call = clean_json_object(parsed_call)
+
+                if isinstance(cleaned_call, dict) and 'name' in cleaned_call and 'arguments' in cleaned_call:
+                    if isinstance(cleaned_call.get('arguments'), dict):
+                        cleaned_call['arguments'] = json.dumps(cleaned_call['arguments'])
+                    results.append({
+                        "id": uuid.uuid4().hex[:8],
+                        "function": cleaned_call,
+                        "type": "function",
+                    })
+                else:
+                    results.append({
+                        "id": uuid.uuid4().hex[:8],
+                        "text": call,
+                        "type": "text",
+                    })
+            except Exception as e:
+                results.append({
+                    "id": uuid.uuid4().hex[:8],
+                    "text": call,
+                    "type": "text",
+                })
+        else:
+            results.append({
+                "id": uuid.uuid4().hex[:8],
+                "text": segment['text'],
+                "type": "text",
+            })
+
+    return results
+
+def json_to_markdown(json_obj):
+    """Convert a JSON object to a formatted markdown string."""
+    markdown = ""
+    for item in json_obj:
+        if item.get("type") == "text":
+            # For text items, just add the text content
+            markdown += item.get("text", "") + "\n\n"
+        elif item.get("type") == "function":
+            # For function calls, format as JSON
+            markdown += "```json\n"
+            markdown += json.dumps(item.get("function", {}), indent=2)
+            markdown += "\n```\n\n"
+    return markdown.strip()
+
+if __name__ == "__main__":
+    # Test the function with a sample input
+    # output_str = '''Some text before starttoolcall{"name": "funcA", "arguments": {"param1": 1}endtoolcall
+    # More text starttoolcall{"name": "funcB", "arguments": {"param2": "test"}}endtoolcall'''
+    
+    # output_str = '''starttoolcall{"name": "get_current_weather", "arguments": {"location": "San Francisco", "unit": "celsius"}}endtoolcall starttoolcall{"name": "get_current_weather", "arguments": {"location": "Tokyo", "unit": "celsius"}}endtoolcall okay great '''
+    output_str = '''starttoolcall{"name": "get_current_weather", "arguments": {"location": "San Francisco", "unit": "celsius"}}endtoolcall starttoolcall{"name": "get_current_weather", "arguments": {"location": "Tokyo", "unit": "celsius"}}endtoolcall starttoolcall{"name": "get_current_weather", "arguments": {"location": "Paris", "unit": '''
+    parsed_json = postprocess_output(output_str)
+    print(json.dumps(parsed_json, indent=2))
+
+    print("-----")
+    print(json_to_markdown(parsed_json))

preprocess.py

@@ -0,0 +1,256 @@
+from typing import List
+import json
+
+
+TOOL_SYSTEM_PROMPT_RUBRA = (
+    "You have access to the following tools: {tool_text}\n"
+    "You can choose to respond with one or more tool calls at once, or with a chat message back to the user. "
+    "Ensure you have all necessary details before making tool calls. If additional information is needed, "
+    "ask the user appropriately. Any tool call you make must correspond to the functions listed above.\n"
+    "If you decide to call a tool, format it like this: "
+    'starttoolcall{{"name": "<function_name>", "arguments": {{"<arg1_name>": "<arg1_value>", "<arg2_name>": "<arg2_value>", ...}}}}endtoolcall '
+    "where the JSON wrapped between starttoolcall and endtoolcall represents the function call.\n"
+)
+
+def json_schema_to_typescript_type(schema, param_name):
+    ts_type = "any"  # default type
+    enum_comment = ""
+    integer_comment = ""
+    description_comment = ""
+
+    if isinstance(schema, dict) and "type" in schema:
+        json_type = schema["type"]
+        if json_type == "array":
+            item_type = (
+                "any"
+                if "items" not in schema
+                else json_schema_to_typescript_type(schema["items"], param_name)[0]
+            )
+            ts_type = f"{item_type}[]"
+        elif json_type == "number":
+            ts_type = "number"
+        elif json_type == "integer":
+            ts_type = (
+                "number"  # TypeScript doesn't differentiate between number and integer
+            )
+            integer_comment = f" * @param {param_name} - Integer"
+        elif json_type == "object":
+            ts_type, _ = generate_typescript_interface(schema, param_name)
+        elif json_type == "boolean":
+            ts_type = "boolean"
+        elif json_type == "null":
+            ts_type = "null"
+        elif json_type == "string":
+            ts_type = "string"
+
+    if "enum" in schema:
+        enum_comment = f" * @enum {param_name} - Possible values: " + ", ".join(
+            [f'"{enum_value}"' for enum_value in schema["enum"]]
+        )
+        ts_type = "string"
+    if "description" in schema:
+        description_comment = f' * @param {param_name} - {schema["description"]}'
+
+    # Return only the type for nested objects to avoid duplicating comments
+    if isinstance(schema, dict) and schema.get("type") == "object":
+        return ts_type, "", "", ""
+
+    return ts_type, enum_comment, integer_comment, description_comment
+
+
+def generate_typescript_interface(schema, interface_name):
+    properties = schema.get("properties", {})
+    required = schema.get("required", [])
+
+    interface_body = []
+    descriptions = []
+    for prop_name, prop_schema in properties.items():
+        prop_type, enum_comment, integer_comment, description_comment = (
+            json_schema_to_typescript_type(prop_schema, prop_name)
+        )
+        is_optional = prop_name not in required
+        interface_body.append(
+            f'    {prop_name}{"?" if is_optional else ""}: {prop_type};'
+        )
+        if description_comment:
+            descriptions.append(description_comment)
+        if enum_comment:
+            descriptions.append(enum_comment)
+        if integer_comment:
+            descriptions.append(integer_comment)
+
+    comments = "\n".join(descriptions)
+    interface_definition = (
+        f"interface {interface_name} {{\n" + "\n".join(interface_body) + "\n}"
+    )
+    return interface_definition, comments
+
+
+def convert_parameters_list_to_dict(parameters):
+    properties = {}
+    required = []
+    for param in parameters:
+        properties[param["name"]] = param
+        if "default" not in param:
+            required.append(param["name"])
+    return {"properties": properties, "required": required}
+
+
+def generate_typescript_function(function_schema) -> str:
+    func_name = function_schema["name"]
+    description = function_schema.get("description", "")
+
+    # Check if parameters is a list and convert if necessary
+    parameters_info = function_schema.get("parameters", {})
+    if isinstance(parameters_info, list):
+        parameters_info = convert_parameters_list_to_dict(parameters_info)
+    if parameters_info is None:
+        parameters_info = {}
+
+    parameters_schema = parameters_info.get("properties", {})
+    required_params = parameters_info.get("required", [])
+
+    args_list = []
+    comments_list = []
+    interfaces = []
+    for param_name, param_schema in parameters_schema.items():
+        ts_type, enum_comment, integer_comment, description_comment = (
+            json_schema_to_typescript_type(param_schema, param_name)
+        )
+        if ts_type.startswith("interface"):
+            interface_definition, nested_comments = generate_typescript_interface(
+                param_schema, f"{func_name}_{param_name.capitalize()}Params"
+            )
+            interfaces.append(interface_definition)
+            comments_list.append(nested_comments)
+            ts_type = f"{func_name}_{param_name.capitalize()}Params"
+        else:
+            if description_comment:
+                comments_list.append(description_comment)
+            if enum_comment:
+                comments_list.append(enum_comment)
+            if integer_comment:
+                comments_list.append(integer_comment)
+        is_optional = param_name not in required_params
+        args_list.append(f'{param_name}{"?" if is_optional else ""}: {ts_type}')
+
+    args_str = ", ".join(args_list)
+    comments_str = "\n".join(comments_list)
+    interfaces_str = "\n\n".join(interfaces)
+
+    description_comment = f" * {description}\n" if description else ""
+    typescript_func_declaration = (
+        "/**\n"
+        + description_comment
+        + (comments_str + "\n" if comments_str else "")
+        + " */\n"
+        + (interfaces_str + "\n\n" if interfaces_str else "")
+        + f"function {func_name}({args_str}): any {{}}"
+    )
+
+    return typescript_func_declaration
+
+
+
+def format_tools(tools: List[dict]) -> str:
+    func_defs = []
+    for t in tools:
+        tool_schema = t["function"] if "function" in t else t
+        func_defs.append(generate_typescript_function(tool_schema))
+    
+    typescript_functions_str = "\n\n".join(func_defs)
+    res = TOOL_SYSTEM_PROMPT_RUBRA.format(tool_text=typescript_functions_str)
+    return res
+
+
+
+def preprocess_input(msgs: List[dict], tools: List[dict]):
+    tool_system_prompt = format_tools(tools)
+    processed_msgs = process_messages(msgs, tool_system_prompt)
+    return processed_msgs
+
+
+def process_messages(messages: List[dict], function_str: str):
+    func_observation_map = {}
+    processed_msg = []
+
+    for i in range(len(messages)):
+        
+        if messages[i]["role"] != "tool" and len(func_observation_map) > 0:
+            # func_observation_array = [f'{k}: {func_observation_map[k] if func_observation_map[k] != "" else "done"}' for k in func_observation_map]
+            func_observation_array = [f'{func_observation_map[k] if func_observation_map[k] != "" else "done"}' for k in func_observation_map]
+            observation_str = json.dumps(func_observation_array)
+            observation_call = {"role": "user", "content": "start observation " + observation_str + " end observation"}
+            processed_msg.append(observation_call)
+            func_observation_map.clear()
+
+        if i == 0:
+            if messages[0]["role"] == "system":
+                old_content = messages[0]["content"]
+                sys_msg = {"role": "system", "content": old_content + "\n" + function_str}
+                processed_msg.append(sys_msg)
+            else:
+                # Insert a system message of tool definition before the first message
+                sys_msg = {"role": "system", "content": "You are a helpful assistant.\n" + function_str}
+                processed_msg.append(sys_msg)
+                processed_msg.append(messages[0]) # first message is always either system or user msg
+
+        elif messages[i]["role"] == "assistant" and "tool_calls" in messages[i]:
+            # Convert OpenAI function call format to Rubra format
+            tool_call_str = construct_tool_call_str(messages[i]["tool_calls"], func_observation_map)
+            function_call = {"role": "assistant", "content": tool_call_str}
+            processed_msg.append(function_call)
+
+        elif messages[i]["role"] == "tool":
+            tool_call_id = messages[i]["tool_call_id"]
+            if tool_call_id in func_observation_map:
+                func_observation_map[tool_call_id] = messages[i]["content"]
+            else:
+                print(func_observation_map)
+                print(f"Tool call id not found in the map: {tool_call_id}")
+                # TODO: the input is not valid in this case, should return an error
+
+        else:
+            processed_msg.append(messages[i])
+        
+
+    if len(func_observation_map) > 0:
+        # func_observation_array = [f'{k}: {func_observation_map[k] if func_observation_map[k] != "" else "done"}' for k in func_observation_map]
+        func_observation_array = [f'{func_observation_map[k] if func_observation_map[k] != "" else "done"}' for k in func_observation_map]
+        observation_str = json.dumps(func_observation_array)
+        observation_call = {"role": "user", "content": "start observation " + observation_str + " end observation"}
+        processed_msg.append(observation_call)
+        func_observation_map.clear()
+
+    return processed_msg
+
+
+def construct_tool_call_str(tool_calls, func_observation_map) -> str:
+    tool_list = []
+    for tool_call in tool_calls:
+        tool_call_id = tool_call["id"]
+        func_observation_map[tool_call_id] = ""  # Initialize with empty value, updated later from the message with tool role
+        
+        if type(tool_call["function"]["arguments"]) == str:
+            tool_call["function"]["arguments"] = json.loads(tool_call["function"]["arguments"])
+        tool_list.append("starttoolcall"+str(tool_call["function"]) + "endtoolcall")
+
+    # Converting the Python dictionary to a YAML formatted string
+    tool_call_str = "".join(tool_list)
+    return tool_call_str
+
+
+if __name__ == "__main__":
+    tools = [{
+            "type": "function",
+            "function": {
+                "name": "dummy",
+                "description": "just to say hi",
+                "parameters": None,
+            }
+        },{"type": "function","function":{"name":"calculate_distance","description":"Calculate the distance between two locations","parameters":{"type":"object","properties":{"origin":{"type":"string","description":"The starting location"},"destination":{"type":"string","description":"The destination location"},"mode":{"type":"string","description":"The mode of transportation"}},"required":["origin","destination","mode"]}}},{"type": "function","function":{"name":"generate_password","description":"Generate a random password","parameters":{"type":"object","properties":{"length":{"type":"integer","description":"The length of the password"}},"required":["length"]}}}]
+    # msgs = [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'What is the distance between San Francisco and Cupertino by driving and by air from both directions?'}, {'role': 'assistant', 'tool_calls': [{'id': '0', 'function': {'name': 'calculate_distance', 'arguments': '{"origin":"San Francisco","destination":"Cupertino","mode":"drive"}'}, 'type': 'function'},{'id': '1', 'function': {'name': 'calculate_distance', 'arguments': '{"origin":"San Francisco","destination":"Cupertino","mode":"air"}'}, 'type': 'function'}]}, {'role': 'tool', 'tool_call_id': '0', 'name': 'calculate_distance', 'content': 'Distance is 50 miles.'}, {'role': 'tool', 'tool_call_id': '1', 'name': 'calculate_distance', 'content': ''}]
+    msgs = [{'role': 'user', 'content': "\nYou are task oriented system.\nYou receive input from a user, process the input from the given instructions, and then output the result.\nYour objective is to provide consistent and correct results.\nYou do not need to explain the steps taken, only provide the result to the given instructions.\nYou are referred to as a tool.\nYou don't move to the next step until you have a result.\n\nDownload https://www.sqlitetutorial.net/wp-content/uploads/2018/03/chinook.zip to a\nrandom file. Then expand the archive to a temporary location as there is a sqlite\ndatabase in it.\n\nFirst inspect the schema of the database to understand the table structure.\n\nForm and run a SQL query to find the artist with the most number of albums and output\nthe result of that.\n\nWhen done remove the database file and the downloaded content."}, {'role': 'assistant', 'content': '', 'tool_calls': [{'id': 'cdf7eb2e', 'type': 'function', 'function': {'name': 'download', 'arguments': '{"url":"https://www.sqlitetutorial.net/wp-content/uploads/2018/03/chinook.zip"}'}}]}, {'role': 'tool', 'content': '/tmp/gpt-download2815959664.zip', 'name': 'download', 'tool_call_id': 'cdf7eb2e'}, {'role': 'assistant', 'content': '', 'tool_calls': [{'id': '07cdee86', 'type': 'function', 'function': {'name': 'exec', 'arguments': '{"command":"unzip /tmp/gpt-download2815959664.zip","directory":"."}'}}]}, {'role': 'tool', 'content': 'Archive:  /tmp/gpt-download2815959664.zip\n  inflating: chinook.db              \n', 'name': 'exec', 'tool_call_id': '07cdee86'}, {'role': 'assistant', 'content': '', 'tool_calls': [{'id': 'e8042027', 'type': 'function', 'function': {'name': 'download', 'arguments': '{"url":"https://www.sqlitetutorial.net/wp-content/uploads/2018/03/chinook.zip"}'}}]}, {'role': 'tool', 'content': '/tmp/gpt-download3736408077.zip', 'name': 'download', 'tool_call_id': 'e8042027'}]
+    new_msgs = preprocess_input(msgs, tools)
+    print(json.dumps(new_msgs, indent=2))
+    

requirements.txt

@@ -1 +1,11 @@
-huggingface_hub==0.22.2
+#!npm install -g npm@latest
+#!npm install -g jsonrepair
+accelerate==0.30.1
+bitsandbytes==0.43.1
+gradio==4.36.1
+scipy==1.13.0
+sentencepiece==0.2.0
+spaces==0.28.3
+torch==2.0.0
+transformers==4.41.0
+pythonmonkey

style.css

@@ -0,0 +1,35 @@
+/* Main container styling */
+.gradio-container {
+    height: auto;
+    overflow: hidden;  /* Prevents overflow outside the main container */
+  }
+  
+  /* Main row configuration to fit within the screen */
+  #main-row {
+    display: flex; /* Ensures that columns within this row are aligned horizontally */
+    overflow: hidden; /* Prevents overflow outside the main row */
+  }
+  
+  /* Configuration for the column containing the chatbot to ensure it doesn't grow too long */
+  .gr-column {
+    height: 100%; /* Sets column height to fill the parent container */
+    display: flex;
+    flex-direction: column;
+    overflow: hidden; /* Ensures no overflow outside the column */
+  }
+  
+  /* Chatbot specific styling */
+  #chatbot {
+    flex-grow: 1; /* Allows chatbot to expand within the column space */
+    overflow-y: auto; /* Allows vertical scrolling within the chatbot */
+  }
+  
+  /* General improvements for component layout and aesthetics */
+  .gr-row > .wrap > div, .gr-column > .wrap {
+    margin-bottom: 10px;
+  }
+  
+  .gr-column {
+    gap: 10px;
+    align-items: flex-start; /* Aligns items to the start, preventing stretch */
+  }