Upload 11 files

assets/config.json

@@ -0,0 +1,8 @@
+{
+    "HOST": "0.0.0.0",
+    "PORT": 5000,
+    "DEBUG": false,
+    
+    "use_global": true,
+    "use_addons": true
+}

assets/source/__init__.py

@@ -0,0 +1,9 @@
+from .api import Api, non_streamed_format
+
+# create api instance
+api = Api()
+
+__all__ = [
+    "api",
+    "non_streamed_format"
+]

assets/source/addons/__init__.py

@@ -0,0 +1,12 @@
+"""
+All addons should be imported here
+"""
+from .translation import translate, message_translation
+from .expose import create_cloudflare_tunnel
+
+# here the addons are exported
+__all__ = [
+    'translate',
+    'message_translation',
+    'create_cloudflare_tunnel'
+]

assets/source/addons/expose.md

@@ -0,0 +1,8 @@
+# DeepInfra-Wrapper Addon - Cloudflare Quick-tunnels.
+
+This addon is needed to globally expose our flask application and create a URL that can be accessed from anywhere.
+
+## Usage
+
+To use, call `create_cloudflare_tunnel` function and pass the port as a paremeter. You will then see Cloudflare emit the created ULR inside the console.
+Remember, you need to have `use_addons` set to `true` in the config.json

assets/source/addons/expose.py

@@ -0,0 +1,130 @@
+"""
+Code credits: ColabKobold
+
+https://github.com/lostruins/koboldcpp
+"""
+
+import os, sys, threading, time, subprocess, re
+
+def create_cloudflare_tunnel(port: int):
+
+    """
+    Create a cloudflare tunnel for the server
+    """
+
+    try:
+
+        def run_tunnel():
+            
+            """
+            Run the cloudflare tunnel
+            """
+
+            # variables to be used in the thread
+            tunnel_process = None
+            tunnel_output = ""
+            tunnel_raw_dump = ""
+
+            time.sleep(0.2)
+            if os.name == 'nt':
+
+                print("Starting Cloudflare Tunnel for Windows, please wait...", flush=True)
+                tunnel_process = subprocess.Popen(f"cloudflared.exe tunnel --url localhost:{port}", text=True, encoding='utf-8', shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+            
+            elif sys.platform=="darwin":
+                print("Starting Cloudflare Tunnel for MacOS, please wait...", flush=True)
+                tunnel_process = subprocess.Popen(f"./cloudflared tunnel --url http://localhost:{port}", text=True, encoding='utf-8', shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+            
+            else:
+                print("Starting Cloudflare Tunnel for Linux, please wait...", flush=True)
+                tunnel_process = subprocess.Popen(f"./cloudflared-linux-amd64 tunnel --url http://localhost:{port}", text=True, encoding='utf-8', shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
+            
+            # wait for the tunnel to be ready
+            time.sleep(10)
+
+            def tunnel_reader():
+
+                """
+                Read the stderr of the cloudflare tunnel process
+                """
+
+                nonlocal tunnel_process, tunnel_output, tunnel_raw_dump
+
+                pattern: str = r'https://[\w\.-]+\.trycloudflare\.com'
+
+                while True:
+
+                    line = tunnel_process.stderr.readline() #cloudflare writes to stderr for some reason
+                    tunnel_raw_dump += line+"\n"
+
+                    if not line:
+
+                        # if the line is empty, the tunnel is closed
+                        return
+                    
+                    found = re.findall(pattern, line)
+
+                    for x in found:
+
+                        tunneloutput = x
+
+                        return
+
+            tunnel_reader_thread = threading.Thread(target=tunnel_reader)
+            tunnel_reader_thread.start()
+
+            time.sleep(5) # wait for the tunnel to be ready
+
+            if tunnel_output=="":
+
+                print(f"Error: Could not create cloudflare tunnel!\nMore Info:\n{tunnel_raw_dump}", flush=True)
+
+            time.sleep(0.5)
+            tunnel_process.wait()
+
+        # Windows
+        if os.name == 'nt':
+
+            if os.path.exists("cloudflared.exe") and os.path.getsize("cloudflared.exe") > 1000000:
+
+                print("Cloudflared file exists, reusing it...")
+
+            else:
+
+                print("Downloading Cloudflare Tunnel for Windows...")
+                subprocess.run("curl -fL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-windows-amd64.exe -o cloudflared.exe", shell=True, capture_output=True, text=True, check=True, encoding='utf-8')
+        
+        # MacOS
+        elif sys.platform=="darwin":
+
+            if os.path.exists("cloudflared") and os.path.getsize("cloudflared") > 1000000:
+                print("Cloudflared file exists, reusing it...")
+
+            else:
+
+                print("Downloading Cloudflare Tunnel for MacOS...")
+                subprocess.run("curl -fL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-darwin-amd64.tgz -o cloudflared-darwin-amd64.tgz", shell=True, capture_output=True, text=True, check=True, encoding='utf-8')
+                subprocess.run("tar -xzf cloudflared-darwin-amd64.tgz", shell=True)
+                subprocess.run("chmod +x 'cloudflared'", shell=True)
+
+        else:
+
+            if os.path.exists("cloudflared-linux-amd64") and os.path.getsize("cloudflared-linux-amd64") > 1000000:
+                print("Cloudflared file exists, reusing it...")
+
+            else:
+
+                print("Downloading Cloudflare Tunnel for Linux...")
+                subprocess.run("curl -fL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 -o cloudflared-linux-amd64", shell=True, capture_output=True, text=True, check=True, encoding='utf-8')
+                subprocess.run("chmod +x 'cloudflared-linux-amd64'", shell=True)
+
+        print("Attempting to start tunnel thread...", flush=True)
+        tunnel_thread = threading.Thread(target=run_tunnel)
+        tunnel_thread.start()
+
+    except Exception as ex:
+
+        print("Remote Tunnel Failed!")
+        print(str(ex))
+        return None
+    

assets/source/addons/translation.md

@@ -0,0 +1,10 @@
+# DeepInfra-Wrapper Addon - Translation
+
+This addon is used to translate OpenAI model names to the actual model names. The names are semi-random.
+
+## Usage
+
+To use this addon, you can call the `translate` function with a model name as the input. The function will return the corresponding actual model name.
+Remember, you need to have `use_addons` set to `true` in the config.json
+
+For regular use cases, turn this off.

assets/source/addons/translation.py

@@ -0,0 +1,58 @@
+"""
+This addon is used to make OpenAI model names point to one of these models. The names are semi-random picked.
+"""
+
+model_map = [
+    {"gpt-4-0613": "mistralai/Mistral-7B-Instruct-v0.3"},
+    {"davinci-002": ""},
+    {"gpt-4": "microsoft/WizardLM-2-8x22B"},
+    {"gpt-4o": "mistralai/Mistral-7B-Instruct-v0.2"},
+    {"gpt-4-0125-preview": "mistralai/Mixtral-8x7B-Instruct-v0.1"},
+    {"gpt-4-turbo-preview": "mistralai/Mixtral-8x22B-Instruct-v0.1"},
+    {"gpt-4-1106-preview": "cognitivecomputations/dolphin-2.6-mixtral-8x7b"},
+    {"gpt-3.5-turbo-16k": "meta-llama/Meta-Llama-3-8B-Instruct"},
+    {"gpt-3.5-turbo-instruct-0914": "meta-llama/Meta-Llama-3-70B-Instruct"},
+]
+
+def translate(model: str) -> str:
+
+    """
+    Translate model names to the actual model names
+    """
+
+    for i in model_map:
+        if model in i:
+            return i[model]
+
+    return model
+
+def message_translation(messages: list[dict[str, str]]) -> list[dict[str, str]]:
+
+    """
+    Translate messages
+    
+    Current Message format
+    
+    messages = [{"role": "user", "content": "Whatever"}]
+    
+    Old format:
+    
+    messages = [{"role": "user", "content": [{"text": "Whatever"}]}]
+    
+    :param messages: list of messages
+    
+    :return: list of messages
+    """
+    translated_messages = []
+    
+    for message in messages:
+        # Check if the message content is in the old format
+        if isinstance(message['content'], list):
+            # Extract the text from the old format and assign it to the new format
+            content = message['content'][0]['text']
+            message['content'] = content
+        
+        # Append the message to the list of translated messages
+        translated_messages.append(message)
+    
+    return translated_messages

assets/source/api.py

@@ -0,0 +1,132 @@
+# ---------------------------------------- IMPORTS ---------------------------------------- #
+
+# this module allows us to get the headers from the get_headers function
+from .headers.get_headers import get_headers
+
+# this module allows us to get a random user agent
+from fake_useragent import UserAgent
+
+# this module allows us to generate random numbers
+import secrets
+
+# to make requests
+import requests
+
+# type hints
+from typing import Generator, Any, Dict, List
+
+# ---------------------------------------- FUNCTIONS ---------------------------------------- #
+def non_streamed_format(model: str, content: str) -> Dict[str, Any]:
+
+    return {
+        "object": "chat.completion",
+        "model": f"{model}",
+        "choices": [{
+            "index": 0,
+            "message": {
+                "role": "assistant",
+                "content": f"{content}",
+            },
+            "finish_reason": "stop",
+        }],
+    }
+# ---------------------------------------- API CLASS ---------------------------------------- #
+
+class Api(object):
+
+    # initialize the class and set some variables here
+    def __init__(self) -> None:
+
+        """initialize the class"""
+        
+        self.url: str = "https://api.deepinfra.com/v1/openai/chat/completions"
+
+        self.headers = get_headers(UserAgent().random, secrets.randbelow(500)) # get random headers
+
+        self.session = requests.Session() # create a session
+
+        # codec for encoding and decoding
+        self.codec = 'utf-8'
+
+    def get_models(self) -> Dict[str, List[Dict[str, str]]]:
+
+        """get all models"""
+
+        return {'data': [
+            {"id": "meta-llama/Meta-Llama-3-70B-Instruct", "context": 8192},
+            {"id": "meta-llama/Meta-Llama-3-8B-Instruct", "context": 8192},
+            {"id": "codellama/CodeLlama-34b-Instruct-hf", "context": 4096},
+            {"id": "jondurbin/airoboros-l2-70b-gpt4-1.4.1", "context": 4096},
+            {"id": "mistralai/Mistral-7B-Instruct-v0.2", "context": 32768},
+            {"id": "mistralai/Mistral-7B-Instruct-v0.3", "context": 32768},
+            {"id": "mistralai/Mixtral-8x7B-Instruct-v0.1", "context": 32768},
+            {"id": "mistralai/Mixtral-8x22B-Instruct-v0.1", "context": 65536},
+            {"id": "cognitivecomputations/dolphin-2.6-mixtral-8x7b", "context": 32768},
+            {"id": "lizpreciatior/lzlv_70b_fp16_hf", "context": 32768},
+            {"id": "deepinfra/airoboros-70b", "context": 4096},
+            {"id": "HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1", "context": 65536},
+            {"id": "microsoft/WizardLM-2-8x22B", "context": 65536},
+        ]}
+    
+
+    def chat(
+            self,
+            messages: List[Dict[str, str]], 
+            model: str, 
+            stream: bool = True, 
+            temperature: int = 0.7, 
+            max_tokens: int = 150, 
+            top_p: float = 1.0, 
+            top_k: int = 50,
+            presence_penalty: float = 0.0,
+            frequency_penalty: float = 0.0
+        ) -> Generator[str, Any, Any] | Dict[str, Any]:
+
+        """
+        Chat with the DeepInfra models.
+
+        :param messages: list of messages
+        :param model: model name
+
+        :param stream: stream the response
+        :param temperature: temperature
+        :param max_tokens: max tokens
+
+        :param top_p: top p
+        :param top_k: top k
+        
+        :param presence_penalty: presence penalty
+        :param frequency_penalty: frequency penalty
+        
+        :return: generator or dict
+        """
+
+        # compile the data
+        data = {
+            'messages': messages,
+            'model': model,
+            'stream': stream,
+
+            'temperature': temperature,
+            'max_tokens': max_tokens,
+
+            'top_p': top_p,
+            'top_k': top_k,
+
+            'presence_penalty': presence_penalty,
+            'frequency_penalty': frequency_penalty,
+        }
+
+        # make a post request to the api
+        response = self.session.post(self.url, headers=self.headers, json=data, stream=stream)
+
+        # raise for status
+        response.raise_for_status()
+
+        # if stream is False
+        if not stream:
+            return non_streamed_format(model, response.json()['choices'][0]['message']['content'])
+
+        return response.iter_lines()
+
+# Path: assets/source/api.py

assets/source/headers/__init__.py

@@ -0,0 +1,3 @@
+from .get_headers import get_headers
+
+__all__ = ['get_headers']

assets/source/headers/get_headers.py

@@ -0,0 +1,21 @@
+def get_headers(user_agent: str, content_length: int):
+
+    return {
+        'Authority': 'api.deepinfra.com',
+        'Host': 'api.deepinfra.com',
+        'User-Agent': f'{user_agent}',
+        'Accept': 'text/event-stream',
+        'Accept-Language': 'en-US,en;q=0.5',
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Referer': 'https://deepinfra.com/',
+        'Content-Type': 'application/json',
+        'X-Deepinfra-Source': 'web-page',
+        'Content-Length': f'{content_length}',
+        'Origin': 'https://deepinfra.com',
+        'Connection': 'keep-alive',
+        'Sec-Fetch-Dest': 'empty',
+        'Sec-Fetch-Mode': 'cors',
+        'Sec-Fetch-Site': 'same-site'
+}
+
+# Path: assets/source/headers/get_headers.py

assets/templates/index.html

@@ -0,0 +1,74 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>DeepInfra Wrapper - Developer Reference</title>
+    <style>
+        body {
+            background-color: #1a1a1a;
+            color: #e0e0e0;
+            font-family: 'Arial', sans-serif;
+            margin: 0;
+            padding: 0;
+        }
+        .container {
+            width: 80%;
+            margin: 0 auto;
+            padding: 20px;
+        }
+        header {
+            text-align: center;
+            padding: 50px 0;
+        }
+        header h1 {
+            color: #00ff00;
+        }
+        .status {
+            text-align: center;
+            padding: 10px;
+            margin-bottom: 20px;
+            background-color: #006400;
+            border-radius: 5px;
+        }
+        .routes {
+            margin-top: 20px;
+        }
+        .routes h2 {
+            color: #00ff00;
+        }
+        .routes ul {
+            list-style: none;
+            padding: 0;
+        }
+        .routes ul li {
+            background-color: #333333;
+            padding: 10px;
+            margin: 5px 0;
+            border-radius: 5px;
+        }
+        .routes ul li a {
+            color: #00ff00;
+            text-decoration: none;
+        }
+        .routes ul li a:hover {
+            text-decoration: underline;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <header>
+            <h1>DeepInfra Wrapper</h1>
+            <div class="status">Status: Active</div>
+        </header>
+        <div class="routes">
+            <h2>Available Routes</h2>
+            <ul>
+                <li><a href="/chat/completions">/chat/completions</a></li>
+                <li><a href="/models">/models</a></li>
+            </ul>
+        </div>
+    </div>
+</body>
+</html>