all files

README.md

@@ -5,7 +5,7 @@ colorFrom: green
 colorTo: purple
 sdk: gradio
 sdk_version: 3.33.1
-app_file: app.py
+app_file: script.py
 pinned: false
 license: mit
 ---

blocking_api.py

@@ -0,0 +1,131 @@
+import json
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from threading import Thread
+
+from util import build_parameters, try_start_cloudflared
+from gpt4 import ask_gpt
+# from modules import shared
+# from modules.chat import generate_chat_reply
+# from modules.text_generation import encode, generate_reply, stop_everything_event
+
+
+class Handler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path == '/api/v1/model':
+            self.send_response(200)
+            self.end_headers()
+            response = json.dumps({
+                'result': 'GPT4 mindsdb OpenAI original'
+            })
+
+            self.wfile.write(response.encode('utf-8'))
+        else:
+            self.send_error(404)
+
+    def do_POST(self):
+        content_length = int(self.headers['Content-Length'])
+        body = json.loads(self.rfile.read(content_length).decode('utf-8'))
+
+        if self.path == '/api/v1/generate':
+            self.send_response(200)
+            self.send_header('Content-Type', 'application/json')
+            self.end_headers()
+
+            prompt = body['prompt']
+            generate_params = build_parameters(body)
+            stopping_strings = generate_params.pop('stopping_strings')
+            generate_params['stream'] = False
+
+            # generator = generate_reply(
+            #     prompt, generate_params, stopping_strings=stopping_strings, is_chat=False)
+
+            answer = ask_gpt(prompt)
+
+            response = json.dumps({
+                'results': [{
+                    'text': answer
+                }]
+            })
+
+            self.wfile.write(response.encode('utf-8'))
+
+        elif self.path == '/api/v1/chat':
+            self.send_response(200)
+            self.send_header('Content-Type', 'application/json')
+            self.end_headers()
+
+            user_input = body['user_input']
+            history = body['history']
+            regenerate = body.get('regenerate', False)
+            _continue = body.get('_continue', False)
+
+            generate_params = build_parameters(body, chat=True)
+            generate_params['stream'] = False
+            generator = 'error'
+            # generator = generate_chat_reply(
+            #     user_input, history, generate_params, regenerate=regenerate, _continue=_continue, loading_message=False)
+
+            answer = history
+            for a in generator:
+                answer = a
+
+            response = json.dumps({
+                'results': [{
+                    'history': answer
+                }]
+            })
+
+            self.wfile.write(response.encode('utf-8'))
+
+        elif self.path == '/api/v1/stop-stream':
+            self.send_response(200)
+            self.send_header('Content-Type', 'application/json')
+            self.end_headers()
+
+            # stop_everything_event()
+
+            response = json.dumps({
+                'results': 'error'
+            })
+
+            self.wfile.write(response.encode('utf-8'))
+
+        elif self.path == '/api/v1/token-count':
+            self.send_response(200)
+            self.send_header('Content-Type', 'application/json')
+            self.end_headers()
+
+            # tokens = encode(body['prompt'])[0]
+            response = json.dumps({
+                'results': [{
+                    'tokens': 'error'
+                }]
+            })
+
+            self.wfile.write(response.encode('utf-8'))
+        else:
+            self.send_error(404)
+
+
+def _run_server(port: int, share: bool = False):
+    address = '0.0.0.0' if 0 else '127.0.0.1'
+
+    server = ThreadingHTTPServer((address, port), Handler)
+
+    def on_start(public_url: str):
+        print(f'Starting non-streaming server at public url {public_url}/api')
+
+    if share:
+        try:
+            try_start_cloudflared(port, max_attempts=3, on_start=on_start)
+        except Exception:
+            pass
+    else:
+        print(
+            f'Starting API at http://{address}:{port}/api')
+
+    server.serve_forever()
+
+
+def start_server(port: int, share: bool = False):
+    Thread(target=_run_server, args=[port, share], daemon=True).start()

gpt4.py

@@ -0,0 +1,32 @@
+import pymysql
+import os
+
+def connect_to_db():
+  try:
+    connection = pymysql.connect(
+      host="cloud.mindsdb.com",
+      user=os.environ['LOGIN'],
+      password=os.environ['PASSWORD'],
+      port=3306,
+      db="mindsdb",
+      charset="utf8mb4",
+      cursorclass=pymysql.cursors.DictCursor)
+    return connection
+  except Exception as e:
+    print(f"Error connecting to database: {e}")
+    return None
+print('---Trying to connect---')
+connection = connect_to_db()
+print('---CONNECTED---')
+
+def ask_gpt(text):
+    # print('ASK GPT:', text)
+    text = text.replace("'", '"')
+
+    query = f"SELECT response FROM mindsdb.gpt4 WHERE text=%s"
+
+    cursor = connection.cursor()
+    cursor.execute(query, (text, ))
+    result = cursor.fetchone()
+    response = result['response']
+    return response

requirements.txt

@@ -0,0 +1,3 @@
+flask_cloudflared==0.0.12
+websockets==11.0.2
+pymysql

script.py

@@ -0,0 +1,10 @@
+import blocking_api
+# import extensions.api.streaming_api as streaming_api
+# from modules import shared
+
+
+def setup():
+    blocking_api.start_server(7860, share=True)
+    # streaming_api.start_server(shared.args.api_streaming_port, share=shared.args.public_api)
+    while True: pass
+setup()

util.py

@@ -0,0 +1,92 @@
+import time
+import traceback
+from threading import Thread
+from typing import Callable, Optional
+
+# from modules import shared
+# from modules.chat import load_character_memoized
+shared = True
+load_character_memoized = lambda: 0
+def build_parameters(body, chat=False):
+
+    generate_params = {
+        'max_new_tokens': int(body.get('max_new_tokens', body.get('max_length', 200))),
+        'do_sample': bool(body.get('do_sample', True)),
+        'temperature': float(body.get('temperature', 0.5)),
+        'top_p': float(body.get('top_p', 1)),
+        'typical_p': float(body.get('typical_p', body.get('typical', 1))),
+        'epsilon_cutoff': float(body.get('epsilon_cutoff', 0)),
+        'eta_cutoff': float(body.get('eta_cutoff', 0)),
+        'tfs': float(body.get('tfs', 1)),
+        'top_a': float(body.get('top_a', 0)),
+        'repetition_penalty': float(body.get('repetition_penalty', body.get('rep_pen', 1.1))),
+        'encoder_repetition_penalty': float(body.get('encoder_repetition_penalty', 1.0)),
+        'top_k': int(body.get('top_k', 0)),
+        'min_length': int(body.get('min_length', 0)),
+        'no_repeat_ngram_size': int(body.get('no_repeat_ngram_size', 0)),
+        'num_beams': int(body.get('num_beams', 1)),
+        'penalty_alpha': float(body.get('penalty_alpha', 0)),
+        'length_penalty': float(body.get('length_penalty', 1)),
+        'early_stopping': bool(body.get('early_stopping', False)),
+        'mirostat_mode': int(body.get('mirostat_mode', 0)),
+        'mirostat_tau': float(body.get('mirostat_tau', 5)),
+        'mirostat_eta': float(body.get('mirostat_eta', 0.1)),
+        'seed': int(body.get('seed', -1)),
+        'add_bos_token': bool(body.get('add_bos_token', True)),
+        'truncation_length': int(body.get('truncation_length', body.get('max_context_length', 2048))),
+        'ban_eos_token': bool(body.get('ban_eos_token', False)),
+        'skip_special_tokens': bool(body.get('skip_special_tokens', True)),
+        'custom_stopping_strings': '',  # leave this blank
+        'stopping_strings': body.get('stopping_strings', []),
+    }
+
+    if chat:
+        character = body.get('character')
+        instruction_template = body.get('instruction_template')
+        name1, name2, _, greeting, context, _ = load_character_memoized(character, str(body.get('your_name', shared.settings['name1'])), shared.settings['name2'], instruct=False)
+        name1_instruct, name2_instruct, _, _, context_instruct, turn_template = load_character_memoized(instruction_template, '', '', instruct=True)
+        generate_params.update({
+            'stop_at_newline': bool(body.get('stop_at_newline', shared.settings['stop_at_newline'])),
+            'chat_prompt_size': int(body.get('chat_prompt_size', shared.settings['chat_prompt_size'])),
+            'chat_generation_attempts': int(body.get('chat_generation_attempts', shared.settings['chat_generation_attempts'])),
+            'mode': str(body.get('mode', 'chat')),
+            'name1': name1,
+            'name2': name2,
+            'context': context,
+            'greeting': greeting,
+            'name1_instruct': name1_instruct,
+            'name2_instruct': name2_instruct,
+            'context_instruct': context_instruct,
+            'turn_template': turn_template,
+            'chat-instruct_command': str(body.get('chat-instruct_command', shared.settings['chat-instruct_command'])),
+        })
+
+    return generate_params
+
+
+def try_start_cloudflared(port: int, max_attempts: int = 3, on_start: Optional[Callable[[str], None]] = None):
+    Thread(target=_start_cloudflared, args=[
+           port, max_attempts, on_start], daemon=True).start()
+
+
+def _start_cloudflared(port: int, max_attempts: int = 3, on_start: Optional[Callable[[str], None]] = None):
+    try:
+        from flask_cloudflared import _run_cloudflared
+    except ImportError:
+        print('You should install flask_cloudflared manually')
+        raise Exception(
+            'flask_cloudflared not installed. Make sure you installed the requirements.txt for this extension.')
+
+    for _ in range(max_attempts):
+        try:
+            public_url = _run_cloudflared(port, port + 1)
+
+            if on_start:
+                on_start(public_url)
+
+            return
+        except Exception:
+            traceback.print_exc()
+            time.sleep(3)
+
+        raise Exception('Could not start cloudflared.')