Upload 7 files

app.py

@@ -0,0 +1,27 @@
+import fn
+import gradio as gr
+from refresh import create_refresh_button
+
+with gr.Blocks() as demo:
+    title = gr.Markdown('# llama-cpp-python server manager')
+    info = gr.Markdown()
+    url = gr.Textbox(label='URL', interactive=True)
+    download_button = gr.Button(value='Download')
+    model = gr.Dropdown(label='Model', choices=fn.list_model(), interactive=True)
+    create_refresh_button(gr, model, lambda: None, lambda: {'choices': fn.list_model()}, 'refresh-button', interactive=True)
+    set_model_button = gr.Button(value='Set Model')
+
+    download_button.click(
+        fn=fn.download,
+        inputs=[url],
+        outputs=[info],
+        )
+
+    set_model_button.click(
+        fn=fn.set_model,
+        inputs=[model],
+        outputs=[],
+        )
+
+if __name__ == '__main__':
+    demo.launch()

fn.py

@@ -0,0 +1,31 @@
+import os
+import requests
+
+def download(url):
+    filename = os.path.basename(url)
+    r = requests.get(url, stream=True)
+    with open(f'models/{filename}', 'wb') as f:
+        for chunk in r.iter_content(chunk_size=1024*1024):
+            if chunk:
+                f.write(chunk)
+                f.flush()
+
+    return filename
+
+def list_model():
+    if not os.path.exists('models'):
+        os.makedirs('models')
+    return os.listdir('models')
+
+def set_model(filename):
+    if not os.path.exists(f'models/{filename}'):
+        return False
+
+    with open(f'models/set.txt', 'w', encoding='utf-8') as f:
+        f.write(filename)
+
+    cmd = f'venv/bin/python -m llama_cpp.server --model models/{filename} --port $1 --n_gpu_layers 999'
+    with open(f'run.sh', 'w', encoding='utf-8') as f:
+        f.write(cmd)
+
+    return True

install.bat

@@ -0,0 +1,56 @@
+@echo off
+
+rem -------------------------------------------
+rem NOT guaranteed to work on Windows
+
+set APPDIR=lcps
+set REPOS=https://huggingface.co/spaces/aka7774/%APPDIR%
+set VENV=venv
+
+rem -------------------------------------------
+
+set INSTALL_DIR=%~dp0
+cd /d %INSTALL_DIR%
+
+:git_clone
+set DL_URL=%REPOS%
+set DL_DST=%APPDIR%
+git clone %DL_URL% %APPDIR%
+if exist %DL_DST% goto install_python
+
+set DL_URL=https://github.com/git-for-windows/git/releases/download/v2.41.0.windows.3/PortableGit-2.41.0.3-64-bit.7z.exe
+set DL_DST=PortableGit-2.41.0.3-64-bit.7z.exe
+curl -L -o %DL_DST% %DL_URL%
+if not exist %DL_DST% bitsadmin /transfer dl %DL_URL% %DL_DST%
+%DL_DST% -y
+del %DL_DST%
+
+set GIT=%INSTALL_DIR%PortableGit\bin\git
+%GIT% clone %REPOS%
+
+:install_python
+set DL_URL=https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.10.13+20240107-i686-pc-windows-msvc-shared-install_only.tar.gz
+set DL_DST="%INSTALL_DIR%python.tar.gz"
+curl -L -o %DL_DST% %DL_URL%
+if not exist %DL_DST% bitsadmin /transfer dl %DL_URL% %DL_DST%
+tar -xzf %DL_DST%
+
+set PYTHON=%INSTALL_DIR%python\python.exe
+set PATH=%PATH%;%INSTALL_DIR%python310\Scripts
+
+:install_venv
+cd %APPDIR%
+%PYTHON% -m venv %VENV%
+set PYTHON=%VENV%\Scripts\python.exe
+
+:install_pip
+set DL_URL=https://bootstrap.pypa.io/get-pip.py
+set DL_DST=%INSTALL_DIR%get-pip.py
+curl -o %DL_DST% %DL_URL%
+if not exist %DL_DST% bitsadmin /transfer dl %DL_URL% %DL_DST%
+%PYTHON% %DL_DST%
+
+%PYTHON% -m pip install gradio
+%PYTHON% -m pip install -r requirements.txt
+
+pause

main.py

@@ -0,0 +1,42 @@
+import os
+import sys
+import time
+import io
+from PIL import Image
+
+from fastapi import FastAPI, Request, status, Form, UploadFile
+from fastapi.staticfiles import StaticFiles
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel, Field
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import Response
+
+import fn
+import gradio as gr
+from app import demo
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=['*'],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+gr.mount_gradio_app(app, demo, path="/gradio")
+
+@app.post("/download")
+async def api_download(url: str):
+    filename = fn.download(url)
+    return {"status": 0, "filename": filename}
+
+@app.get("/list_model")
+async def api_list_model():
+    return {"status": 0, "models": fn.list_model()}
+
+@app.post("/set_model")
+async def api_set_model(model: str):
+    fn.set_model(model)
+    return {"status": 0}

refresh.py

@@ -0,0 +1,23 @@
+
+def create_refresh_button(gr, refresh_component, refresh_method, refreshed_args, elem_class, interactive=True):
+    """
+    Copied from https://github.com/AUTOMATIC1111/stable-diffusion-webui
+    """
+    refresh_symbol = '🔄'
+    def refresh():
+        refresh_method()
+        args = refreshed_args() if callable(refreshed_args) else refreshed_args
+
+        for k, v in args.items():
+            setattr(refresh_component, k, v)
+
+        return gr.update(**(args or {}))
+
+    refresh_button = gr.Button(refresh_symbol, elem_classes=elem_class, interactive=interactive)
+    refresh_button.click(
+        fn=refresh,
+        inputs=[],
+        outputs=[refresh_component]
+    )
+
+    return refresh_button

requirements.txt

@@ -0,0 +1,3 @@
+fastapi
+uvicorn
+python-multipart

venv.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/bash
+
+python3 -m venv venv
+curl -kL https://bootstrap.pypa.io/get-pip.py | venv/bin/python
+
+CMAKE_ARGS="-DLLAMA_CUBLAS=on" venv/bin/python -m pip install llama-cpp-python --upgrade --force-reinstall --no-cache-dir
+venv/bin/python -m pip install 'llama-cpp-python[server]'
+
+venv/bin/python -m pip install gradio
+venv/bin/python -m pip install -r requirements.txt