Create script.sh

script.sh

@@ -0,0 +1,244 @@
+
+set -e
+
+echo "=========================================="
+echo "Phi-4-mini-instruct Agentic Model Server"
+echo "=========================================="
+echo ""
+
+# Configuration
+MODEL_REPO="${MODEL_REPO:-unsloth/Phi-4-mini-instruct-GGUF}"
+MODEL_FILE="${MODEL_FILE:-Phi-4-mini-instruct-Q4_K_M.gguf}"
+N_CTX="${N_CTX:-8192}"
+N_THREADS="${N_THREADS:-2}"
+HOST="${HOST:-0.0.0.0}"
+PORT="${PORT:-7860}"
+MODEL_DIR="${MODEL_DIR:-/app/models}"
+
+echo "Configuration:"
+echo "  Model Repo: $MODEL_REPO"
+echo "  Model File: $MODEL_FILE"
+echo "  Context Length: $N_CTX"
+echo "  Threads: $N_THREADS"
+echo "  Host: $HOST"
+echo "  Port: $PORT"
+echo ""
+
+# Create model directory
+mkdir -p "$MODEL_DIR"
+
+# Download model if not exists
+MODEL_PATH="$MODEL_DIR/$MODEL_FILE"
+if [ ! -f "$MODEL_PATH" ]; then
+    echo "Downloading model from HuggingFace..."
+    echo "This may take a few minutes on first run..."
+    python3 << 'PYTHON_DOWNLOAD'
+import os
+from huggingface_hub import hf_hub_download
+
+repo = os.environ.get("MODEL_REPO", "unsloth/Phi-4-mini-instruct-GGUF")
+filename = os.environ.get("MODEL_FILE", "Phi-4-mini-instruct-Q4_K_M.gguf")
+model_dir = os.environ.get("MODEL_DIR", "/app/models")
+
+print(f"Downloading {filename} from {repo}...")
+model_path = hf_hub_download(
+    repo_id=repo,
+    filename=filename,
+    local_dir=model_dir,
+    local_dir_use_symlinks=False
+)
+print(f"Model saved to: {model_path}")
+PYTHON_DOWNLOAD
+    echo "Model download complete!"
+else
+    echo "Model already cached at: $MODEL_PATH"
+fi
+
+echo ""
+echo "Starting Phi-4-mini-instruct server on port $PORT..."
+echo "OpenAI-compatible API endpoints available:"
+echo "  - POST /v1/chat/completions"
+echo "  - POST /v1/completions"
+echo "  - GET /v1/models"
+echo "  - GET /health"
+echo ""
+echo "Tool calling example:"
+echo '  curl -X POST http://localhost:7860/v1/chat/completions \\'
+echo '    -H "Content-Type: application/json" \\'
+echo '    -d '"'"'{"model": "phi-4-mini", "messages": [...], "tools": [...]}'"'"''
+echo ""
+
+# Start the FastAPI server
+python3 << 'PYTHON_SERVER'
+import os
+import json
+import time
+from typing import List, Optional, Dict, Any, Union
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import StreamingResponse, JSONResponse
+from pydantic import BaseModel, Field
+import uvicorn
+from llama_cpp import Llama
+
+# Configuration
+MODEL_PATH = os.environ.get("MODEL_DIR", "/app/models") + "/" + os.environ.get("MODEL_FILE", "Phi-4-mini-instruct-Q4_K_M.gguf")
+N_CTX = int(os.environ.get("N_CTX", "8192"))
+N_THREADS = int(os.environ.get("N_THREADS", "2"))
+
+# Global model instance
+llm = None
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    global llm
+    print(f"Loading model from: {MODEL_PATH}")
+    print(f"Context length: {N_CTX}, Threads: {N_THREADS}")
+    
+    llm = Llama(
+        model_path=MODEL_PATH,
+        n_ctx=N_CTX,
+        n_threads=N_THREADS,
+        verbose=False
+    )
+    print("Model loaded successfully!")
+    yield
+    print("Shutting down...")
+
+app = FastAPI(
+    title="Phi-4-mini-instruct API",
+    description="OpenAI-compatible API for Phi-4-mini-instruct with tool calling support",
+    version="1.0.0",
+    lifespan=lifespan
+)
+
+# Pydantic models for OpenAI compatibility
+class ChatMessage(BaseModel):
+    role: str
+    content: Optional[str] = None
+    name: Optional[str] = None
+    tool_calls: Optional[List[Dict]] = None
+    tool_call_id: Optional[str] = None
+
+class ToolFunction(BaseModel):
+    name: str
+    description: Optional[str] = ""
+    parameters: Optional[Dict] = {}
+
+class Tool(BaseModel):
+    type: str = "function"
+    function: ToolFunction
+
+class ChatCompletionRequest(BaseModel):
+    model: str = "phi-4-mini"
+    messages: List[ChatMessage]
+    tools: Optional[List[Tool]] = None
+    tool_choice: Optional[Union[str, Dict]] = "auto"
+    temperature: Optional[float] = 0.7
+    max_tokens: Optional[int] = 2048
+    stream: Optional[bool] = False
+
+class ModelInfo(BaseModel):
+    id: str
+    object: str = "model"
+    created: int
+    owned_by: str = "microsoft"
+
+@app.get("/health")
+async def health_check():
+    return {"status": "healthy", "model_loaded": llm is not None}
+
+@app.get("/v1/models")
+async def list_models():
+    return {
+        "object": "list",
+        "data": [
+            {
+                "id": "phi-4-mini",
+                "object": "model",
+                "created": int(time.time()),
+                "owned_by": "microsoft"
+            }
+        ]
+    }
+
+@app.post("/v1/chat/completions")
+async def chat_completions(request: ChatCompletionRequest):
+    if llm is None:
+        raise HTTPException(status_code=503, detail="Model not loaded")
+    
+    # Convert messages to llama.cpp format
+    messages = []
+    for msg in request.messages:
+        message = {"role": msg.role, "content": msg.content or ""}
+        if msg.tool_calls:
+            message["tool_calls"] = msg.tool_calls
+        if msg.tool_call_id:
+            message["tool_call_id"] = msg.tool_call_id
+        messages.append(message)
+    
+    # Prepare tools if provided
+    tools = None
+    if request.tools:
+        tools = [t.model_dump() for t in request.tools]
+    
+    try:
+        response = llm.create_chat_completion(
+            messages=messages,
+            tools=tools,
+            tool_choice=request.tool_choice if tools else None,
+            temperature=request.temperature,
+            max_tokens=request.max_tokens,
+            stream=request.stream
+        )
+        
+        if request.stream:
+            async def generate():
+                for chunk in response:
+                    yield f"data: {json.dumps(chunk)}\n\n"
+                yield "data: [DONE]\n\n"
+            return StreamingResponse(generate(), media_type="text/event-stream")
+        
+        return JSONResponse(content=response)
+    
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/v1/completions")
+async def completions(request: dict):
+    if llm is None:
+        raise HTTPException(status_code=503, detail="Model not loaded")
+    
+    prompt = request.get("prompt", "")
+    max_tokens = request.get("max_tokens", 2048)
+    temperature = request.get("temperature", 0.7)
+    stream = request.get("stream", False)
+    
+    try:
+        response = llm(
+            prompt=prompt,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            stream=stream
+        )
+        
+        if stream:
+            async def generate():
+                for chunk in response:
+                    yield f"data: {json.dumps(chunk)}\n\n"
+                yield "data: [DONE]\n\n"
+            return StreamingResponse(generate(), media_type="text/event-stream")
+        
+        return JSONResponse(content=response)
+    
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+if __name__ == "__main__":
+    uvicorn.run(
+        app,
+        host=os.environ.get("HOST", "0.0.0.0"),
+        port=int(os.environ.get("PORT", 7860))
+    )
+PYTHON_SERVER