from llama_cpp import Llama from huggingface_hub import hf_hub_download from flask import Flask, request, jsonify app = Flask(__name__) hf_hub_download("TheBloke/phi-2-GGUF", "phi-2.Q8_0.gguf", local_dir="./") phi = Llama(model_path="./phi-2.Q8_0.gguf", n_ctx=2048, n_gpu_layers=999) app.route("/spaces/MrOvkill/phastfi/", methods=["GET"]) def index(): return "

Use API

Use /completion as POST with a prompt in a JSON query.

" @app.route("/spaces/MrOvkill/phastfi/generate", methods=["POST"]) def completion(): prompt = request.json["prompt"] res = phi( prompt, temperature=0.33, top_p=0.95, top_k=42, max_tokens=1024, num_completions=2, ) return jsonify({ "responses": [ res["choices"][0]["text"], res["choices"][1]["text"] ] }) if __name__ == "__main__": app.run(host="0.0.0.0", port=1140)