File size: 633 Bytes
9f39652 9f50db3 9f39652 9f50db3 9f39652 9f50db3 9f39652 db2cc08 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 |
from flask import Flask, request, jsonify, render_template
from llama_cpp import Llama
app = Flask(__name__)
llm = Llama(model_path="./yuna-ai-v3-q3_k_m.gguf", verbose=False)
@app.route('/')
def index():
return render_template('index.html')
@app.route('/api/generate', methods=['POST'])
def generate():
user_message = request.json['message']
output = llm(
f"Yuki: {user_message}\nYuna:",
max_tokens=16,
stop=["Yuki:", "Yuna:", "\n"],
echo=False
)
return jsonify({'response': output['choices'][0]['text']})
if __name__ == '__main__':
app.run(host='0.0.0.0', port=7860) |