File size: 2,176 Bytes
2b0f671 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 |
import gradio as gr
from tqdm import tqdm
import requests
import os
def ensure_file(filename, src):
if not os.path.exists(filename):
response = requests.get(src, stream=True)
total_size = int(response.headers.get('content-length', 0))
with open(filename, 'wb') as file:
with tqdm(total=total_size, unit='B', unit_scale=True, desc=filename, ncols=80) as progress_bar:
for data in response.iter_content(chunk_size=1024):
if data:
file.write(data)
progress_bar.update(len(data))
print(f"Download completed.")
ensure_file("mmproj-model-f16.gguf", "https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/mmproj-model-f16.gguf")
ensure_file("ggml-model-q4_k.gguf", "https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/ggml-model-q4_k.gguf")
import ctypes
import json
import argparse
import os
import array
import sys
from llama_cpp import (Llama, clip_model_load, llava_image_embed_make_with_filename, llava_image_embed_make_with_bytes,
llava_image_embed_p, llava_image_embed_free, llava_validate_embed_size, llava_eval_image_embed)
ctx_clip = clip_model_load("mmproj-model-f16.gguf".encode('utf-8'))
llm = Llama(model_path="ggml-model-q4_k.gguf", n_ctx=2048)
def generate(image, ins="The image shows"):
if len(ins) < 1:
ins = "The image shows"
image_embed = llava_image_embed_make_with_filename(ctx_clip=ctx_clip, n_threads=1, filename=image.encode('utf8'))
n_past = ctypes.c_int(llm.n_tokens)
n_past_p = ctypes.byref(n_past)
llava_eval_image_embed(llm.ctx, image_embed, llm.n_batch, n_past_p)
llm.n_tokens = n_past.value
llava_image_embed_free(image_embed)
llm.eval(llm.tokenize(ins.encode('utf8')))
max_target_len = 256
res = ""
for i in range(max_target_len):
t_id = llm.sample(temp=0.1)
t = llm.detokenize([t_id]).decode('utf8')
if t == "</s>":
break
res += t
llm.eval([t_id])
return res
iface = gr.Interface(generate, inputs=[gr.Image(type="filepath"), gr.Textbox()], outputs="text")
iface.launch() |