| import gradio as gr | |
| from tqdm import tqdm | |
| import requests | |
| import os | |
| def ensure_file(filename, src): | |
| if not os.path.exists(filename): | |
| response = requests.get(src, stream=True) | |
| total_size = int(response.headers.get('content-length', 0)) | |
| with open(filename, 'wb') as file: | |
| with tqdm(total=total_size, unit='B', unit_scale=True, desc=filename, ncols=80) as progress_bar: | |
| for data in response.iter_content(chunk_size=1024): | |
| if data: | |
| file.write(data) | |
| progress_bar.update(len(data)) | |
| print(f"Download completed.") | |
| ensure_file("mmproj-model-f16.gguf", "https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/mmproj-model-f16.gguf") | |
| ensure_file("ggml-model-q4_k.gguf", "https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/ggml-model-q4_k.gguf") | |
| import ctypes | |
| import json | |
| import argparse | |
| import os | |
| import array | |
| import sys | |
| from llama_cpp import (Llama, clip_model_load, llava_image_embed_make_with_filename, llava_image_embed_make_with_bytes, | |
| llava_image_embed_p, llava_image_embed_free, llava_validate_embed_size, llava_eval_image_embed) | |
| ctx_clip = clip_model_load("mmproj-model-f16.gguf".encode('utf-8')) | |
| llm = Llama(model_path="ggml-model-q4_k.gguf", n_ctx=2048) | |
| def generate(image, ins="The image shows"): | |
| if len(ins) < 1: | |
| ins = "The image shows" | |
| image_embed = llava_image_embed_make_with_filename(ctx_clip=ctx_clip, n_threads=1, filename=image.encode('utf8')) | |
| n_past = ctypes.c_int(llm.n_tokens) | |
| n_past_p = ctypes.byref(n_past) | |
| llava_eval_image_embed(llm.ctx, image_embed, llm.n_batch, n_past_p) | |
| llm.n_tokens = n_past.value | |
| llava_image_embed_free(image_embed) | |
| llm.eval(llm.tokenize(ins.encode('utf8'))) | |
| max_target_len = 256 | |
| res = "" | |
| for i in range(max_target_len): | |
| t_id = llm.sample(temp=0.1) | |
| t = llm.detokenize([t_id]).decode('utf8') | |
| if t == "</s>": | |
| break | |
| res += t | |
| llm.eval([t_id]) | |
| return res | |
| iface = gr.Interface(generate, inputs=[gr.Image(type="filepath"), gr.Textbox()], outputs="text") | |
| iface.launch() |