Spaces:
Runtime error
Runtime error
| import os | |
| os.system("pip install deepspeed") | |
| os.system("pip freeze") | |
| import gradio as gr | |
| import re | |
| from magma import Magma | |
| from magma.image_input import ImageInput | |
| from huggingface_hub import hf_hub_url, cached_download | |
| checkpoint_url = hf_hub_url(repo_id="osanseviero/magma", filename="model.pt") | |
| checkpoint_path = cached_download(checkpoint_url) | |
| model = Magma.from_checkpoint( | |
| config_path = "configs/MAGMA_v1.yml", | |
| checkpoint_path = checkpoint_path, | |
| device = 'cuda:0' | |
| ) | |
| def generate(image,context, length, temperature, top_k,rearrange): | |
| # context = context.strip() | |
| # url_regex = r'https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)' | |
| # lines = context.split('\n') | |
| # inputs = [] | |
| # for line in lines: | |
| # if re.match(url_regex, line): | |
| # try: | |
| # inputs.append(ImageInput(line)) | |
| # except Exception as e: | |
| # return str(e) | |
| # else: | |
| # inputs.append(line) | |
| if rearrange: | |
| inputs =[ | |
| ## supports urls and path/to/image | |
| context, | |
| ImageInput(image) | |
| ] | |
| else: | |
| inputs =[ | |
| ## supports urls and path/to/image | |
| ImageInput(image), | |
| context | |
| ] | |
| ## returns a tensor of shape: (1, 149, 4096) | |
| embeddings = model.preprocess_inputs(inputs) | |
| ## returns a list of length embeddings.shape[0] (batch size) | |
| output = model.generate( | |
| embeddings = embeddings, | |
| max_steps = length, | |
| temperature = (0.01 if temperature == 0 else temperature), | |
| top_k = top_k | |
| ) | |
| return output[0] | |
| examples=[["woods_hi.jpeg","Describe the painting:",15,0.7,0,False], ["E8EB3C7B-291C-400A-81F2-AE9229D9CE23.jpeg", "Q: Is the person in the image older than 35?\nA: " , 15, 0.7, 0, False]] | |
| title="MAGMA" | |
| description="Gradio Demo for MAGMA -- Multimodal Augmentation of Generative Models through Adapter-based Finetuning by Constantin Eichenberg, Sid Black, Samuel Weinbach, Letitia Parcalabescu, and Anette Frank<br> <br><a href='https://arxiv.org/abs/2112.05253' target='_blank'>arXiv</a> | <a href='https://github.com/Aleph-Alpha/magma' target='_blank'>Github Repo</a>" | |
| article = "" | |
| iface = gr.Interface( | |
| fn=generate, | |
| inputs=[ | |
| gr.inputs.Image(type="filepath",label="Image Prompt"),gr.inputs.Textbox( | |
| label="Text Prompt:", | |
| default="Describe the painting:", | |
| lines=7), | |
| gr.inputs.Slider(minimum=1, maximum=100, default=15, step=1, label="Output tokens:"), | |
| gr.inputs.Slider(minimum=0.0, maximum=1.0, default=0.7, label='Temperature'), | |
| gr.inputs.Slider(minimum=0, maximum=100, default=0, step=1, label='Top K'), | |
| gr.inputs.Checkbox(default=False, label="Rearrange Prompt", optional=False) | |
| ], | |
| outputs=["textbox"], | |
| examples=examples, | |
| title=title, | |
| description=description, | |
| article=article | |
| ).launch(enable_queue=True,cache_examples=True) | |