Spaces:
Runtime error
Runtime error
| import os | |
| import torch | |
| import openai | |
| import requests | |
| import gradio as gr | |
| import transformers | |
| from transformers import Pix2StructForConditionalGeneration, Pix2StructProcessor | |
| #from peft import PeftModel | |
| if torch.cuda.is_available(): | |
| device = "cuda" | |
| else: | |
| device = "cpu" | |
| try: | |
| if torch.backends.mps.is_available(): | |
| device = "mps" | |
| except: | |
| pass | |
| ## CoT prompts | |
| def _add_markup(table): | |
| try: | |
| parts = [p.strip() for p in table.splitlines(keepends=False)] | |
| if parts[0].startswith('TITLE'): | |
| result = f"Title: {parts[0].split(' | ')[1].strip()}\n" | |
| rows = parts[1:] | |
| else: | |
| result = '' | |
| rows = parts | |
| prefixes = ['Header: '] + [f'Row {i+1}: ' for i in range(len(rows) - 1)] | |
| return result + '\n'.join(prefix + row for prefix, row in zip(prefixes, rows)) | |
| except: | |
| # just use the raw table if parsing fails | |
| return table | |
| _TABLE = """Year | Democrats | Republicans | Independents | |
| 2004 | 68.1% | 45.0% | 53.0% | |
| 2006 | 58.0% | 42.0% | 53.0% | |
| 2007 | 59.0% | 38.0% | 45.0% | |
| 2009 | 72.0% | 49.0% | 60.0% | |
| 2011 | 71.0% | 51.2% | 58.0% | |
| 2012 | 70.0% | 48.0% | 53.0% | |
| 2013 | 72.0% | 41.0% | 60.0%""" | |
| _INSTRUCTION = 'Read the table below to answer the following questions.' | |
| _TEMPLATE = f"""First read an example then the complete question for the second table. | |
| ------------ | |
| {_INSTRUCTION} | |
| {_add_markup(_TABLE)} | |
| Q: In which year republicans have the lowest favor rate? | |
| A: Let's find the column of republicans. Then let's extract the favor rates, they [45.0, 42.0, 38.0, 49.0, 51.2, 48.0, 41.0]. The smallest number is 38.0, that's Row 3. Row 3 is year 2007. The answer is 2007. | |
| Q: What is the sum of Democrats' favor rates of 2004, 2012, and 2013? | |
| A: Let's find the rows of years 2004, 2012, and 2013. We find Row 1, 6, 7. The favor dates of Demoncrats on that 3 rows are 68.1, 70.0, and 72.0. 68.1+70.0+72=210.1. The answer is 210.1. | |
| Q: By how many points do Independents surpass Republicans in the year of 2011? | |
| A: Let's find the row with year = 2011. We find Row 5. We extract Independents and Republicans' numbers. They are 58.0 and 51.2. 58.0-51.2=6.8. The answer is 6.8. | |
| Q: Which group has the overall worst performance? | |
| A: Let's sample a couple of years. In Row 1, year 2004, we find Republicans having the lowest favor rate 45.0 (since 45.0<68.1, 45.0<53.0). In year 2006, Row 2, we find Republicans having the lowest favor rate 42.0 (42.0<58.0, 42.0<53.0). The trend continues to other years. The answer is Republicans. | |
| Q: Which party has the second highest favor rates in 2007? | |
| A: Let's find the row of year 2007, that's Row 3. Let's extract the numbers on Row 3: [59.0, 38.0, 45.0]. 45.0 is the second highest. 45.0 is the number of Independents. The answer is Independents. | |
| {_INSTRUCTION}""" | |
| ## alpaca-lora | |
| # assert ( | |
| # "LlamaTokenizer" in transformers._import_structure["models.llama"] | |
| # ), "LLaMA is now in HuggingFace's main branch.\nPlease reinstall it: pip uninstall transformers && pip install git+https://github.com/huggingface/transformers.git" | |
| # from transformers import LlamaTokenizer, LlamaForCausalLM, GenerationConfig | |
| # tokenizer = LlamaTokenizer.from_pretrained("decapoda-research/llama-7b-hf") | |
| # BASE_MODEL = "decapoda-research/llama-7b-hf" | |
| # LORA_WEIGHTS = "tloen/alpaca-lora-7b" | |
| # if device == "cuda": | |
| # model = LlamaForCausalLM.from_pretrained( | |
| # BASE_MODEL, | |
| # load_in_8bit=False, | |
| # torch_dtype=torch.float16, | |
| # device_map="auto", | |
| # ) | |
| # model = PeftModel.from_pretrained( | |
| # model, LORA_WEIGHTS, torch_dtype=torch.float16, force_download=True | |
| # ) | |
| # elif device == "mps": | |
| # model = LlamaForCausalLM.from_pretrained( | |
| # BASE_MODEL, | |
| # device_map={"": device}, | |
| # torch_dtype=torch.float16, | |
| # ) | |
| # model = PeftModel.from_pretrained( | |
| # model, | |
| # LORA_WEIGHTS, | |
| # device_map={"": device}, | |
| # torch_dtype=torch.float16, | |
| # ) | |
| # else: | |
| # model = LlamaForCausalLM.from_pretrained( | |
| # BASE_MODEL, device_map={"": device}, low_cpu_mem_usage=True | |
| # ) | |
| # model = PeftModel.from_pretrained( | |
| # model, | |
| # LORA_WEIGHTS, | |
| # device_map={"": device}, | |
| # ) | |
| # if device != "cpu": | |
| # model.half() | |
| # model.eval() | |
| # if torch.__version__ >= "2": | |
| # model = torch.compile(model) | |
| ## FLAN-UL2 | |
| HF_TOKEN = os.environ.get("API_TOKEN", None) | |
| API_URL = "https://api-inference.huggingface.co/models/google/flan-ul2" | |
| headers = {"Authorization": f"Bearer {HF_TOKEN}"} | |
| def query(payload): | |
| response = requests.post(API_URL, headers=headers, json=payload) | |
| return response.json() | |
| ## OpenAI models | |
| openai.api_key = os.environ.get("OPENAI_TOKEN", None) | |
| def set_openai_api_key(api_key): | |
| if api_key and api_key.startswith("sk-") and len(api_key) > 50: | |
| openai.api_key = api_key | |
| def get_response_from_openai(prompt, model="gpt-3.5-turbo", max_output_tokens=256): | |
| messages = [{"role": "assistant", "content": prompt}] | |
| response = openai.ChatCompletion.create( | |
| model=model, | |
| messages=messages, | |
| temperature=0.7, | |
| max_tokens=max_output_tokens, | |
| top_p=1, | |
| frequency_penalty=0, | |
| presence_penalty=0, | |
| ) | |
| ret = response.choices[0].message['content'] | |
| return ret | |
| ## deplot models | |
| model_deplot = Pix2StructForConditionalGeneration.from_pretrained("google/deplot", torch_dtype=torch.bfloat16) | |
| if device == "cuda": | |
| model_deplot = model_deplot.to(0) | |
| processor_deplot = Pix2StructProcessor.from_pretrained("google/deplot") | |
| def evaluate( | |
| table, | |
| question, | |
| llm="alpaca-lora", | |
| input=None, | |
| temperature=0.1, | |
| top_p=0.75, | |
| top_k=40, | |
| num_beams=4, | |
| max_new_tokens=128, | |
| **kwargs, | |
| ): | |
| prompt_0shot = _INSTRUCTION + "\n" + _add_markup(table) + "\n" + "Q: " + question + "\n" + "A:" | |
| prompt = _TEMPLATE + "\n" + _add_markup(table) + "\n" + "Q: " + question + "\n" + "A:" | |
| if llm == "alpaca-lora": | |
| inputs = tokenizer(prompt, return_tensors="pt") | |
| input_ids = inputs["input_ids"].to(device) | |
| generation_config = GenerationConfig( | |
| temperature=temperature, | |
| top_p=top_p, | |
| top_k=top_k, | |
| num_beams=num_beams, | |
| **kwargs, | |
| ) | |
| with torch.no_grad(): | |
| generation_output = model.generate( | |
| input_ids=input_ids, | |
| generation_config=generation_config, | |
| return_dict_in_generate=True, | |
| output_scores=True, | |
| max_new_tokens=max_new_tokens, | |
| ) | |
| s = generation_output.sequences[0] | |
| output = tokenizer.decode(s) | |
| elif llm == "flan-ul2": | |
| try: | |
| output = query({"inputs": prompt_0shot})[0]["generated_text"] | |
| except: | |
| output = "<flan-ul2 inference API error - try later>" | |
| elif llm == "gpt-3.5-turbo": | |
| try: | |
| output = get_response_from_openai(prompt_0shot) | |
| except: | |
| output = "<Remember to input your OpenAI API key ☺>" | |
| else: | |
| RuntimeError(f"No such LLM: {llm}") | |
| return output | |
| def process_document(image, question, llm): | |
| # image = Image.open(image) | |
| inputs = processor_deplot(images=image, text="Generate the underlying data table for the figure below:", return_tensors="pt").to(torch.bfloat16) | |
| if device == "cuda": | |
| inputs = inputs.to(0) | |
| predictions = model_deplot.generate(**inputs, max_new_tokens=512) | |
| table = processor_deplot.decode(predictions[0], skip_special_tokens=True).replace("<0x0A>", "\n") | |
| # send prompt+table to LLM | |
| res = evaluate(table, question, llm=llm) | |
| if llm == "alpaca-lora": | |
| return [table, res.split("A:")[-1]] | |
| else: | |
| return [table, res] | |
| # theme = gr.themes.Monochrome( | |
| # primary_hue="indigo", | |
| # secondary_hue="blue", | |
| # neutral_hue="slate", | |
| # radius_size=gr.themes.sizes.radius_sm, | |
| # font=[gr.themes.GoogleFont("Open Sans"), "ui-sans-serif", "system-ui", "sans-serif"], | |
| # ) | |
| with gr.Blocks(theme="gradio/soft") as demo: | |
| with gr.Column(): | |
| # gr.Markdown( | |
| # """<h1><center>DePlot+LLM: Multimodal chain-of-thought reasoning on plots</center></h1> | |
| # <p> | |
| # This is a demo of DePlot+LLM for QA and summarisation. <a href='https://arxiv.org/abs/2212.10505' target='_blank'>DePlot</a> is an image-to-text model that converts plots and charts into a textual sequence. The sequence then is used to prompt LLM for chain-of-thought reasoning. The current underlying LLMs are <a href='https://huggingface.co/spaces/tloen/alpaca-lora' target='_blank'>alpaca-lora</a>, <a href='https://huggingface.co/google/flan-ul2' target='_blank'>flan-ul2</a>, and <a href='https://openai.com/blog/chatgpt' target='_blank'>gpt-3.5-turbo</a>. To use it, simply upload your image and type a question or instruction and click 'submit', or click one of the examples to load them. Read more at the links below. | |
| # </p> | |
| # """ | |
| # ) | |
| gr.Markdown( | |
| """<h1><center>DePlot+LLM: Multimodal chain-of-thought reasoning on plot📊</center></h1> | |
| <h3> | |
| <center> | |
| <a href='https://arxiv.org/abs/2212.09662' target='_blank'>[paper]</a> <a href='https://ai.googleblog.com/2023/05/foundation-models-for-reasoning-on.html' target='_blank'>[google-ai blog]</a> <a href='https://github.com/google-research/google-research/tree/master/deplot' target='_blank'>[code]</a> | |
| </center> | |
| </h3> | |
| <p> | |
| This is a demo of DePlot+LLM for QA and summarisation. <a href='https://arxiv.org/abs/2212.10505' target='_blank'>DePlot</a> is an image-to-text model that converts plots and charts into a textual sequence. The sequence then is used to prompt LLM for chain-of-thought reasoning. The current underlying LLMs are <a href='https://huggingface.co/google/flan-ul2' target='_blank'>flan-ul2</a> and <a href='https://openai.com/blog/chatgpt' target='_blank'>gpt-3.5-turbo</a>. To use it, simply upload your image and type a question or instruction and click 'submit', or click one of the examples to load them. | |
| </p> | |
| """ | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=2): | |
| input_image = gr.Image(label="Input Image", type="pil", interactive=True) | |
| #input_image.style(height=512, width=512) | |
| instruction = gr.Textbox(placeholder="Enter your instruction/question...", label="Question/Instruction") | |
| #llm = gr.Dropdown(["alpaca-lora", "flan-ul2", "gpt-3.5-turbo"], label="LLM") | |
| llm = gr.Dropdown(["flan-ul2", "gpt-3.5-turbo"], label="LLM") | |
| openai_api_key_textbox = gr.Textbox(value='', | |
| placeholder="Paste your OpenAI API key (sk-...) and hit Enter (if using OpenAI models, otherwise leave empty)", | |
| show_label=False, lines=1, type='password') | |
| submit = gr.Button("Submit", variant="primary") | |
| with gr.Column(scale=2): | |
| with gr.Accordion("Show intermediate table", open=False): | |
| output_table = gr.Textbox(lines=8, label="Intermediate Table") | |
| output_text = gr.Textbox(lines=8, label="Output") | |
| gr.Examples( | |
| examples=[ | |
| ["deplot_case_study_6.png", "Rank the four methods according to average model performances. By how much does deplot outperform the second strongest approach on average across the two sets? Show the computation.", "gpt-3.5-turbo"], # ex 1 | |
| ["deplot_case_study_4.png", "What are the acceptance rates? And how does the acceptance change over the years?", "gpt-3.5-turbo"], # ex 2 | |
| ["deplot_case_study_m1.png", "Summarise the chart for me please.", "gpt-3.5-turbo"], # ex 3 | |
| #["deplot_case_study_m1.png", "What is the sum of numbers of Indonesia and Ireland? Remember to think step by step.", "alpaca-lora"], | |
| #["deplot_case_study_3.png", "By how much did China's growth rate drop? Think step by step.", "alpaca-lora"], | |
| #["deplot_case_study_4.png", "How many papers are submitted in 2020?", "flan-ul2"], | |
| ["deplot_case_study_5.png", "Which sales channel has the second highest portion?", "flan-ul2"], # ex 4 | |
| #["deplot_case_study_x2.png", "Summarise the chart for me please.", "alpaca-lora"], | |
| #["deplot_case_study_4.png", "How many papers are submitted in 2020?", "alpaca-lora"], | |
| #["deplot_case_study_m1.png", "Summarise the chart for me please.", "alpaca-lora"], | |
| #["deplot_case_study_4.png", "acceptance rate = # accepted / #submitted . What is the acceptance rate of 2010?", "flan-ul2"], | |
| #["deplot_case_study_m1.png", "Summarise the chart for me please.", "flan-ul2"], | |
| ], | |
| cache_examples=True, | |
| inputs=[input_image, instruction, llm], | |
| outputs=[output_table, output_text], | |
| fn=process_document | |
| ) | |
| gr.Markdown( | |
| """<p style='text-align: center'><a href='https://arxiv.org/abs/2212.10505' target='_blank'>DePlot: One-shot visual language reasoning by plot-to-table translation</a></p>""" | |
| ) | |
| openai.api_key = "" | |
| openai_api_key_textbox.change(set_openai_api_key, | |
| inputs=[openai_api_key_textbox], | |
| outputs=[]) | |
| openai_api_key_textbox.submit(set_openai_api_key, | |
| inputs=[openai_api_key_textbox], | |
| outputs=[]) | |
| submit.click(process_document, inputs=[input_image, instruction, llm], outputs=[output_table, output_text]) | |
| instruction.submit( | |
| process_document, inputs=[input_image, instruction, llm], outputs=[output_table, output_text] | |
| ) | |
| demo.queue(concurrency_count=1).launch() |