import gradio as gr from datetime import datetime import pandas as pd from transformers import pipeline # # Load in packages # + import os # Need to overwrite version of gradio present in Huggingface spaces as it doesn't have like buttons/avatars (Oct 2023) #os.system("pip uninstall -y gradio") os.system("pip install gradio==3.50.0") from typing import TypeVar #from langchain.embeddings import HuggingFaceEmbeddings#, HuggingFaceInstructEmbeddings #from langchain.vectorstores import FAISS import gradio as gr from transformers import AutoTokenizer # Alternative model sources import ctransformers PandasDataFrame = TypeVar('pd.core.frame.DataFrame') import chatfuncs.chatfuncs as chatf # Disable cuda devices if necessary #os.environ['CUDA_VISIBLE_DEVICES'] = '-1' def create_hf_model(model_name): tokenizer = AutoTokenizer.from_pretrained(model_name, model_max_length = chatf.context_length) summariser = pipeline("summarization", model=model_name, tokenizer=tokenizer) # philschmid/bart-large-cnn-samsum #from transformers import AutoModelForSeq2SeqLM, AutoModelForCausalLM # if torch_device == "cuda": # if "flan" in model_name: # model = AutoModelForSeq2SeqLM.from_pretrained(model_name, device_map="auto") # else: # model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto") # else: # if "flan" in model_name: # model = AutoModelForSeq2SeqLM.from_pretrained(model_name) # else: # model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True) return summariser, tokenizer, model_name def load_model(model_type, gpu_layers, gpu_config=None, cpu_config=None, torch_device=None): print("Loading model ", model_type) # Default values inside the function if gpu_config is None: gpu_config = chatf.gpu_config if cpu_config is None: cpu_config = chatf.cpu_config if torch_device is None: torch_device = chatf.torch_device if model_type == "Mistral Open Orca (larger, slow)": hf_checkpoint = 'TheBloke/MistralLite-7B-GGUF' if torch_device == "cuda": gpu_config.update_gpu(gpu_layers) else: gpu_config.update_gpu(gpu_layers) cpu_config.update_gpu(gpu_layers) print("Loading with", cpu_config.gpu_layers, "model layers sent to GPU.") print(vars(gpu_config)) print(vars(cpu_config)) #try: #model = ctransformers.AutoModelForCausalLM.from_pretrained('Aryanne/Orca-Mini-3B-gguf', model_type='llama', model_file='q5_0-orca-mini-3b.gguf', **vars(gpu_config)) # **asdict(CtransRunConfig_cpu()) #model = ctransformers.AutoModelForCausalLM.from_pretrained('Aryanne/Wizard-Orca-3B-gguf', model_type='llama', model_file='q4_1-wizard-orca-3b.gguf', **vars(gpu_config)) # **asdict(CtransRunConfig_cpu()) #model = ctransformers.AutoModelForCausalLM.from_pretrained('TheBloke/Mistral-7B-OpenOrca-GGUF', model_type='mistral', model_file='mistral-7b-openorca.Q4_K_M.gguf', **vars(gpu_config), hf=True) # **asdict(CtransRunConfig_cpu()) #except: #model = ctransformers.AutoModelForCausalLM.from_pretrained('Aryanne/Orca-Mini-3B-gguf', model_type='llama', model_file='q5_0-orca-mini-3b.gguf', **vars(cpu_config)) #**asdict(CtransRunConfig_gpu()) #model = ctransformers.AutoModelForCausalLM.from_pretrained('Aryanne/Wizard-Orca-3B-gguf', model_type='llama', model_file='q4_1-wizard-orca-3b.gguf', **vars(cpu_config)) # **asdict(CtransRunConfig_cpu()) #model = ctransformers.AutoModelForCausalLM.from_pretrained('TheBloke/Mistral-7B-OpenOrca-GGUF', model_type='mistral', model_file='mistral-7b-openorca.Q4_K_M.gguf', **vars(cpu_config), hf=True) # **asdict(CtransRunConfig_cpu()) #tokenizer = ctransformers.AutoTokenizer.from_pretrained(model) #summariser = pipeline("text-generation", model=model, tokenizer=tokenizer) model = [] tokenizer = [] summariser = [] if model_type == "flan-t5-large-stacked-samsum": # Huggingface chat model hf_checkpoint = 'stacked-summaries/flan-t5-large-stacked-samsum-1024'#'declare-lab/flan-alpaca-base' # # # summariser, tokenizer, model_type = create_hf_model(model_name = hf_checkpoint) if model_type == "flan-t5-small-stacked-samsum": # Huggingface chat model hf_checkpoint = 'stacked-summaries/flan-t5-small-stacked-samsum-1024' #'philschmid/flan-t5-small-stacked-samsum'#'declare-lab/flan-alpaca-base' # # # summariser, tokenizer, model_type = create_hf_model(model_name = hf_checkpoint) chatf.model = summariser chatf.tokenizer = tokenizer chatf.model_type = model_type load_confirmation = "Finished loading model: " + model_type print(load_confirmation) return model_type, load_confirmation, model_type # Both models are loaded on app initialisation so that users don't have to wait for the models to be downloaded #model_type = "Mistral Open Orca (larger, slow)" #load_model(model_type, chatf.gpu_layers, chatf.gpu_config, chatf.cpu_config, chatf.torch_device) model_type = "flan-t5-large-stacked-samsum" load_model(model_type, chatf.gpu_layers, chatf.gpu_config, chatf.cpu_config, chatf.torch_device) model_type = "flan-t5-small-stacked-samsum" load_model(model_type, 0, chatf.gpu_config, chatf.cpu_config, chatf.torch_device) today = datetime.now().strftime("%d%m%Y") today_rev = datetime.now().strftime("%Y%m%d") def summarise_text(text, text_df, length_slider, in_colnames, model_type): if text_df == None: in_colnames="text" in_colnames_list_first = in_colnames in_text_df = pd.DataFrame({in_colnames_list_first:[text]}) else: in_text_df = pd.read_csv(text_df.name, delimiter = ",", low_memory=False, encoding='cp1252') in_colnames_list_first = in_colnames.tolist()[0][0] if model_type != "Mistral Open Orca (larger, slow)": summarised_text = chatf.model(list(in_text_df[in_colnames_list_first]), max_length=length_slider) if model_type == "Mistral Open Orca (larger, slow)": length = str(length_slider) prompt = """<|im_start|>system You are an AI assistant that follows instruction extremely well. Help as much as you can. <|im_start|>user Summarise the following text in less than {length} words. Text: {text} Answer:<|im_end|>""" formatted_string = prompt.format(length=length, text=text) print(formatted_string) #summarised_text = chatf.model(formatted_string, max_new_tokens=length_slider) summarised_text = "Mistral Open Orca summaries currently not working. Sorry!" if text_df == None: if model_type != "Mistral Open Orca (larger, slow)": summarised_text_out = summarised_text[0].values() if model_type == "Mistral Open Orca (larger, slow)": summarised_text_out = summarised_text else: summarised_text_out = [d['summary_text'] for d in summarised_text] #summarised_text[0].values() output_name = "summarise_output_" + today_rev + ".csv" output_df = pd.DataFrame({"Original text":in_text_df[in_colnames_list_first], "Summarised text":summarised_text_out}) summarised_text_out_str = str(output_df["Summarised text"][0])#.str.replace("dict_values([","").str.replace("])","")) output_df.to_csv(output_name, index = None) return summarised_text_out_str, output_name # ## Gradio app - summarise block = gr.Blocks(theme = gr.themes.Base()) with block: model_type_state = gr.State(model_type) gr.Markdown( """ # Text summariser Enter open text below to get a summary. You can copy and paste text directly, or upload a file and specify the column that you want to summarise. Note that summarisation with Mistral Open Orca is still in development and does not currently work. """) with gr.Tab("Summariser"): current_model = gr.Textbox(label="Current model", value=model_type, scale = 3) with gr.Accordion("Paste open text", open = False): in_text = gr.Textbox(label="Copy and paste your open text here", lines = 5) with gr.Accordion("Summarise open text from a file", open = False): in_text_df = gr.File(label="Input text from file") in_colnames = gr.Dataframe(label="Write the column name for the open text to summarise", type="numpy", row_count=(1,"fixed"), col_count = (1,"fixed"), headers=["Open text column name"])#, "Address column name 2", "Address column name 3", "Address column name 4"]) with gr.Row(): summarise_btn = gr.Button("Summarise") length_slider = gr.Slider(minimum = 30, maximum = 200, value = 100, step = 10, label = "Maximum length of summary") with gr.Row(): output_single_text = gr.Textbox(label="Output example (first example in dataset)") output_file = gr.File(label="Output file") with gr.Tab("Advanced features"): #out_passages = gr.Slider(minimum=1, value = 2, maximum=10, step=1, label="Choose number of passages to retrieve from the document. Numbers greater than 2 may lead to increased hallucinations or input text being truncated.") #temp_slide = gr.Slider(minimum=0.1, value = 0.1, maximum=1, step=0.1, label="Choose temperature setting for response generation.") with gr.Row(): model_choice = gr.Radio(label="Choose a summariser model", value="flan-t5-small-stacked-samsum", choices = ["flan-t5-small-stacked-samsum", "flan-t5-large-stacked-samsum", "Mistral Open Orca (larger, slow)"]) change_model_button = gr.Button(value="Load model", scale=0) with gr.Accordion("Choose number of model layers to send to GPU (WARNING: please don't modify unless you are sure you have a GPU).", open = False): gpu_layer_choice = gr.Slider(label="Choose number of model layers to send to GPU.", value=0, minimum=0, maximum=5, step = 1, visible=True) load_text = gr.Text(label="Load status") change_model_button.click(fn=load_model, inputs=[model_choice, gpu_layer_choice], outputs = [model_type_state, load_text, current_model]) summarise_btn.click(fn=summarise_text, inputs=[in_text, in_text_df, length_slider, in_colnames, model_type_state], outputs=[output_single_text, output_file], api_name="summarise_single_text") block.queue(concurrency_count=1).launch() # -