Upgraded Gradio version. Changed large model to Phi 3 128k. Now uses Llama cpp Python

README.md

@@ -4,7 +4,7 @@ emoji: 🚀
 colorFrom: green
 colorTo: gray
 sdk: gradio
-sdk_version: 3.50.0
+sdk_version: 4.36.0
 app_file: app.py
 pinned: false
 license: apache-2.0

app.py

@@ -5,15 +5,17 @@ from transformers import pipeline, AutoTokenizer
 import os
 from typing import Type
 import gradio as gr
-import ctransformers
-# Concurrent futures is used to cancel processes that are taking too long
-import concurrent.futures
+
+from llama_cpp import Llama
+from huggingface_hub import hf_hub_download
 
 PandasDataFrame = Type[pd.DataFrame]
 
 import chatfuncs.chatfuncs as chatf
+import chatfuncs.summarise_funcs as sumf
 
-from chatfuncs.helper_functions import dummy_function, display_info, put_columns_in_df, put_columns_in_join_df, get_temp_folder_path, empty_folder
+from chatfuncs.helper_functions import dummy_function, put_columns_in_df
+from chatfuncs.summarise_funcs import summarise_text
 
 # Disable cuda devices if necessary
 #os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 
@@ -36,22 +38,7 @@ def create_hf_model(model_name):
 
     tokenizer = AutoTokenizer.from_pretrained(model_name, model_max_length = chatf.context_length)
 
-    summariser = pipeline("summarization", model=model_name, tokenizer=tokenizer) # philschmid/bart-large-cnn-samsum
-
-    #from transformers import AutoModelForSeq2SeqLM,  AutoModelForCausalLM
-    
-    #     if torch_device == "cuda":
-    #         if "flan" in model_name:
-    #             model = AutoModelForSeq2SeqLM.from_pretrained(model_name, device_map="auto")
-    #         else:
-    #             model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")
-    #     else:
-    #         if "flan" in model_name:
-    #             model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
-    #         else: 
-    #             model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
-
-    
+    summariser = pipeline("summarization", model=model_name, tokenizer=tokenizer) # philschmid/bart-large-cnn-samsum    
 
     return summariser, tokenizer, model_name
 
@@ -66,46 +53,40 @@ def load_model(model_type, gpu_layers, gpu_config=None, cpu_config=None, torch_d
     if torch_device is None:
         torch_device = chatf.torch_device
 
-    if model_type == "Mistral Nous Capybara 4k (larger, slow)":
-        hf_checkpoint = 'NousResearch/Nous-Capybara-7B-V1.9-GGUF'
-
+    if model_type == "Phi 3 128k (larger, slow)":
         if torch_device == "cuda":
             gpu_config.update_gpu(gpu_layers)
+            print("Loading with", gpu_config.n_gpu_layers, "model layers sent to GPU.")
         else:
             gpu_config.update_gpu(gpu_layers)
             cpu_config.update_gpu(gpu_layers)
 
-        print("Loading with", cpu_config.gpu_layers, "model layers sent to GPU.")
+            print("Loading with", cpu_config.n_gpu_layers, "model layers sent to GPU.")
 
         print(vars(gpu_config))
         print(vars(cpu_config))
 
         try:
-            #model = ctransformers.AutoModelForCausalLM.from_pretrained('Aryanne/Orca-Mini-3B-gguf', model_type='llama', model_file='q5_0-orca-mini-3b.gguf', **vars(gpu_config)) # **asdict(CtransRunConfig_cpu())
-            #model = ctransformers.AutoModelForCausalLM.from_pretrained('Aryanne/Wizard-Orca-3B-gguf', model_type='llama', model_file='q4_1-wizard-orca-3b.gguf', **vars(gpu_config)) # **asdict(CtransRunConfig_cpu())
-            #model = ctransformers.AutoModelForCausalLM.from_pretrained('TheBloke/Mistral-7B-OpenOrca-GGUF', model_type='mistral', model_file='mistral-7b-openorca.Q4_K_M.gguf', **vars(gpu_config), hf=True) # **asdict(CtransRunConfig_cpu())
-            #model = ctransformers.AutoModelForCausalLM.from_pretrained('TheBloke/OpenHermes-2.5-Mistral-7B-16k-GGUF', model_type='mistral', model_file='openhermes-2.5-mistral-7b-16k.Q4_K_M.gguf', **vars(gpu_config), hf=True) # **asdict(CtransRunConfig_cpu())
-            model = ctransformers.AutoModelForCausalLM.from_pretrained('NousResearch/Nous-Capybara-7B-V1.9-GGUF', model_type='mistral', model_file='Capybara-7B-V1.9-Q5_K_M.gguf', **vars(gpu_config), hf=True) # **asdict(CtransRunConfig_cpu())
-
-
-            tokenizer = AutoTokenizer.from_pretrained("NousResearch/Nous-Capybara-7B-V1.9")
-            summariser = pipeline("text-generation", model=model, tokenizer=tokenizer)
-
-        except:
-            #model = ctransformers.AutoModelForCausalLM.from_pretrained('Aryanne/Orca-Mini-3B-gguf', model_type='llama', model_file='q5_0-orca-mini-3b.gguf', **vars(cpu_config)) #**asdict(CtransRunConfig_gpu())
-            #model = ctransformers.AutoModelForCausalLM.from_pretrained('Aryanne/Wizard-Orca-3B-gguf', model_type='llama', model_file='q4_1-wizard-orca-3b.gguf', **vars(cpu_config)) # **asdict(CtransRunConfig_cpu())
-            #model = ctransformers.AutoModelForCausalLM.from_pretrained('TheBloke/Mistral-7B-OpenOrca-GGUF', model_type='mistral', model_file='mistral-7b-openorca.Q4_K_M.gguf', **vars(cpu_config), hf=True) # **asdict(CtransRunConfig_cpu())
-            #model = ctransformers.AutoModelForCausalLM.from_pretrained('TheBloke/OpenHermes-2.5-Mistral-7B-16k-GGUF', model_type='mistral', model_file='openhermes-2.5-mistral-7b-16k.Q4_K_M.gguf', **vars(gpu_config), hf=True) # **asdict(CtransRunConfig_cpu())
-            model = ctransformers.AutoModelForCausalLM.from_pretrained('NousResearch/Nous-Capybara-7B-V1.9-GGUF', model_type='mistral', model_file='Capybara-7B-V1.9-Q5_K_M.gguf', **vars(gpu_config), hf=True) # **asdict(CtransRunConfig_cpu())
-            
-            #tokenizer = ctransformers.AutoTokenizer.from_pretrained(model)
-
-            tokenizer = AutoTokenizer.from_pretrained("NousResearch/Nous-Capybara-7B-V1.9")
-            summariser = pipeline("text-generation", model=model, tokenizer=tokenizer) # model
-
-        #model = []
-        #tokenizer = []
-        #summariser = []
+            summariser = Llama(
+            model_path=hf_hub_download(
+            repo_id=os.environ.get("REPO_ID", "QuantFactory/Phi-3-mini-128k-instruct-GGUF"),# "QuantFactory/Phi-3-mini-128k-instruct-GGUF"), # "QuantFactory/Meta-Llama-3-8B-Instruct-GGUF-v2"), #"microsoft/Phi-3-mini-4k-instruct-gguf"),#"TheBloke/Mistral-7B-OpenOrca-GGUF"),
+            filename=os.environ.get("MODEL_FILE", "Phi-3-mini-128k-instruct.Q4_K_M.gguf") #"Phi-3-mini-128k-instruct.Q4_K_M.gguf")  #"Meta-Llama-3-8B-Instruct-v2.Q6_K.gguf") #"Phi-3-mini-4k-instruct-q4.gguf")#"mistral-7b-openorca.Q4_K_M.gguf"),
+        ),
+        **vars(gpu_config) # change n_gpu_layers if you have more or less VRAM 
+        )
+        
+        except Exception as e:
+            print("GPU load failed")
+            print(e)
+            summariser = Llama(
+            model_path=hf_hub_download(
+            repo_id=os.environ.get("REPO_ID", "QuantFactory/Phi-3-mini-128k-instruct-GGUF"), #"QuantFactory/Phi-3-mini-128k-instruct-GGUF"), #, "microsoft/Phi-3-mini-4k-instruct-gguf"),#"QuantFactory/Meta-Llama-3-8B-Instruct-GGUF-v2"), #"microsoft/Phi-3-mini-4k-instruct-gguf"),#"TheBloke/Mistral-7B-OpenOrca-GGUF"),
+            filename=os.environ.get("MODEL_FILE", "Phi-3-mini-128k-instruct.Q4_K_M.gguf"), # "Phi-3-mini-128k-instruct.Q4_K_M.gguf") # , #"Meta-Llama-3-8B-Instruct-v2.Q6_K.gguf") #"Phi-3-mini-4k-instruct-q4.gguf"),#"mistral-7b-openorca.Q4_K_M.gguf"),
+        ),
+        **vars(cpu_config)
+        )
+
+        tokenizer = []
 
     if model_type == "Flan T5 Large Stacked Samsum 1k":
         # Huggingface chat model
@@ -118,9 +99,9 @@ def load_model(model_type, gpu_layers, gpu_config=None, cpu_config=None, torch_d
         hf_checkpoint = 'pszemraj/long-t5-tglobal-base-16384-book-summary' #'philschmid/flan-t5-small-stacked-samsum'#'declare-lab/flan-alpaca-base' # # #
         summariser, tokenizer, model_type = create_hf_model(model_name = hf_checkpoint)
 
-    chatf.model = summariser
-    chatf.tokenizer = tokenizer
-    chatf.model_type = model_type
+    sumf.model = summariser
+    sumf.tokenizer = tokenizer
+    sumf.model_type = model_type
 
     load_confirmation = "Finished loading model: " + model_type
 
@@ -128,7 +109,7 @@ def load_model(model_type, gpu_layers, gpu_config=None, cpu_config=None, torch_d
     return model_type, load_confirmation, model_type
 
 # Both models are loaded on app initialisation so that users don't have to wait for the models to be downloaded
-model_type = "Mistral Nous Capybara 4k (larger, slow)"
+model_type = "Phi 3 128k (larger, slow)"
 load_model(model_type, chatf.gpu_layers, chatf.gpu_config, chatf.cpu_config, chatf.torch_device)
 
 model_type = "Flan T5 Large Stacked Samsum 1k"
@@ -140,115 +121,6 @@ load_model(model_type, chatf.gpu_layers, chatf.gpu_config, chatf.cpu_config, cha
 today = datetime.now().strftime("%d%m%Y")
 today_rev = datetime.now().strftime("%Y%m%d")
 
-def summarise_text(text, text_df, length_slider, in_colname, model_type, progress=gr.Progress()):      
-         
-        if text_df.empty:
-            in_colname="text"
-            in_colname_list_first = in_colname
-
-            in_text_df = pd.DataFrame({in_colname_list_first:[text]})
-            
-        else: 
-            in_text_df = text_df
-            in_colname_list_first = in_colname
-
-        print(model_type)
-
-        texts_list = list(in_text_df[in_colname_list_first])
-
-        if model_type != "Mistral Nous Capybara 4k (larger, slow)":
-            summarised_texts = []
-
-            for single_text in progress.tqdm(texts_list, desc = "Summarising texts", unit = "texts"):
-                summarised_text = chatf.model(single_text, max_length=length_slider)
-
-                #print(summarised_text)
-
-                summarised_text_str = summarised_text[0]['summary_text']
-
-                summarised_texts.append(summarised_text_str)
-
-                print(summarised_text_str)
-
-                #pd.Series(summarised_texts).to_csv("summarised_texts_out.csv")
-
-            #print(summarised_texts)
-
-        if model_type == "Mistral Nous Capybara 4k (larger, slow)":
-
-
-            # Define a function that calls your model
-            def call_model(formatted_string, max_length=10000):
-                return chatf.model(formatted_string, max_length=max_length)
-
-            # Set your timeout duration (in seconds)
-            timeout_duration = 300  # Adjust this value as needed
-
-            length = str(length_slider)
-
-            from chatfuncs.prompts import nous_capybara_prompt
-
-            summarised_texts = []
-
-            for single_text in progress.tqdm(texts_list, desc = "Summarising texts", unit = "texts"):
-
-                formatted_string = nous_capybara_prompt.format(length=length, text=single_text)
-
-                # Use ThreadPoolExecutor to enforce a timeout
-                with concurrent.futures.ThreadPoolExecutor() as executor:
-                    future = executor.submit(call_model, formatted_string, 10000)
-                    try:
-                        output = future.result(timeout=timeout_duration)
-                        # Process the output here
-                    except concurrent.futures.TimeoutError:
-                        error_text = f"Timeout (five minutes) occurred for text: {single_text}. Consider using a smaller model."
-                        print(error_text)
-                        return error_text, None
-
-                print(output)
-
-                output_str = output[0]['generated_text']
-
-                # Find the index of 'ASSISTANT: ' to select only text after this location
-                index = output_str.find('ASSISTANT: ')
-
-                # Check if 'ASSISTANT: ' is found in the string
-                if index != -1:
-                    # Add the length of 'ASSISTANT: ' to the index to start from the end of this substring
-                    start_index = index + len('ASSISTANT: ')
-                    
-                    # Slice the string from this point to the end
-                    assistant_text = output_str[start_index:]
-                else:
-                    assistant_text = "ASSISTANT: not found in text"
-
-                print(assistant_text)
-
-                summarised_texts.append(assistant_text)
-
-                #print(summarised_text)
-                
-                #pd.Series(summarised_texts).to_csv("summarised_texts_out.csv")
-
-        if text_df.empty:
-            #if model_type != "Mistral Nous Capybara 4k (larger, slow)":
-            summarised_text_out = summarised_texts[0]#.values()
-
-            #if model_type == "Mistral Nous Capybara 4k (larger, slow)":
-            #    summarised_text_out = summarised_texts[0]
-
-        else: 
-            summarised_text_out = summarised_texts #[d['summary_text'] for d in summarised_texts] #summarised_text[0].values()
-
-        output_name = "summarise_output_" + today_rev + ".csv"
-        output_df = pd.DataFrame({"Original text":in_text_df[in_colname_list_first],
-                                    "Summarised text":summarised_text_out})
-
-        summarised_text_out_str = str(output_df["Summarised text"][0])#.str.replace("dict_values([","").str.replace("])",""))
-
-        output_df.to_csv(output_name, index = None)
-
-        return summarised_text_out_str, output_name
 
 # ## Gradio app - summarise
 block = gr.Blocks(theme = gr.themes.Base())
@@ -261,21 +133,21 @@ with block:
     gr.Markdown(
     """
     # Text summariser
-    Enter open text below to get a summary. You can copy and paste text directly, or upload a file and specify the column that you want to summarise. The default small model will be able to summarise up to about 16,000 words, but the quality may not be great. The larger model around 900 words of better quality. Summarisation with Mistral Nous Capybara 4k works on up to around 4,000 words, and may give a higher quality summary, but will be slow, and it may not respect your desired maximum word count.
+    Enter open text below to get a summary. You can copy and paste text directly, or upload a file and specify the column that you want to summarise. The default small model will be able to summarise up to about 16,000 words, but the quality may not be great. The larger model around 900 words of better quality. Summarisation with Phi 3 128k works on up to around 4,000 words, and may give a higher quality summary, but will be slow, and it may not respect your desired maximum word count.
     """)    
     
     with gr.Tab("Summariser"):
         current_model = gr.Textbox(label="Current model", value=model_type, scale = 3)
 
-        with gr.Accordion("Paste open text", open = False):
-            in_text = gr.Textbox(label="Copy and paste your open text here", lines = 5)
-            
-        with gr.Accordion("Summarise open text from a file", open = False):
+        with gr.Accordion("Summarise open text from a file", open = True):
             in_text_df = gr.File(label="Input text from file", file_count='multiple')
             in_colname = gr.Dropdown(label="Write the column name for the open text to summarise")
+
+        with gr.Accordion("Paste open text", open = False):
+            in_text = gr.Textbox(label="Copy and paste your open text here", lines = 5)
     
         with gr.Row():
-            summarise_btn = gr.Button("Summarise")
+            summarise_btn = gr.Button("Summarise", variant="primary")
             stop = gr.Button(value="Interrupt processing", variant="secondary", scale=0)
             length_slider = gr.Slider(minimum = 30, maximum = 500, value = 100, step = 10, label = "Maximum length of summary")
         
@@ -284,35 +156,104 @@ with block:
             output_file = gr.File(label="Output file")
 
     with gr.Tab("Advanced features"):
-        #out_passages = gr.Slider(minimum=1, value = 2, maximum=10, step=1, label="Choose number of passages to retrieve from the document. Numbers greater than 2 may lead to increased hallucinations or input text being truncated.")
-        #temp_slide = gr.Slider(minimum=0.1, value = 0.1, maximum=1, step=0.1, label="Choose temperature setting for response generation.")
         with gr.Row():
-            model_choice = gr.Radio(label="Choose a summariser model", value="Long T5 Global Base 16k Book Summary", choices = ["Long T5 Global Base 16k Book Summary", "Flan T5 Large Stacked Samsum 1k", "Mistral Nous Capybara 4k (larger, slow)"])
+            model_choice = gr.Radio(label="Choose a summariser model", value="Long T5 Global Base 16k Book Summary", choices = ["Long T5 Global Base 16k Book Summary", "Flan T5 Large Stacked Samsum 1k", "Phi 3 128k (larger, slow)"])
             change_model_button = gr.Button(value="Load model", scale=0)
         with gr.Accordion("Choose number of model layers to send to GPU (WARNING: please don't modify unless you are sure you have a GPU).", open = False):
             gpu_layer_choice = gr.Slider(label="Choose number of model layers to send to GPU.", value=0, minimum=0, maximum=100, step = 1, visible=True)
+        with gr.Accordion("LLM parameters"):
+            temp_slide = gr.Slider(minimum=0.1, value = 0.5, maximum=1, step=0.1, label="Choose temperature setting for response generation.")
 
         load_text = gr.Text(label="Load status")
 
-
      # Update dropdowns upon initial file load
     in_text_df.upload(put_columns_in_df, inputs=[in_text_df, in_colname], outputs=[in_colname, data_state])
 
     change_model_button.click(fn=load_model, inputs=[model_choice, gpu_layer_choice], outputs = [model_type_state, load_text, current_model])
 
     summarise_click = summarise_btn.click(fn=summarise_text, inputs=[in_text, data_state, length_slider, in_colname, model_type_state],
-                        outputs=[output_single_text, output_file], api_name="summarise_single_text")
-    summarise_enter = summarise_btn.click(fn=summarise_text, inputs=[in_text, data_state, length_slider, in_colname, model_type_state],
-                        outputs=[output_single_text, output_file])
+                       outputs=[output_single_text, output_file], api_name="summarise_single_text")
+    # summarise_enter = summarise_btn.submit(fn=summarise_text, inputs=[in_text, data_state, length_slider, in_colname, model_type_state],
+    #                    outputs=[output_single_text, output_file])
+    
+    #summarise_click = summarise_btn.click(chatf.llama_cpp_streaming, [chatbot, instruction_prompt_out, model_type_state, temp_slide], chatbot)
     
     # Stop processing if it's taking too long
-    stop.click(fn=None, inputs=None, outputs=None, cancels=[summarise_click, summarise_enter])
+    stop.click(fn=None, inputs=None, outputs=None, cancels=[summarise_click])
 
     # Dummy function to allow dropdown modification to work correctly (strange thing needed for Gradio 3.50, will be deprecated upon upgrading Gradio version)
     in_colname.change(dummy_function, in_colname, None)
 
-block.queue(concurrency_count=1).launch()
-# -
+block.queue().launch()
+
+# def load_model(model_type, gpu_layers, gpu_config=None, cpu_config=None, torch_device=None):
+#     print("Loading model ", model_type)
+
+#     # Default values inside the function
+#     if gpu_config is None:
+#         gpu_config = chatf.gpu_config
+#     if cpu_config is None:
+#         cpu_config = chatf.cpu_config
+#     if torch_device is None:
+#         torch_device = chatf.torch_device
+
+#     if model_type == "Phi 3 128k (larger, slow)":
+#         hf_checkpoint = 'NousResearch/Nous-Capybara-7B-V1.9-GGUF'
+
+#         if torch_device == "cuda":
+#             gpu_config.update_gpu(gpu_layers)
+#         else:
+#             gpu_config.update_gpu(gpu_layers)
+#             cpu_config.update_gpu(gpu_layers)
+
+#         print("Loading with", cpu_config.gpu_layers, "model layers sent to GPU.")
+
+#         print(vars(gpu_config))
+#         print(vars(cpu_config))
+
+#         try:
+#             #model = ctransformers.AutoModelForCausalLM.from_pretrained('Aryanne/Orca-Mini-3B-gguf', model_type='llama', model_file='q5_0-orca-mini-3b.gguf', **vars(gpu_config)) # **asdict(CtransRunConfig_cpu())
+#             #model = ctransformers.AutoModelForCausalLM.from_pretrained('Aryanne/Wizard-Orca-3B-gguf', model_type='llama', model_file='q4_1-wizard-orca-3b.gguf', **vars(gpu_config)) # **asdict(CtransRunConfig_cpu())
+#             #model = ctransformers.AutoModelForCausalLM.from_pretrained('TheBloke/Mistral-7B-OpenOrca-GGUF', model_type='mistral', model_file='mistral-7b-openorca.Q4_K_M.gguf', **vars(gpu_config), hf=True) # **asdict(CtransRunConfig_cpu())
+#             #model = ctransformers.AutoModelForCausalLM.from_pretrained('TheBloke/OpenHermes-2.5-Mistral-7B-16k-GGUF', model_type='mistral', model_file='openhermes-2.5-mistral-7b-16k.Q4_K_M.gguf', **vars(gpu_config), hf=True) # **asdict(CtransRunConfig_cpu())
+#             model = ctransformers.AutoModelForCausalLM.from_pretrained('NousResearch/Nous-Capybara-7B-V1.9-GGUF', model_type='mistral', model_file='Capybara-7B-V1.9-Q5_K_M.gguf', **vars(gpu_config), hf=True) # **asdict(CtransRunConfig_cpu())
+
+
+#             tokenizer = AutoTokenizer.from_pretrained("NousResearch/Nous-Capybara-7B-V1.9")
+#             summariser = pipeline("text-generation", model=model, tokenizer=tokenizer)
+
+#         except:
+#             #model = ctransformers.AutoModelForCausalLM.from_pretrained('Aryanne/Orca-Mini-3B-gguf', model_type='llama', model_file='q5_0-orca-mini-3b.gguf', **vars(cpu_config)) #**asdict(CtransRunConfig_gpu())
+#             #model = ctransformers.AutoModelForCausalLM.from_pretrained('Aryanne/Wizard-Orca-3B-gguf', model_type='llama', model_file='q4_1-wizard-orca-3b.gguf', **vars(cpu_config)) # **asdict(CtransRunConfig_cpu())
+#             #model = ctransformers.AutoModelForCausalLM.from_pretrained('TheBloke/Mistral-7B-OpenOrca-GGUF', model_type='mistral', model_file='mistral-7b-openorca.Q4_K_M.gguf', **vars(cpu_config), hf=True) # **asdict(CtransRunConfig_cpu())
+#             #model = ctransformers.AutoModelForCausalLM.from_pretrained('TheBloke/OpenHermes-2.5-Mistral-7B-16k-GGUF', model_type='mistral', model_file='openhermes-2.5-mistral-7b-16k.Q4_K_M.gguf', **vars(gpu_config), hf=True) # **asdict(CtransRunConfig_cpu())
+#             model = ctransformers.AutoModelForCausalLM.from_pretrained('NousResearch/Nous-Capybara-7B-V1.9-GGUF', model_type='mistral', model_file='Capybara-7B-V1.9-Q5_K_M.gguf', **vars(gpu_config), hf=True) # **asdict(CtransRunConfig_cpu())
+            
+#             #tokenizer = ctransformers.AutoTokenizer.from_pretrained(model)
+
+#             tokenizer = AutoTokenizer.from_pretrained("NousResearch/Nous-Capybara-7B-V1.9")
+#             summariser = pipeline("text-generation", model=model, tokenizer=tokenizer) # model
+
+#         #model = []
+#         #tokenizer = []
+#         #summariser = []
+
+#     if model_type == "Flan T5 Large Stacked Samsum 1k":
+#         # Huggingface chat model
+#         hf_checkpoint = 'stacked-summaries/flan-t5-large-stacked-samsum-1024'#'declare-lab/flan-alpaca-base' # # #
+
+#         summariser, tokenizer, model_type = create_hf_model(model_name = hf_checkpoint)
+
+#     if model_type == "Long T5 Global Base 16k Book Summary":
+#         # Huggingface chat model
+#         hf_checkpoint = 'pszemraj/long-t5-tglobal-base-16384-book-summary' #'philschmid/flan-t5-small-stacked-samsum'#'declare-lab/flan-alpaca-base' # # #
+#         summariser, tokenizer, model_type = create_hf_model(model_name = hf_checkpoint)
 
+#     chatf.model = summariser
+#     chatf.tokenizer = tokenizer
+#     chatf.model_type = model_type
 
+#     load_confirmation = "Finished loading model: " + model_type
 
+#     print(load_confirmation)
+#     return model_type, load_confirmation, model_type

chatfuncs/chatfuncs.py

@@ -4,6 +4,7 @@ from typing import TypeVar
 # Model packages
 import torch.cuda
 from transformers import pipeline
+import time
 
 torch.cuda.empty_cache()
 
@@ -46,41 +47,29 @@ sample = True
 
 
 class CtransInitConfig_gpu:
-    def __init__(self, temperature=temperature,
-                 top_k=top_k,
-                 top_p=top_p,
-                 repetition_penalty=repetition_penalty,
+    def __init__(self,
                  last_n_tokens=last_n_tokens,
-                 max_new_tokens=max_new_tokens,
                  seed=seed,
-                 reset=reset,
-                 stream=stream,
-                 threads=threads,
-                 batch_size=batch_size,
-                 context_length=context_length,
-                 gpu_layers=gpu_layers):
-        self.temperature = temperature
-        self.top_k = top_k
-        self.top_p = top_p
-        self.repetition_penalty = repetition_penalty# repetition_penalty
+                 n_threads=threads,
+                 n_batch=batch_size,
+                 n_ctx=32768,
+                 n_gpu_layers=gpu_layers):
+
         self.last_n_tokens = last_n_tokens
-        self.max_new_tokens = max_new_tokens
         self.seed = seed
-        self.reset = reset
-        self.stream = True #stream
-        self.threads = threads
-        self.batch_size = batch_size
-        self.context_length = context_length
-        self.gpu_layers = gpu_layers
+        self.n_threads = n_threads
+        self.n_batch = n_batch
+        self.n_ctx = n_ctx
+        self.n_gpu_layers = n_gpu_layers
         # self.stop: list[str] = field(default_factory=lambda: [stop_string])
 
     def update_gpu(self, new_value):
-        self.gpu_layers = new_value
+        self.n_gpu_layers = new_value
 
 class CtransInitConfig_cpu(CtransInitConfig_gpu):
     def __init__(self):
         super().__init__()
-        self.gpu_layers = 0
+        self.n_gpu_layers = 0
 
 gpu_config = CtransInitConfig_gpu()
 cpu_config = CtransInitConfig_cpu()
@@ -90,22 +79,68 @@ class CtransGenGenerationConfig:
     def __init__(self, temperature=temperature,
                  top_k=top_k,
                  top_p=top_p,
-                 repetition_penalty=repetition_penalty,
-                 last_n_tokens=last_n_tokens,
+                 repeat_penalty=repetition_penalty,
                  seed=seed,
-                 threads=threads,
-                 batch_size=batch_size,
-                 reset=True
+                 stream=stream,
+                 max_tokens=max_new_tokens
                  ):
         self.temperature = temperature
         self.top_k = top_k
         self.top_p = top_p
-        self.repetition_penalty = repetition_penalty# repetition_penalty
-        self.last_n_tokens = last_n_tokens
+        self.repeat_penalty = repeat_penalty
         self.seed = seed
-        self.threads = threads
-        self.batch_size = batch_size
-        self.reset = reset
+        self.max_tokens=max_tokens
+        self.stream = stream
 
     def update_temp(self, new_value):
-        self.temperature = new_value
+        self.temperature = new_value
+
+
+def llama_cpp_streaming(history, full_prompt, model_type,
+        temperature=temperature,
+        max_new_tokens=max_new_tokens,
+        sample=sample,
+        repetition_penalty=repetition_penalty,
+        top_p=top_p,
+        top_k=top_k
+):
+#print("Model type is: ", model_type)
+
+#if not full_prompt.strip():
+#    if history is None:
+#        history = []
+
+#    return history
+
+        #tokens = model.tokenize(full_prompt)
+
+    gen_config = CtransGenGenerationConfig()
+    gen_config.update_temp(temperature)
+
+    print(vars(gen_config))
+
+    # Pull the generated text from the streamer, and update the model output.
+    start = time.time()
+    NUM_TOKENS=0
+    print('-'*4+'Start Generation'+'-'*4)
+
+    output = model(
+    full_prompt, **vars(gen_config))
+
+    history[-1][1] = ""
+    for out in output:
+
+        if "choices" in out and len(out["choices"]) > 0 and "text" in out["choices"][0]:
+            history[-1][1] += out["choices"][0]["text"]
+            NUM_TOKENS+=1
+            yield history
+        else:
+            print(f"Unexpected output structure: {out}") 
+
+    time_generate = time.time() - start
+    print('\n')
+    print('-'*4+'End Generation'+'-'*4)
+    print(f'Num of generated tokens: {NUM_TOKENS}')
+    print(f'Time for complete generation: {time_generate}s')
+    print(f'Tokens per secound: {NUM_TOKENS/time_generate}')
+    print(f'Time per token: {(time_generate/NUM_TOKENS)*1000}ms')

chatfuncs/helper_functions.py

@@ -73,11 +73,11 @@ def read_file(filename):
     print("Loading in file")
 
     if file_type == 'csv':
-        file = pd.read_csv(filename, low_memory=False).reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
+        file = pd.read_csv(filename, low_memory=False).reset_index(drop=True).drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
     elif file_type == 'xlsx':
-        file = pd.read_excel(filename).reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
+        file = pd.read_excel(filename).reset_index(drop=True).drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
     elif file_type == 'parquet':
-        file = pd.read_parquet(filename).reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
+        file = pd.read_parquet(filename).reset_index(drop=True).drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
     elif file_type == 'pkl.gz':
         with gzip.open(filename, 'rb') as file:
             file = pickle.load(file)

chatfuncs/prompts.py

@@ -27,4 +27,14 @@ In 1994, when a major affiliation switch threatened to leave CBS without an affi
 ASSISTANT:"""
 
 nous_capybara_prompt = """USER:Summarise the following text in less than {length} words "{text}"
-ASSISTANT:"""
+ASSISTANT:"""
+
+instruction_prompt_phi3 = """<|user|>\n
+Summarise the following text in less than {length} words: "{text}"\n
+Summary:<|end|>\n
+<|assistant|>"""
+
+instruction_prompt_llama3 = """<|start_header_id|>system<|end_header_id|>\n
+You are an AI assistant that follows instruction extremely well. Help as much as you can.<|eot_id|><|start_header_id|>user<|end_header_id|>\n
+Summarise the following text in less than {length} words: "{text}"\n
+Summary:<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n"""

chatfuncs/summarise_funcs.py

@@ -0,0 +1,268 @@
+import pandas as pd
+import concurrent.futures
+import gradio as gr
+from chatfuncs.chatfuncs import model, CtransGenGenerationConfig, temperature
+from datetime import datetime
+
+today = datetime.now().strftime("%d%m%Y")
+today_rev = datetime.now().strftime("%Y%m%d")
+
+def summarise_text(text, text_df, length_slider, in_colname, model_type, progress=gr.Progress()):      
+         
+        if text_df.empty:
+            in_colname="text"
+            in_colname_list_first = in_colname
+
+            in_text_df = pd.DataFrame({in_colname_list_first:[text]})
+            
+        else: 
+            in_text_df = text_df
+            in_colname_list_first = in_colname
+
+        print(model_type)
+
+        texts_list = list(in_text_df[in_colname_list_first])
+
+        if model_type != "Phi 3 128k (larger, slow)":
+            summarised_texts = []
+
+            for single_text in progress.tqdm(texts_list, desc = "Summarising texts", unit = "texts"):
+
+                summarised_text = model(single_text, max_length=length_slider)
+
+                #print(summarised_text)
+
+                summarised_text_str = summarised_text[0]['summary_text']
+
+                summarised_texts.append(summarised_text_str)
+
+                print(summarised_text_str)
+
+                #pd.Series(summarised_texts).to_csv("summarised_texts_out.csv")
+
+            #print(summarised_texts)
+
+        if model_type == "Phi 3 128k (larger, slow)":
+
+            gen_config = CtransGenGenerationConfig()
+            gen_config.update_temp(temperature)
+
+            print(gen_config)
+
+            # Define a function that calls your model
+            # def call_model(formatted_string):#, vars):
+            #     return model(formatted_string)#, vars)
+            
+            def call_model(formatted_string, gen_config):
+                """
+                Calls your generation model with parameters from the CtransGenGenerationConfig object.
+
+                Args:
+                    formatted_string (str): The formatted input text for the model.
+                    gen_config (CtransGenGenerationConfig): An object containing generation parameters.
+                """
+                # Extracting parameters from the gen_config object
+                temperature = gen_config.temperature
+                top_k = gen_config.top_k
+                top_p = gen_config.top_p
+                repeat_penalty = gen_config.repeat_penalty
+                seed = gen_config.seed
+                max_tokens = gen_config.max_tokens
+                stream = gen_config.stream
+
+                # Now you can call your model directly, passing the parameters:
+                output = model(
+                    formatted_string, 
+                    temperature=temperature, 
+                    top_k=top_k, 
+                    top_p=top_p, 
+                    repeat_penalty=repeat_penalty, 
+                    seed=seed,
+                    max_tokens=max_tokens,
+                    stream=stream,
+                )
+
+                return output
+
+            # Set your timeout duration (in seconds)
+            timeout_duration = 300  # Adjust this value as needed
+
+            length = str(length_slider)
+
+            from chatfuncs.prompts import instruction_prompt_phi3
+
+            summarised_texts = []
+
+            for single_text in progress.tqdm(texts_list, desc = "Summarising texts", unit = "texts"):
+
+                formatted_string = instruction_prompt_phi3.format(length=length, text=single_text)
+
+                # Use ThreadPoolExecutor to enforce a timeout
+                with concurrent.futures.ThreadPoolExecutor() as executor:
+                    #future = executor.submit(call_model, formatted_string)#, **vars(gen_config))
+                    future = executor.submit(call_model, formatted_string, gen_config)
+                    try:
+                        output = future.result(timeout=timeout_duration)
+                        # Process the output here
+                    except concurrent.futures.TimeoutError:
+                        error_text = f"Timeout (five minutes) occurred for text: {single_text}. Consider using a smaller model."
+                        print(error_text)
+                        return error_text, None
+
+                print(output)
+
+                output_str = output['choices'][0]['text']
+
+                # Find the index of 'ASSISTANT: ' to select only text after this location
+                # index = output_str.find('ASSISTANT: ')
+
+                # # Check if 'ASSISTANT: ' is found in the string
+                # if index != -1:
+                #     # Add the length of 'ASSISTANT: ' to the index to start from the end of this substring
+                #     start_index = index + len('ASSISTANT: ')
+                    
+                #     # Slice the string from this point to the end
+                #     assistant_text = output_str[start_index:]
+                # else:
+                #     assistant_text = "ASSISTANT: not found in text"
+
+                # print(assistant_text)
+
+                #summarised_texts.append(assistant_text)
+
+                summarised_texts.append(output_str)
+
+                #print(summarised_text)
+                
+                #pd.Series(summarised_texts).to_csv("summarised_texts_out.csv")
+
+        if text_df.empty:
+            #if model_type != "Phi 3 128k (larger, slow)":
+            summarised_text_out = summarised_texts[0]#.values()
+
+            #if model_type == "Phi 3 128k (larger, slow)":
+            #    summarised_text_out = summarised_texts[0]
+
+        else: 
+            summarised_text_out = summarised_texts #[d['summary_text'] for d in summarised_texts] #summarised_text[0].values()
+
+        output_name = "summarise_output_" + today_rev + ".csv"
+        output_df = pd.DataFrame({"Original text":in_text_df[in_colname_list_first],
+                                    "Summarised text":summarised_text_out})
+
+        summarised_text_out_str = str(output_df["Summarised text"][0])#.str.replace("dict_values([","").str.replace("])",""))
+
+        output_df.to_csv(output_name, index = None)
+
+        return summarised_text_out_str, output_name
+
+
+# def summarise_text(text, text_df, length_slider, in_colname, model_type, progress=gr.Progress()):      
+         
+#         if text_df.empty:
+#             in_colname="text"
+#             in_colname_list_first = in_colname
+
+#             in_text_df = pd.DataFrame({in_colname_list_first:[text]})
+            
+#         else: 
+#             in_text_df = text_df
+#             in_colname_list_first = in_colname
+
+#         print(model_type)
+
+#         texts_list = list(in_text_df[in_colname_list_first])
+
+#         if model_type != "Phi 3 128k (larger, slow)":
+#             summarised_texts = []
+
+#             for single_text in progress.tqdm(texts_list, desc = "Summarising texts", unit = "texts"):
+#                 summarised_text = chatf.model(single_text, max_length=length_slider)
+
+#                 #print(summarised_text)
+
+#                 summarised_text_str = summarised_text[0]['summary_text']
+
+#                 summarised_texts.append(summarised_text_str)
+
+#                 print(summarised_text_str)
+
+#                 #pd.Series(summarised_texts).to_csv("summarised_texts_out.csv")
+
+#             #print(summarised_texts)
+
+#         if model_type == "Phi 3 128k (larger, slow)":
+
+
+#             # Define a function that calls your model
+#             def call_model(formatted_string, max_length=10000):
+#                 return chatf.model(formatted_string, max_length=max_length)
+
+#             # Set your timeout duration (in seconds)
+#             timeout_duration = 300  # Adjust this value as needed
+
+#             length = str(length_slider)
+
+#             from chatfuncs.prompts import nous_capybara_prompt
+
+#             summarised_texts = []
+
+#             for single_text in progress.tqdm(texts_list, desc = "Summarising texts", unit = "texts"):
+
+#                 formatted_string = nous_capybara_prompt.format(length=length, text=single_text)
+
+#                 # Use ThreadPoolExecutor to enforce a timeout
+#                 with concurrent.futures.ThreadPoolExecutor() as executor:
+#                     future = executor.submit(call_model, formatted_string, 10000)
+#                     try:
+#                         output = future.result(timeout=timeout_duration)
+#                         # Process the output here
+#                     except concurrent.futures.TimeoutError:
+#                         error_text = f"Timeout (five minutes) occurred for text: {single_text}. Consider using a smaller model."
+#                         print(error_text)
+#                         return error_text, None
+
+#                 print(output)
+
+#                 output_str = output[0]['generated_text']
+
+#                 # Find the index of 'ASSISTANT: ' to select only text after this location
+#                 index = output_str.find('ASSISTANT: ')
+
+#                 # Check if 'ASSISTANT: ' is found in the string
+#                 if index != -1:
+#                     # Add the length of 'ASSISTANT: ' to the index to start from the end of this substring
+#                     start_index = index + len('ASSISTANT: ')
+                    
+#                     # Slice the string from this point to the end
+#                     assistant_text = output_str[start_index:]
+#                 else:
+#                     assistant_text = "ASSISTANT: not found in text"
+
+#                 print(assistant_text)
+
+#                 summarised_texts.append(assistant_text)
+
+#                 #print(summarised_text)
+                
+#                 #pd.Series(summarised_texts).to_csv("summarised_texts_out.csv")
+
+#         if text_df.empty:
+#             #if model_type != "Phi 3 128k (larger, slow)":
+#             summarised_text_out = summarised_texts[0]#.values()
+
+#             #if model_type == "Phi 3 128k (larger, slow)":
+#             #    summarised_text_out = summarised_texts[0]
+
+#         else: 
+#             summarised_text_out = summarised_texts #[d['summary_text'] for d in summarised_texts] #summarised_text[0].values()
+
+#         output_name = "summarise_output_" + today_rev + ".csv"
+#         output_df = pd.DataFrame({"Original text":in_text_df[in_colname_list_first],
+#                                     "Summarised text":summarised_text_out})
+
+#         summarised_text_out_str = str(output_df["Summarised text"][0])#.str.replace("dict_values([","").str.replace("])",""))
+
+#         output_df.to_csv(output_name, index = None)
+
+#         return summarised_text_out_str, output_name

requirements.txt

@@ -1,6 +1,6 @@
-gradio==3.50.0
+gradio==4.36.0
 transformers
-torch
 pyarrow
 openpyxl
-ctransformers[cuda]
+llama-cpp-python==0.2.77 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121
+torch==2.3.1 --extra-index-url https://download.pytorch.org/whl/cu121