update for T5

app.py

@@ -2,8 +2,7 @@ import torch
 import nltk
 import validators
 import streamlit as st
-from nltk.tokenize import sent_tokenize
-from transformers import pipeline, T5Tokenizer, T5ForConditionalGeneration
+from transformers import T5Tokenizer, T5ForConditionalGeneration
 
 # local modules
 from extractive_summarizer.model_processors import Summarizer
@@ -68,6 +67,7 @@ if __name__ == "__main__":
                 text_to_summarize = clean_txt
                 abs_tokenizer, abs_model = load_abs_model()
                 if not is_url:
+                    # list of chunks
                     text_to_summarize = preprocess_text_for_abstractive_summarization(
                         tokenizer=abs_tokenizer, text=clean_txt
                     )

src/abstractive_summarizer.py

@@ -5,13 +5,11 @@ from transformers import T5Tokenizer
 
 def abstractive_summarizer(tokenizer, model, text):
     # inputs to the model
-    inputs = [
-        tokenizer.encode(f"summarize: {chunk}", return_tensors="pt") for chunk in text
-    ]
+    inputs = [tokenizer(f"summarize: {chunk}", return_tensors="pt") for chunk in text]
     abs_summarized_text = []
     for input in inputs:
-        output = model.generate(**input)
-        tmp_sum = tokenizer.decode(*output, skip_special_tokens=True)
+        output = model.generate(input["input_ids"])
+        tmp_sum = tokenizer.decode(output[0], skip_special_tokens=True)
         abs_summarized_text.append(tmp_sum)
 
     abs_summarized_text = " ".join([summ for summ in abs_summarized_text])