|
|
|
import re |
|
import nltk |
|
import torch |
|
import spacy |
|
import numpy as np |
|
import math |
|
import gradio as gr |
|
|
|
from nltk.tokenize import sent_tokenize |
|
from transformers import pipeline |
|
nltk.download('punkt') |
|
|
|
|
|
def clean_text(text): |
|
text = text.encode("ascii", errors="ignore").decode( |
|
"ascii" |
|
) |
|
|
|
text = re.sub(r"\n", " ", text) |
|
text = re.sub(r"\n\n", " ", text) |
|
text = re.sub(r"\t", " ", text) |
|
text = text.strip(" ") |
|
text = re.sub( |
|
" +", " ", text |
|
).strip() |
|
return text |
|
|
|
|
|
from transformers import BartTokenizer, BartForConditionalGeneration |
|
model = BartForConditionalGeneration.from_pretrained("sshleifer/distilbart-cnn-12-6") |
|
tokenizer = BartTokenizer.from_pretrained("sshleifer/distilbart-cnn-12-6") |
|
nlp = spacy.load("en_core_web_sm") |
|
|
|
|
|
|
|
def final_summary(text): |
|
|
|
text = clean_text(text) |
|
|
|
chunks = [] |
|
sentences = nlp(text) |
|
for sentence in sentences.sents: |
|
chunks.append(str(sentence)) |
|
|
|
output = [] |
|
sentences_remaining = len(chunks) |
|
i = 0 |
|
|
|
|
|
while sentences_remaining > 0: |
|
chunks_remaining = math.ceil(sentences_remaining / 10.0) |
|
next_chunk_size = math.ceil(sentences_remaining / chunks_remaining) |
|
sentence = "".join(chunks[i:i+next_chunk_size]) |
|
|
|
i += next_chunk_size |
|
sentences_remaining -= next_chunk_size |
|
|
|
inputs = tokenizer(sentence, return_tensors="pt", padding="longest") |
|
|
|
original_input_length = len(inputs["input_ids"][0]) |
|
|
|
|
|
if original_input_length < 100: |
|
output.append(sentence) |
|
|
|
|
|
|
|
elif original_input_length > 1024: |
|
sent = sent_tokenize(sentence) |
|
length_sent = len(sent) |
|
|
|
j = 0 |
|
sent_remaining = math.ceil(length_sent / 2) |
|
|
|
|
|
while length_sent > 0: |
|
halved_sentence = "".join(sent[j:j+sent_remaining]) |
|
halved_inputs = tokenizer(halved_sentence, return_tensors="pt") |
|
|
|
halved_summary_ids = model.generate(halved_inputs["input_ids"]) |
|
j += sent_remaining |
|
length_sent -= sent_remaining |
|
|
|
|
|
if len(halved_summary_ids[0]) < len(halved_inputs["input_ids"][0]): |
|
halved_summary = tokenizer.batch_decode(halved_summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] |
|
output.append(halved_summary) |
|
|
|
else: |
|
summary_ids = model.generate(inputs["input_ids"]) |
|
|
|
if len(summary_ids[0]) < original_input_length: |
|
summary = tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] |
|
output.append(summary) |
|
|
|
|
|
|
|
|
|
|
|
lines = [] |
|
for summary in output: |
|
summary = nlp(summary) |
|
for line in summary.sents: |
|
line = str(line) |
|
if line != " ": |
|
lines.append(line.replace(" .", ".").strip()) |
|
|
|
for i in range(len(lines)): |
|
lines[i] = "* " + lines[i] |
|
|
|
|
|
summary_bullet = "\n".join(lines) |
|
|
|
return summary_bullet |
|
|
|
|
|
|
|
|
|
demo = gr.Interface(final_summary, inputs=[gr.Textbox(label="Drop your article here")], |
|
title = "ARTICLE SUMMARIZER", |
|
outputs=[gr.Textbox(label="Summary")], |
|
theme= "darkhuggingface") |
|
|
|
|
|
if __name__ == "__main__": |
|
demo.launch(debug=True) |