Spaces:
Runtime error
Runtime error
File size: 4,806 Bytes
86756d8 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 |
"""
This module is for text summarization
"""
# ref: https://towardsdatascience.com/understand-text-summarization-and-create-your-own-summarizer-in-python-b26a9f09fc70
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
from nltk.cluster.util import cosine_distance
import pandas as pd
import numpy as np
import networkx as nx
class SummarizationClass:
def read_text(text):
text = text.replace("\"","")
article = text.split(". ")
sentences = []
for sentence in article:
#print(sentence)
sentences.append(sentence.replace("[^a-zA-Z]", " ").split(" "))
#sentences.pop()
return sentences
def sentence_similarity(sent1, sent2, stopwords=None):
if stopwords is None:
stopwords = []
sent1 = [w.lower() for w in sent1]
sent2 = [w.lower() for w in sent2]
all_words = list(set(sent1 + sent2))
vector1 = [0] * len(all_words)
vector2 = [0] * len(all_words)
# build the vector for the first sentence
for w in sent1:
if w in stopwords:
continue
vector1[all_words.index(w)] += 1
# build the vector for the second sentence
for w in sent2:
if w in stopwords:
continue
vector2[all_words.index(w)] += 1
return 1 - cosine_distance(vector1, vector2)
def build_similarity_matrix(sentences, stop_words):
# Create an empty similarity matrix
similarity_matrix = np.zeros((len(sentences), len(sentences)))
for idx1 in range(len(sentences)):
for idx2 in range(len(sentences)):
if idx1 == idx2: #ignore if both are same sentences
continue
similarity_matrix[idx1][idx2] = SummarizationClass.sentence_similarity(sentences[idx1], sentences[idx2], stop_words)
return similarity_matrix
def generate_summary(text, top_n=5):
stop_words = stopwords.words('english')
summarize_text = []
# Step 1 - Read text anc split it
sentences = SummarizationClass.read_text(text)
# Step 2 - Generate Similary Martix across sentences
sentence_similarity_martix = SummarizationClass.build_similarity_matrix(sentences, stop_words)
# Step 3 - Rank sentences in similarity martix
sentence_similarity_graph = nx.from_numpy_array(sentence_similarity_martix)
#print(sentence_similarity_graph)
try:
scores = nx.pagerank(sentence_similarity_graph)
# Step 4 - Sort the rank and pick top sentences
ranked_sentence = sorted(((scores[i],s) for i,s in enumerate(sentences)), reverse=True)
for i in range(top_n):
summarize_text.append(" ".join(ranked_sentence[i][1]))
except nx.exception.PowerIterationFailedConvergence:
print(f'text={text} was bad for nx')
return ''
# Step 5 - Offcourse, output the summarize texr
return ". ".join(summarize_text)
class SummarizationClassRun:
''' class for running the summarization class algorithm with given parameters '''
def __init__(self,input_file_path,text_column,output_file_path_keep_original_text_column):
self.input_file_path = input_file_path
self.text_column = text_column
self.output_file_path_keep_original_text_column = output_file_path_keep_original_text_column
self.output_file_path_override_text_column = output_file_path_override_text_column
def run(self):
# read input file as a dataframe
df = pd.read_csv(self.input_file_path,encoding='cp1255')
# add column with summarization of the text in the text column
df['summarized_text'] = df[self.text_column].apply(lambda x: SummarizationClass.generate_summary(x, 1))
# export output with the original text column to CSV file
df.to_csv(self.output_file_path_keep_original_text_column,encoding='cp1255',index=False)
# override original text column
df[self.text_column] = df['summarized_text']
del df['summarized_text']
# export output with the overridden text column to CSV file
df.to_csv(self.output_file_path_override_text_column,encoding='cp1255',index=False)
if __name__ == '__main__':
# Arguments
INPUT_FILE_PATH = 'tagging_MMD_db.csv'
TEXT_COLUMN = 'text'
OUTPUT_FILE_PATH_KEEP_ORIGINAL_TEXT_COLUMN = 'tagging_MMD_db_with_summarized.csv'
OUTPUT_FILE_PATH_OVERRIDE_TEXT_COLUMN = 'summarized_tagging_MMD_db.csv'
obj = SummarizationClassRun(INPUT_FILE_PATH,TEXT_COLUMN,OUTPUT_FILE_PATH_KEEP_ORIGINAL_TEXT_COLUMN,OUTPUT_FILE_PATH_OVERRIDE_TEXT_COLUMN)
obj.run() |