Spaces:
Runtime error
Runtime error
| """ | |
| This module is for text summarization | |
| """ | |
| # ref: https://towardsdatascience.com/understand-text-summarization-and-create-your-own-summarizer-in-python-b26a9f09fc70 | |
| import nltk | |
| nltk.download('stopwords') | |
| from nltk.corpus import stopwords | |
| from nltk.cluster.util import cosine_distance | |
| import pandas as pd | |
| import numpy as np | |
| import networkx as nx | |
| class SummarizationClass: | |
| def read_text(text): | |
| text = text.replace("\"","") | |
| article = text.split(". ") | |
| sentences = [] | |
| for sentence in article: | |
| #print(sentence) | |
| sentences.append(sentence.replace("[^a-zA-Z]", " ").split(" ")) | |
| #sentences.pop() | |
| return sentences | |
| def sentence_similarity(sent1, sent2, stopwords=None): | |
| if stopwords is None: | |
| stopwords = [] | |
| sent1 = [w.lower() for w in sent1] | |
| sent2 = [w.lower() for w in sent2] | |
| all_words = list(set(sent1 + sent2)) | |
| vector1 = [0] * len(all_words) | |
| vector2 = [0] * len(all_words) | |
| # build the vector for the first sentence | |
| for w in sent1: | |
| if w in stopwords: | |
| continue | |
| vector1[all_words.index(w)] += 1 | |
| # build the vector for the second sentence | |
| for w in sent2: | |
| if w in stopwords: | |
| continue | |
| vector2[all_words.index(w)] += 1 | |
| return 1 - cosine_distance(vector1, vector2) | |
| def build_similarity_matrix(sentences, stop_words): | |
| # Create an empty similarity matrix | |
| similarity_matrix = np.zeros((len(sentences), len(sentences))) | |
| for idx1 in range(len(sentences)): | |
| for idx2 in range(len(sentences)): | |
| if idx1 == idx2: #ignore if both are same sentences | |
| continue | |
| similarity_matrix[idx1][idx2] = SummarizationClass.sentence_similarity(sentences[idx1], sentences[idx2], stop_words) | |
| return similarity_matrix | |
| def generate_summary(text, top_n=5): | |
| stop_words = stopwords.words('english') | |
| summarize_text = [] | |
| # Step 1 - Read text anc split it | |
| sentences = SummarizationClass.read_text(text) | |
| # Step 2 - Generate Similary Martix across sentences | |
| sentence_similarity_martix = SummarizationClass.build_similarity_matrix(sentences, stop_words) | |
| # Step 3 - Rank sentences in similarity martix | |
| sentence_similarity_graph = nx.from_numpy_array(sentence_similarity_martix) | |
| #print(sentence_similarity_graph) | |
| try: | |
| scores = nx.pagerank(sentence_similarity_graph) | |
| # Step 4 - Sort the rank and pick top sentences | |
| ranked_sentence = sorted(((scores[i],s) for i,s in enumerate(sentences)), reverse=True) | |
| for i in range(top_n): | |
| summarize_text.append(" ".join(ranked_sentence[i][1])) | |
| except nx.exception.PowerIterationFailedConvergence: | |
| print(f'text={text} was bad for nx') | |
| return '' | |
| # Step 5 - Offcourse, output the summarize texr | |
| return ". ".join(summarize_text) | |
| class SummarizationClassRun: | |
| ''' class for running the summarization class algorithm with given parameters ''' | |
| def __init__(self,input_file_path,text_column,output_file_path_keep_original_text_column): | |
| self.input_file_path = input_file_path | |
| self.text_column = text_column | |
| self.output_file_path_keep_original_text_column = output_file_path_keep_original_text_column | |
| self.output_file_path_override_text_column = output_file_path_override_text_column | |
| def run(self): | |
| # read input file as a dataframe | |
| df = pd.read_csv(self.input_file_path,encoding='cp1255') | |
| # add column with summarization of the text in the text column | |
| df['summarized_text'] = df[self.text_column].apply(lambda x: SummarizationClass.generate_summary(x, 1)) | |
| # export output with the original text column to CSV file | |
| df.to_csv(self.output_file_path_keep_original_text_column,encoding='cp1255',index=False) | |
| # override original text column | |
| df[self.text_column] = df['summarized_text'] | |
| del df['summarized_text'] | |
| # export output with the overridden text column to CSV file | |
| df.to_csv(self.output_file_path_override_text_column,encoding='cp1255',index=False) | |
| if __name__ == '__main__': | |
| # Arguments | |
| INPUT_FILE_PATH = 'tagging_MMD_db.csv' | |
| TEXT_COLUMN = 'text' | |
| OUTPUT_FILE_PATH_KEEP_ORIGINAL_TEXT_COLUMN = 'tagging_MMD_db_with_summarized.csv' | |
| OUTPUT_FILE_PATH_OVERRIDE_TEXT_COLUMN = 'summarized_tagging_MMD_db.csv' | |
| obj = SummarizationClassRun(INPUT_FILE_PATH,TEXT_COLUMN,OUTPUT_FILE_PATH_KEEP_ORIGINAL_TEXT_COLUMN,OUTPUT_FILE_PATH_OVERRIDE_TEXT_COLUMN) | |
| obj.run() |