lexrank1

app.py

@@ -0,0 +1,41 @@
+import gradio as gr
+import os
+import lexrank as lr
+import nltk
+import metrics
+import pandas as pd
+
+
+def summarize(in_text):
+
+    if len(in_text)==0:
+        return 'Error: No text provided', None
+
+    nltk_file = '/Users/hujo/nltk_data/tokenizers/punkt.zip'
+    #nltk_file = '/home/user/nltk_data/tokenizers/punkt.zip'
+    if os.path.exists(nltk_file):
+        print('nltk punkt file exists in ', nltk_file)
+    else:
+        nltk.download('punkt')
+
+    out_text = lr.get_Summary(in_text)
+    n_words = metrics.num_words(out_text)
+    n_sents = metrics.num_sentences(out_text)
+    n_chars = metrics.num_chars(out_text)
+    n_tokens= metrics.num_tokens(out_text)
+
+    return out_text, n_words, n_sents, n_chars, n_tokens
+
+
+demo = gr.Interface(summarize, 
+       inputs=["text"] , 
+       outputs=[gr.Textbox(label="Extractive Summary"), 
+                gr.Number(label="Number of Words"),
+                gr.Number(label="Number of Sentences"),
+                gr.Number(label="Number of Characters"),
+                gr.Number(label="Number of Tokens")],
+       allow_flagging="never")
+
+
+if __name__ == "__main__":
+    demo.launch()

lexrank.py

@@ -0,0 +1,39 @@
+#import nltk
+#nltk.download('punkt')
+
+from sumy.parsers.html import HtmlParser
+from sumy.parsers.plaintext import PlaintextParser
+from sumy.nlp.tokenizers import Tokenizer
+from sumy.summarizers.lex_rank import LexRankSummarizer
+from sumy.nlp.stemmers import Stemmer
+from sumy.utils import get_stop_words
+
+def get_Summary(in_text):
+    
+    sentences = in_text.split('. ')
+    # summarize small part of the text
+    nr_sentences = 3 #len(sentences)
+    print('nr_sentences: '+str(nr_sentences))
+    
+    if nr_sentences == 0:
+        return 'Error: No sentences available', None 
+    list_summary = get_Lexrank(in_text,nr_sentences)
+    # it can happen that for lexrank a sentence consists of multiple actual sentences, 
+    # that are separated with full stops. Then the correspoinding timestamp cannot be found
+    # all items from the lexrank summary must be concatinated and split up by full stops.
+    concat_list_summary = '. '.join([str(item).replace('.','') for item in list_summary])#.split('. ')
+    
+    return concat_list_summary
+
+def get_Lexrank(text, nr_sentences):
+    summary=[]
+    LANGUAGE = "english"
+    SENTENCES_COUNT = nr_sentences
+    parser = PlaintextParser.from_string(text, Tokenizer(LANGUAGE))
+    stemmer = Stemmer(LANGUAGE)
+    summarizer = LexRankSummarizer(stemmer)
+    summarizer.stop_words = get_stop_words(LANGUAGE)
+    for sentence in summarizer(parser.document, SENTENCES_COUNT):
+        summary.append(sentence)
+
+    return summary

metrics.py

@@ -0,0 +1,60 @@
+# Import nltk library for natural language processing
+import nltk
+
+
+# Define a function that takes some text as input and returns the number of tokens
+def token_count(text):
+    # Import the Encoder class from bpe
+    from bpe import Encoder
+    # Create an encoder object with a vocabulary size of 10
+    encoder = Encoder(vocab_size=14735746)
+
+    # Train the encoder on the text
+    encoder.fit(text.split())
+
+    # Encode the text into tokens
+    tokens = encoder.tokenize(text)
+
+    # Return the number of tokens
+    return tokens
+
+def num_tokens(text):
+    from transformers import AutoTokenizer
+
+    tokenizer = AutoTokenizer.from_pretrained("gpt2")
+
+    token_ids = tokenizer.encode(text)
+
+    token_size = len(token_ids)
+
+    return token_size
+
+def num_words(text):
+    sentences = nltk.sent_tokenize(text)
+    # Tokenize each sentence into words using nltk.word_tokenize()
+    words = []
+    for sentence in sentences:
+        words.extend(nltk.word_tokenize(sentence))
+    
+    num_words = len(words)
+
+    return num_words
+
+def num_sentences(text):
+    # Tokenize the text into sentences using nltk.sent_tokenize()
+    sentences = nltk.sent_tokenize(text)
+    num_sentences = len(sentences)
+    return num_sentences
+
+
+def num_chars(text):
+    num_characters = len(text)
+    return num_characters
+
+
+# Print out the results
+# print(f"Number of sentences: {num_sentences}")
+# print(f"Number of words: {num_words}")
+# print(f"Number of tokens: {num_tokens}")
+# print(f"Number of trans_tokens: {trans_tokens}")
+# print(f"Number of characters: {num_characters}")