Update app.py

app.py

@@ -3,6 +3,7 @@ from goose3 import Goose
 from fake_useragent import UserAgent
 from bs4 import BeautifulSoup   
 from transformers import pipeline
+import re
 #from newsplease import NewsPlease
 import validators
 import streamlit as st
@@ -21,7 +22,15 @@ def article_text_extractor(url: str):
   return text
   
 
-  
+def preprocess_text(x):
+  x = x.encode("ascii", "ignore").decode()  # unicode
+  x = re.sub(r"https*\S+", " ", x)  # url
+  x = re.sub(r"@\S+", " ", x)  # mentions
+  x = re.sub(r"#\S+", " ", x)  # hastags
+  x = re.sub(r"\s{2,}", " ", x)  # over spaces
+  x = re.sub("[^.,!?A-Za-z0-9]+", " ", x)  # special charachters except .,!?
+
+  return x
   
 @st.cache(allow_output_mutation=True)
 def extractive_model():    
@@ -77,13 +86,13 @@ if is_url:
 summarize = st.button("Summarize")
 
 if summarize:
-  text_to_summarize = clean_text if is_url else plain_text
+  text_to_summarize = preprocess_text(clean_text) if is_url else preprocess_text(plain_text)
   
   with st.spinner(text="Loading Model and creating summary. This might take a few seconds depending on the length of your text..."):
     model =  model()
     #summarized_text = text_to_summarize if len(text_to_summarize) > 60 else ''.join(model(body, min_length=60))
-    min_ = min(150,len(text_to_summarize)/5)
-    max_ = min(800,len(text_to_summarize)/2)
+    min_ = min(100,len(text_to_summarize)/5)
+    max_ = min(600,len(text_to_summarize)/2)
     summarized_text = ''.join(model(text_to_summarize, min_length=min_,max_length=max_)) if summary_type == "Extractive" else model(text_to_summarize, min_length=min_,max_length=max_)[0]['summary_text']
     
   st.subheader("Original text")