Adding wordcloud and enchant

app.py

@@ -11,12 +11,13 @@ from fastai.text.all import *
 import gradio as gr
 import requests
 from bs4 import BeautifulSoup
-#import enchant
+import enchant
 import re
 import random
 from collections import Counter
 import hashlib
 import pickle
+from wordcloud import WordCloud
 
 
 # %% ../nbs/01_data.ipynb 8
@@ -83,19 +84,19 @@ class Webpage:
                     continue
                 self.text.append(p_text)
 
-    def clean_html_text(self, max_words, enchant_dict="", ignore=[], rx="[^a-zA-Z ]+", min_word_len=2):
+    def clean_html_text(self, max_words, enchant_dict="en_US", ignore=[], rx="[^a-zA-Z ]+", min_word_len=2):
         all_text = ' '.join(self.text).lower()
         regex_text = re.sub(rx,'',all_text).strip()
         split = regex_text.split()
         split = [word for word in split if word not in ignore]
-        #if enchant_dict != "": d = enchant.Dict(enchant_dict)
+        if enchant_dict != "": d = enchant.Dict(enchant_dict)
         for word in split:
             if len(self.cleaned_text) >= max_words: break
             if len(word) >= min_word_len:
                 if enchant_dict == "":
                     self.cleaned_text.append(word)
-                #elif d.check(word): 
-                #    self.cleaned_text.append(word)
+                elif d.check(word): 
+                    self.cleaned_text.append(word)
 
     def k_common_words(self, k=10, ignore=[]):
         if self.cleaned_text == "":
@@ -176,12 +177,59 @@ def predict(url):
         text = ' '.join(page.cleaned_text)
         with learn.no_bar(), learn.no_logging():
             pred,idx,probs = learn.predict(text)
-        return dict(zip(categories, map(float,probs)))
+        wordcloud = WordCloud(width = 800, height = 800,
+                background_color ='white',
+                min_font_size = 10).generate(text)
+                
+        # plot the WordCloud image                      
+        fig = plt.figure(figsize = (8, 8), facecolor = None)
+        plt.imshow(wordcloud)
+        plt.axis("off")
+        plt.tight_layout(pad = 0)
+        return (dict(zip(categories, map(float,probs))), fig)
 
 # %% ../nbs/02_app_gradio.ipynb 8
-text = gr.inputs.Textbox(1)
-label = gr.outputs.Label()
 examples = ['https://www.theskepticsguide.org/about','https://www.foxnews.com/opinion']
 
-intf = gr.Interface(fn=predict, inputs=text, outputs=label, examples=examples)
-intf.launch()
+pseudo_sources = ["http://www.ageofautism.com/",
+ "http://www.naturalnews.com", 
+ "https://foodbabe.com/starthere/",
+ "http://www.chopra.com",
+ "https://www.mercola.com/",
+ "https://www.history.com/",
+ "https://doctoroz.com/",
+ "https://www.disclose.tv/",
+ "https://nationalreport.net/",
+ "https://heartland.org/",
+ "https://www.dailymail.co.uk/",
+ "https://www.motherjones.com/"]
+
+science_sources = ["https://sciencebasedmedicine.org/",
+ "https://www.hopkinsmedicine.org/gim/research/method/ebm.html",
+ "https://www.bbc.com/news/science_and_environment",
+ "https://www.nature.com/",
+ "https://www.science.org/",
+ "https://www.snopes.com/top/",
+ "https://quackwatch.org/",
+ "https://www.skepdic.com/",
+ "http://scibabe.com/",
+ "http://pandasthumb.org/",
+ "https://skepticalscience.com/",
+ "https://www.cdc.gov/",
+ "https://apnews.com/"]
+
+with gr.Blocks() as blocks:
+    gr.Markdown("# Pseudometer")
+    gr.Markdown("Prototype machine learning pseudoscience detector for websites!")
+    text = gr.Textbox(label="Input URL (http format):")
+    label = gr.outputs.Label()
+    btn = gr.Button("Analyze!")
+    with gr.Accordion("Pseudoscience Primary Training Sources"):
+        gr.Markdown(', '.join(pseudo_sources))
+    with gr.Accordion("Science Primary Training Sources"):
+        gr.Markdown(', '.join(science_sources))
+    example = gr.Examples(examples=examples, inputs=text)
+
+    btn.click(fn=predict, inputs=text, outputs=[label, gr.Plot(label="Wordcloud")])
+    
+blocks.launch()

packages.txt

@@ -0,0 +1 @@
+libenchant-dev

requirements.txt

@@ -5,4 +5,5 @@ beautifulsoup4
 pandas 
 matplotlib 
 pyenchant
-gradio
+gradio
+wordcloud