optimize nltk

.ipynb_checkpoints/app-checkpoint.py

@@ -42,8 +42,11 @@ def prepare_model():
 
 @st.experimental_singleton
 def prepare_lists():
-    nltk.download('omw-1.4')
-    nltk.download('wordnet')
+    try:
+        wordnet.synsets("bias")
+    except:
+        nltk.download('omw-1.4')
+        nltk.download('wordnet')
     countries = pd.read_csv("Assets/Countries/combined-countries.csv")
     professions = pd.read_csv("Assets/Professions/soc-professions-2018.csv")
     word_lists = [list(countries.Words.apply(lambda x: x.lower())),list(professions.Words)]

NLselector.py

@@ -181,9 +181,9 @@ def abs_dif(df,seed):
     
     text2 = Nearest Prediction
     text3 = Farthest Prediction'''
-    #seed = process_text(seed)
-    target = df[df['Words'] == seed].pred.iloc[0]
-    sub_df = df[df['Words'] != seed].reset_index()
+    seed = process_text(seed)
+    target = df[df['Words'].str.lower() == seed].pred.iloc[0]
+    sub_df = df[df['Words'].str.lower() != seed].reset_index()
     nearest_prediction = sub_df.pred[(sub_df.pred-target).abs().argsort()[:1]]
     farthest_prediction = sub_df.pred[(sub_df.pred-target).abs().argsort()[-1:]]
     text2 = sub_df.text.iloc[nearest_prediction.index[0]]
@@ -207,15 +207,15 @@ def sampled_alts(df, seed, fixed=False):
 def gen_cf_country(df,_document,selection):
     df['text'] = df.Words.apply(lambda x: re.sub(r'\b'+selection+r'\b',x,_document.text))
     df['pred'] = df.text.apply(eval_pred)
-    df['seed'] = df.Words.apply(lambda x: 'seed' if x == selection else 'alternative')
+    df['seed'] = df.Words.apply(lambda x: 'seed' if x.lower() == selection.lower() else 'alternative')
     df['similarity'] = df.Words.apply(lambda x: nlp(selection).similarity(nlp(x)))
     return df
 
 def gen_cf_profession(df,_document,selection):
-    category = df.loc[df['Words'] == selection, 'Major'].iloc[0]
+    category = df.loc[df['Words'] == selection.lower(), 'Major'].iloc[0]
     df = df[df.Major == category]
     df['text'] = df.Words.apply(lambda x: re.sub(r'\b'+selection+r'\b',x,_document.text))
     df['pred'] = df.text.apply(eval_pred)
-    df['seed'] = df.Words.apply(lambda x: 'seed' if x == selection else 'alternative')
+    df['seed'] = df.Words.apply(lambda x: 'seed' if x == selection.lower() else 'alternative')
     df['similarity'] = df.Words.apply(lambda x: nlp(selection).similarity(nlp(x)))
     return df

app.py

@@ -42,8 +42,11 @@ def prepare_model():
 
 @st.experimental_singleton
 def prepare_lists():
-    nltk.download('omw-1.4')
-    nltk.download('wordnet')
+    try:
+        wordnet.synsets("bias")
+    except:
+        nltk.download('omw-1.4')
+        nltk.download('wordnet')
     countries = pd.read_csv("Assets/Countries/combined-countries.csv")
     professions = pd.read_csv("Assets/Professions/soc-professions-2018.csv")
     word_lists = [list(countries.Words.apply(lambda x: x.lower())),list(professions.Words)]