Update app.py

app.py

@@ -1,15 +1,163 @@
+import os
+import re
+import contractions
+import unicodedata
+import spacy
+import keras
+import requests
+import shutil
+import json
 import gradio as gr
 import pandas as pd
+import numpy as np
+from PIL import Image
+from keras import backend as K
+from keras.utils.data_utils import pad_sequences
+from gensim.models import Word2Vec
+from gensim.models.callbacks import CallbackAny2Vec
+
+import nltk
+nltk.download('punkt')
+nltk.download('stopwords')
+
+os.system('python -m spacy download en_core_web_sm')]
+
+import en_core_web_sm
+nlp = en_core_web_sm.load()
+
+
+def recall_m(y_true, y_pred):
+  true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
+  possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
+  recall = true_positives / (possible_positives + K.epsilon())
+  return recall
+
+def precision_m(y_true, y_pred):
+  true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
+  predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
+  precision = true_positives / (predicted_positives + K.epsilon())
+  return precision
+
+def f1_m(y_true, y_pred):
+  precision = precision_m(y_true, y_pred)
+  recall = recall_m(y_true, y_pred)
+  return 2*((precision*recall)/(precision+recall+K.epsilon()))
+
+
+#initialise callback class
+class callback(CallbackAny2Vec):
+  """
+  Print the loss value after each epoch
+  """
+  def __init__(self):
+    self.epoch = 0
+    #gensim loss is cumulative, so we record previous values to print
+    self.loss_previous_step = 0 
+
+  def on_epoch_end(self, model):
+    loss = model.get_latest_training_loss()
+    if self.epoch % 100 == 0:
+      print('Loss after epoch {}: {}'.format(self.epoch, loss-self.loss_previous_step))
+
+    self.epoch+= 1
+    self.loss_previous_step = loss
+
+
+
+
+
+def spacy_lemmatize_text(text):
+  text = nlp(text)
+  text = ' '.join([word.lemma_ if word.lemma_ != '-PRON-' else word.text for word in text])
+  return text
+
+def remove_accented_chars(text):
+  text = unicodedata.normalize('NFC', text).encode('ascii', 'ignore').decode('utf-8', 'ignore')
+  return text
+ 
+def remove_special_characters(text, remove_digits=False):
+  pattern = r'[^a-zA-Z0-9\s]' if not remove_digits else r'[^a-zA-Z\s]'
+  text = re.sub(pattern, '', text)
+  return text
+  
+def remove_stopwords(text, is_lower_case=False, stopwords=None):
+  if not stopwords:
+      stopwords = nltk.corpus.stopwords.words('english')
+  tokens = nltk.word_tokenize(text)
+  tokens = [token.strip() for token in tokens]
+  
+  if is_lower_case:
+      filtered_tokens = [token for token in tokens if token not in stopwords]
+  else:
+      filtered_tokens = [token for token in tokens if token.lower() not in stopwords]
+  
+  filtered_text = ' '.join(filtered_tokens)    
+  return filtered_text
+
+
+def pre_process():
+  opo_texto_sem_caracteres_especiais = (remove_accented_chars(sentence))
+  sentenceExpanded = contractions.fix(opo_texto_sem_caracteres_especiais)
+  sentenceWithoutPunctuation = remove_special_characters(sentenceExpanded , remove_digits=True)
+  sentenceLowered = sentenceWithoutPunctuation.lower()
+  sentenceLemmatized = spacy_lemmatize_text(sentenceLowered)
+  sentenceLemStopped = remove_stopwords(sentenceLemmatized, is_lower_case=False)
+
+  return nltk.word_tokenize(sentenceLemStopped)
+
+def classify(new_column = True):
+  sentenceWords = json.loads(sentence.replace("'",'"'))
+  
+  aux_vector = []
+  for word in sentenceWords:
+    aux_vector.append(reloaded_w2v_model.wv[word])
+  w2vWords = []
+  w2vWords.append(aux_vector)
+  MCTIinput_vector = pad_sequences(w2vWords, maxlen=2726, padding='pre')
+  
+  value = reconstructed_model_CNN.predict(MCTIinput_vector)[0]
+  
+  if value >= 0.5:
+    return Image.open(r"elegivel.png")
+  else:
+    return Image.open(r"inelegivel.png")
+
+def gen_output(data):
+  return "output.xlsx"
+    
+
+reloaded_w2v_model = Word2Vec.load('word2vec_xp8.model')
+
+reconstructed_model_CNN = keras.models.load_model("best weights CNN.h5", 
+                                                   custom_objects={'f1_m':f1_m, 
+                                                                   "precision_m":precision_m, 
+                                                                   "recall_m":recall_m})
 
 def app(operacao, resultado, dados):
+
   boxes = {'Color': ['Green','Green','Green','Blue','Blue','Red','Red','Red'],
            'Shape': ['Rectangle','Rectangle','Square','Rectangle','Square','Square','Square','Rectangle'],
            'Price': [10,15,5,5,10,15,15,5]
           }
   df = pd.DataFrame(boxes, columns= ['Color','Shape','Price'])
   df.to_excel("output.xlsx")
-  
-  return "output.xlsx"
+
+  if operacao === "Pré-processamento + Classificação" :
+    pre_process()
+    classify(resultado == "Nova Coluna")
+    output = gen_output()
+    
+    return output
+  elif operacao === "Apenas Pré-processamento" :
+    pre_process()
+    output = gen_output()
+
+    return output
+  elif operacao === "Apenas Classificação" :
+    classify(resultado == "Nova Coluna")
+    output = gen_output()
+
+    return output
 
 iface = gr.Interface(
     fn=app,