Update app.py

app.py

@@ -26,116 +26,116 @@ import en_core_web_sm
 nlp = en_core_web_sm.load()
 
 
-# def recall_m(y_true, y_pred):
-#   true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
-#   possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
-#   recall = true_positives / (possible_positives + K.epsilon())
-#   return recall
+def recall_m(y_true, y_pred):
+  true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
+  possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
+  recall = true_positives / (possible_positives + K.epsilon())
+  return recall
 
-# def precision_m(y_true, y_pred):
-#   true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
-#   predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
-#   precision = true_positives / (predicted_positives + K.epsilon())
-#   return precision
+def precision_m(y_true, y_pred):
+  true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
+  predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
+  precision = true_positives / (predicted_positives + K.epsilon())
+  return precision
 
-# def f1_m(y_true, y_pred):
-#   precision = precision_m(y_true, y_pred)
-#   recall = recall_m(y_true, y_pred)
-#   return 2*((precision*recall)/(precision+recall+K.epsilon()))
+def f1_m(y_true, y_pred):
+  precision = precision_m(y_true, y_pred)
+  recall = recall_m(y_true, y_pred)
+  return 2*((precision*recall)/(precision+recall+K.epsilon()))
 
 
-# #initialise callback class
-# class callback(CallbackAny2Vec):
-#   """
-#   Print the loss value after each epoch
-#   """
-#   def __init__(self):
-#     self.epoch = 0
-#     #gensim loss is cumulative, so we record previous values to print
-#     self.loss_previous_step = 0 
+#initialise callback class
+class callback(CallbackAny2Vec):
+  """
+  Print the loss value after each epoch
+  """
+  def __init__(self):
+    self.epoch = 0
+    #gensim loss is cumulative, so we record previous values to print
+    self.loss_previous_step = 0 
 
-#   def on_epoch_end(self, model):
-#     loss = model.get_latest_training_loss()
-#     if self.epoch % 100 == 0:
-#       print('Loss after epoch {}: {}'.format(self.epoch, loss-self.loss_previous_step))
+  def on_epoch_end(self, model):
+    loss = model.get_latest_training_loss()
+    if self.epoch % 100 == 0:
+      print('Loss after epoch {}: {}'.format(self.epoch, loss-self.loss_previous_step))
 
-#     self.epoch+= 1
-#     self.loss_previous_step = loss
+    self.epoch+= 1
+    self.loss_previous_step = loss
 
 
 
 
 
-# def spacy_lemmatize_text(text):
-#   text = nlp(text)
-#   text = ' '.join([word.lemma_ if word.lemma_ != '-PRON-' else word.text for word in text])
-#   return text
+def spacy_lemmatize_text(text):
+  text = nlp(text)
+  text = ' '.join([word.lemma_ if word.lemma_ != '-PRON-' else word.text for word in text])
+  return text
 
-# def remove_accented_chars(text):
-#   text = unicodedata.normalize('NFC', text).encode('ascii', 'ignore').decode('utf-8', 'ignore')
-#   return text
+def remove_accented_chars(text):
+  text = unicodedata.normalize('NFC', text).encode('ascii', 'ignore').decode('utf-8', 'ignore')
+  return text
  
-# def remove_special_characters(text, remove_digits=False):
-#   pattern = r'[^a-zA-Z0-9\s]' if not remove_digits else r'[^a-zA-Z\s]'
-#   text = re.sub(pattern, '', text)
-#   return text
+def remove_special_characters(text, remove_digits=False):
+  pattern = r'[^a-zA-Z0-9\s]' if not remove_digits else r'[^a-zA-Z\s]'
+  text = re.sub(pattern, '', text)
+  return text
   
-# def remove_stopwords(text, is_lower_case=False, stopwords=None):
-#   if not stopwords:
-#       stopwords = nltk.corpus.stopwords.words('english')
-#   tokens = nltk.word_tokenize(text)
-#   tokens = [token.strip() for token in tokens]
+def remove_stopwords(text, is_lower_case=False, stopwords=None):
+  if not stopwords:
+      stopwords = nltk.corpus.stopwords.words('english')
+  tokens = nltk.word_tokenize(text)
+  tokens = [token.strip() for token in tokens]
   
-#   if is_lower_case:
-#       filtered_tokens = [token for token in tokens if token not in stopwords]
-#   else:
-#       filtered_tokens = [token for token in tokens if token.lower() not in stopwords]
+  if is_lower_case:
+      filtered_tokens = [token for token in tokens if token not in stopwords]
+  else:
+      filtered_tokens = [token for token in tokens if token.lower() not in stopwords]
   
-#   filtered_text = ' '.join(filtered_tokens)    
-#   return filtered_text
+  filtered_text = ' '.join(filtered_tokens)    
+  return filtered_text
 
 
-# def pre_process():
-#   opo_texto_sem_caracteres_especiais = (remove_accented_chars(sentence))
-#   sentenceExpanded = contractions.fix(opo_texto_sem_caracteres_especiais)
-#   sentenceWithoutPunctuation = remove_special_characters(sentenceExpanded , remove_digits=True)
-#   sentenceLowered = sentenceWithoutPunctuation.lower()
-#   sentenceLemmatized = spacy_lemmatize_text(sentenceLowered)
-#   sentenceLemStopped = remove_stopwords(sentenceLemmatized, is_lower_case=False)
+def pre_process():
+  opo_texto_sem_caracteres_especiais = (remove_accented_chars(sentence))
+  sentenceExpanded = contractions.fix(opo_texto_sem_caracteres_especiais)
+  sentenceWithoutPunctuation = remove_special_characters(sentenceExpanded , remove_digits=True)
+  sentenceLowered = sentenceWithoutPunctuation.lower()
+  sentenceLemmatized = spacy_lemmatize_text(sentenceLowered)
+  sentenceLemStopped = remove_stopwords(sentenceLemmatized, is_lower_case=False)
 
-#   return nltk.word_tokenize(sentenceLemStopped)
+  return nltk.word_tokenize(sentenceLemStopped)
 
-# def classify(new_column = True):
-#   sentenceWords = json.loads(sentence.replace("'",'"'))
+def classify(new_column = True):
+  sentenceWords = json.loads(sentence.replace("'",'"'))
   
-#   aux_vector = []
-#   for word in sentenceWords:
-#     aux_vector.append(reloaded_w2v_model.wv[word])
-#   w2vWords = []
-#   w2vWords.append(aux_vector)
-#   MCTIinput_vector = pad_sequences(w2vWords, maxlen=2726, padding='pre')
+  aux_vector = []
+  for word in sentenceWords:
+    aux_vector.append(reloaded_w2v_model.wv[word])
+  w2vWords = []
+  w2vWords.append(aux_vector)
+  MCTIinput_vector = pad_sequences(w2vWords, maxlen=2726, padding='pre')
   
-#   value = reconstructed_model_CNN.predict(MCTIinput_vector)[0]
+  value = reconstructed_model_CNN.predict(MCTIinput_vector)[0]
   
-#   if value >= 0.5:
-#     return Image.open(r"elegivel.png")
-#   else:
-#     return Image.open(r"inelegivel.png")
+  # if value >= 0.5:
+  #   return Image.open(r"elegivel.png")
+  # else:
+  #   return Image.open(r"inelegivel.png")
 
-#   dataMCTI['opo_pre_tkn'] = sentencesMCTIList_xp8
-#   dataMCTI['opo_pre'] = sentencesMCTIList_xp8_sentences
+  dataMCTI['opo_pre_tkn'] = sentencesMCTIList_xp8
+  dataMCTI['opo_pre'] = sentencesMCTIList_xp8_sentences
 
-# def gen_output(data):
-#   data.to_excel("output.xlsx", index=False)
-#   return "output.xlsx"
+def gen_output(data):
+  data.to_excel("output.xlsx", index=False)
+  return "output.xlsx"
     
 
-# reloaded_w2v_model = Word2Vec.load('word2vec_xp8.model')
+reloaded_w2v_model = Word2Vec.load('word2vec_xp8.model')
 
-# reconstructed_model_CNN = keras.models.load_model("best weights CNN.h5", 
-#                                                    custom_objects={'f1_m':f1_m, 
-#                                                                    "precision_m":precision_m, 
-#                                                                    "recall_m":recall_m})
+reconstructed_model_CNN = keras.models.load_model("best weights CNN.h5", 
+                                                   custom_objects={'f1_m':f1_m, 
+                                                                   "precision_m":precision_m, 
+                                                                   "recall_m":recall_m})
 
 def app(operacao, resultado, dados):
   data = pd.read_excel(dados)