add BERT model

app.py

@@ -11,9 +11,10 @@ import pandas as pd
 import plotly.express as px
 import keras
 
-
 from underthesea import word_tokenize
 
+from phoBERT import BERT_predict
+
 #Load tokenizer
 fp = Path(__file__).with_name('tokenizer.pkl')
 with open(fp,mode="rb") as f:
@@ -81,26 +82,16 @@ def judge(x):
 
   lstm_pred = LSTM_predict(x)
   gru_pred = GRU_predict(x)
-
+  bert_pred = BERT_predict(x)
   #print(result)
   
   return_result = 'Result'
   result_lstm = np.round(lstm_pred, 2)
   result_gru = np.round(gru_pred, 2)
+  result_bert = np.round(bert_pred, 2)
   for i in range(6):
-    result.append((result_lstm[i]+result_gru[i])/2)
-  
+    result.append((result_lstm[i]+result_gru[i]+result_bert[i])/3)
   
-
-  #print(final_result)
-  return_result += '\nMô hình LSTM\n'
-  return_result += f"{result_lstm}\n"
- 
-
-  return_result += '\nMô hình GRU\n'
-  return_result += f"{result_gru}\n"
-
- 
   return (result)
 
 

phoBERT.py

@@ -0,0 +1,79 @@
+import torch
+from transformers import AutoModel, AutoTokenizer
+from underthesea import word_tokenize
+
+phobert = AutoModel.from_pretrained("vinai/phobert-base")
+tokenizer = AutoTokenizer.from_pretrained("vinai/phobert-base")
+
+class PhoBertModel(torch.nn.Module):
+  def __init__(self):
+    super(PhoBertModel, self).__init__()
+    self.bert = phobert
+    self.pre_classifier = torch.nn.Linear(self.bert.config.hidden_size, self.bert.config.hidden_size)
+    self.dropout = torch.nn.Dropout(0.1)
+    self.classifier = torch.nn.Linear(self.bert.config.hidden_size, 6)
+
+  def forward(self, input_ids, attention_mask, token_type_ids):
+    hidden_state, output_1 = self.bert(
+        input_ids = input_ids,
+        attention_mask=attention_mask,
+        return_dict = False
+    )
+    pooler = self.pre_classifier(output_1)
+    activation_1 = torch.nn.Tanh()(pooler)
+    
+    drop = self.dropout(activation_1)
+    
+    output_2 = self.classifier(drop)
+#     activation_2 = torch.nn.Tanh()(output_2)
+    
+    output = torch.nn.Sigmoid()(output_2)
+    return output
+  
+def getModel():
+    model = torch.load('phoBertModel.pth', map_location=torch.device('cpu'))
+    model.eval()
+    return model
+
+model = getModel()
+
+def tokenize(data):
+  
+  max_length = 200
+
+  for line in data:
+        
+    token = tokenizer.encode_plus(
+        line,
+        max_length=200,
+        add_special_tokens=False,
+        pad_to_max_length=True
+        )
+    
+    ids = torch.tensor([token['input_ids']])
+    mask = torch.tensor([token['attention_mask']])
+    token_type_ids = torch.tensor([token['token_type_ids']])
+    
+    
+    output = {
+        'ids': ids,
+        'mask': mask,
+        'token_type_ids': token_type_ids,
+    }
+    #outputs.append(output)
+
+  return output
+
+def BERT_predict(text):
+    text = word_tokenize(text)
+    text = [text]
+    token = tokenize(text)
+    
+    ids = token['ids']
+    mask = token['mask']
+    token_type_ids = token['token_type_ids']
+    
+    result = model(ids, mask, token_type_ids)
+    print(result)
+    return result.tolist()[0]
+

phoBertModel.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5fca9d837d05b1e8330798e32a59b5200bf677d5cf2f178727dcd131c86230b
+size 542499629

requirements.txt

@@ -7,4 +7,6 @@ pathlib
 plotly
 pandas
 keras==2.15.0
-underthesea
+underthesea
+torch
+transformers