dopset and TaskOne

dopset.py

@@ -0,0 +1,80 @@
+import string
+import torch
+import numpy as np
+from nltk.corpus import stopwords
+stop_words = set(stopwords.words('english'))
+import torch.nn as nn
+import pickle
+from transformers import BertTokenizer, BertModel, DistilBertTokenizer, DistilBertModel
+from sklearn.linear_model import LogisticRegression
+
+
+
+EMBEDDING_DIM = 64
+VOCAB_SIZE = 203310
+embedding_matrix = np.zeros((VOCAB_SIZE, EMBEDDING_DIM))
+embedding_layer = torch.nn.Embedding.from_pretrained(torch.FloatTensor(embedding_matrix))
+
+
+
+
+with open('lstm/vocab_to_int.txt', 'rb') as f:
+    vocab_to_int = pickle.load(f)
+
+class LSTMClassifier(nn.Module):
+    def __init__(self, embedding_dim: int, hidden_size: int = 32) -> None:
+        super().__init__()
+
+        self.embedding_dim = embedding_dim # создаем эмбединг сайз
+        self.hidden_size = hidden_size # создаем хидден сайз
+        self.embedding = embedding_layer # создаем слои модели
+
+        self.lstm = nn.LSTM(
+            input_size=self.embedding_dim,
+            hidden_size=self.hidden_size,
+            batch_first=True
+        )
+        
+        self.clf = nn.Linear(self.hidden_size, 1)
+    
+    def forward(self, x):
+        embedding = self.embedding(x)
+        _, (h_n, _) = self.lstm(embedding)
+        out = self.clf(h_n.squeeze())
+        return out  
+
+
+def data_preprocessing(text: str) -> str:
+    text = text.lower()
+    text = ''.join([c for c in text if c not in string.punctuation])# Remove punctuation
+    text = [word for word in text.split() if word not in stop_words] 
+    text = ' '.join(text)
+    return text
+
+
+
+def padding(review_int: list, seq_len: int) -> np.array:
+    features = np.zeros((len(review_int), seq_len), dtype = int)
+    for i, review in enumerate(review_int):
+        if len(review) <= seq_len:
+            zeros = list(np.zeros(seq_len - len(review)))
+            new = zeros + review
+        else:
+            new = review[: seq_len]
+        features[i, :] = np.array(new)
+            
+    return features
+
+def preprocess_single_string(input_string: str, seq_len: int, vocab_to_int: dict = vocab_to_int) -> list:
+    preprocessed_string = data_preprocessing(input_string)
+    result_list = []
+    for word in preprocessed_string.split():
+        try: 
+            result_list.append(vocab_to_int[word])
+        except KeyError as e:
+            print(f'{e}: not in dictionary!')
+    result_padded = padding([result_list], seq_len)[0]
+
+    return torch.tensor(result_padded)
+
+

pages/TaskOne.py

@@ -0,0 +1,97 @@
+import streamlit as st
+import torch
+import torch.nn as nn
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.linear_model import LogisticRegression
+import pickle
+import time
+from dopset import (LSTMClassifier, preprocess_single_string)
+import numpy as np
+import pandas as pd
+from transformers import DistilBertModel, DistilBertTokenizer
+
+loaded = pickle.load(open('BertWeight/log.pkl', "rb"))
+model_BERT = DistilBertModel.from_pretrained("BertWeight/pt_save_pretrained")
+tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+
+st.title('Отзывы')
+
+@st.cache_resource
+
+def load_model_and_vectorizer():
+    
+    loaded_vectorizer = pickle.load(open('tfidf/tfidf_vectorizer.pkl', 'rb'))
+    loaded_model = pickle.load(open('tfidf/logistic_regression_model.pkl', 'rb'))
+    return loaded_vectorizer, loaded_model
+
+def logreg(text):
+    weight_vect, weight_model = load_model_and_vectorizer()
+
+    start_time = time.time()
+    input_ids = weight_vect.transform([text])
+    output = weight_model.predict(input_ids)
+    end_time = time.time()
+
+    if output == 1:
+         return "положительный", round(end_time - start_time, 5)
+    else:
+        return "Негативный", round(end_time - start_time, 5)
+    
+def lstm_(text):
+    EMBEDDING_DIM = 64
+    HIDDEN_DIM = 16
+    DEVICE = 'cpu'
+
+    model = LSTMClassifier(embedding_dim=EMBEDDING_DIM, hidden_size=HIDDEN_DIM).to(DEVICE)
+    model.load_state_dict(torch.load('lstm/lstm_weights.pt', map_location=DEVICE))
+    start_time = time.time()
+    pred = model(preprocess_single_string(text, seq_len=128).unsqueeze(0).to(DEVICE)).sigmoid().round().item()
+    end_time = time.time()
+    if pred == 1:
+         return "положительный", round(end_time - start_time, 5)
+    else:
+        return "Негативный", round(end_time - start_time, 5)
+    
+def bert_(text, model, loaded_model):
+    start_time = time.time()
+    tokenized_text = tokenizer.encode(text, add_special_tokens=True, truncation=True, max_length=64)
+    input_ids = torch.tensor(tokenized_text).unsqueeze(0)
+    with torch.no_grad():
+        outputs = model(input_ids)
+        vectors = outputs[0][:,0,:].detach().cpu().numpy()
+    
+    prediction = loaded_model.predict(vectors).item()
+    end_time = time.time()
+    if prediction == 1:
+        return "положительный", round(end_time - start_time, 5)
+    else:
+        return "Негативный", round(end_time - start_time, 5)
+
+
+table_f1 = {'Model': ['Tf-IDF + logreg', 'LSTM', 'Bert'],
+            'F1-score':['0.91', '0.94', '0.74']}
+
+df = pd.DataFrame(table_f1)
+
+text = st.text_input("Напишите отзыв")
+
+if text:
+
+    rate, time_ = logreg(text)
+    st.markdown('### Tf-IDF + logreg')
+    st.write('Отзыв:', rate)
+    st.write('Время:', time_)
+
+    rate_lstm, time_lstm = lstm_(text)
+    st.markdown('### LSTM')
+    st.write('Отзыв:', rate_lstm)
+    st.write('Время:', time_lstm)
+
+    rate_bert, time_bert = bert_(text, model_BERT, loaded)
+    st.markdown('### BERT')
+    st.write('Отзыв:', rate_bert)
+    st.write('Время:', time_bert)
+    st.table(df)
+
+    
+