Spaces:
Sleeping
Sleeping
Update dopset.py
Browse files
dopset.py
CHANGED
@@ -1,13 +1,13 @@
|
|
1 |
import string
|
2 |
import torch
|
3 |
import numpy as np
|
4 |
-
from nltk.corpus import stopwords
|
5 |
-
stop_words = set(stopwords.words('english'))
|
6 |
import torch.nn as nn
|
7 |
import pickle
|
8 |
from transformers import BertTokenizer, BertModel, DistilBertTokenizer, DistilBertModel
|
9 |
from sklearn.linear_model import LogisticRegression
|
10 |
-
import nltk
|
11 |
|
12 |
|
13 |
|
@@ -48,7 +48,7 @@ class LSTMClassifier(nn.Module):
|
|
48 |
def data_preprocessing(text: str) -> str:
|
49 |
text = text.lower()
|
50 |
text = ''.join([c for c in text if c not in string.punctuation])# Remove punctuation
|
51 |
-
text = [word for word in text.split() if word not in stop_words]
|
52 |
text = ' '.join(text)
|
53 |
return text
|
54 |
|
|
|
1 |
import string
|
2 |
import torch
|
3 |
import numpy as np
|
4 |
+
# from nltk.corpus import stopwords
|
5 |
+
# stop_words = set(stopwords.words('english'))
|
6 |
import torch.nn as nn
|
7 |
import pickle
|
8 |
from transformers import BertTokenizer, BertModel, DistilBertTokenizer, DistilBertModel
|
9 |
from sklearn.linear_model import LogisticRegression
|
10 |
+
# import nltk
|
11 |
|
12 |
|
13 |
|
|
|
48 |
def data_preprocessing(text: str) -> str:
|
49 |
text = text.lower()
|
50 |
text = ''.join([c for c in text if c not in string.punctuation])# Remove punctuation
|
51 |
+
# text = [word for word in text.split() if word not in stop_words]
|
52 |
text = ' '.join(text)
|
53 |
return text
|
54 |
|