Update tokenizer

src/models/bert.py

@@ -8,7 +8,7 @@ from torch import nn
 from transformers import RobertaTokenizer, RobertaModel, AdamW, RobertaConfig
 from sklearn.model_selection import train_test_split
 
-from transformers import BertTokenizer, BertForSequenceClassification, get_linear_schedule_with_warmup, AutoModel
+from transformers import BertTokenizer, BertForSequenceClassification, get_linear_schedule_with_warmup, AutoModel, AutoTokenizer
 from torch.utils.data import TensorDataset, random_split, DataLoader, RandomSampler, SequentialSampler
 
 
@@ -20,7 +20,8 @@ class BERTClassifier():
         
         self.model_name = model_name
         
-        self.tokenizer = BertTokenizer.from_pretrained(tokenizer_name, do_lower_case=True)
+        self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
+        # self.tokenizer = BertTokenizer.from_pretrained(tokenizer_name, do_lower_case=True)
         
         if model_name.startswith('jeevavijay10'):
             # self.model = torch.load(model_name)