Ruslan-DS commited on
Commit
4718e6a
1 Parent(s): 8ff1293

Update models/preprocess_stage/bert_model.py

Browse files
models/preprocess_stage/bert_model.py CHANGED
@@ -2,14 +2,14 @@ import numpy as np
2
  import torch
3
  from transformers import BertModel, BertTokenizer
4
 
5
- weights = 'DeepPavlov/rubert-base-cased'
6
- tokenizer_bert = BertTokenizer.from_pretrained(weights)
7
- bert_for_logreg = BertModel.from_pretrained(weights)
8
 
9
 
10
  def preprocess_bert(text, MAX_LEN):
11
 
12
- tokenized_text = tokenizer_bert.encode(
13
  text=text,
14
  add_special_tokens=True,
15
  truncation=True,
 
2
  import torch
3
  from transformers import BertModel, BertTokenizer
4
 
5
+ CHECKPOINT = 'DeepPavlov/rubert-base-cased'
6
+ tokenizer = BertTokenizer.from_pretrained(CHECKPOINT)
7
+ model = BertModel.from_pretrained(CHECKPOINT)
8
 
9
 
10
  def preprocess_bert(text, MAX_LEN):
11
 
12
+ tokenized_text = tokenizer.encode(
13
  text=text,
14
  add_special_tokens=True,
15
  truncation=True,