add maxlength
Browse files- classification.py +1 -2
classification.py
CHANGED
|
@@ -12,8 +12,7 @@ def classify_by_topic(articles, central_topics):
|
|
| 12 |
def compute_similarity(articles, central_topics):
|
| 13 |
|
| 14 |
model = AutoModel.from_pretrained("distilbert-base-multilingual-cased")
|
| 15 |
-
tokenizer = AutoTokenizer.from_pretrained(
|
| 16 |
-
"distilbert-base-multilingual-cased")
|
| 17 |
|
| 18 |
def sentence_to_vector(sentence, context):
|
| 19 |
|
|
|
|
| 12 |
def compute_similarity(articles, central_topics):
|
| 13 |
|
| 14 |
model = AutoModel.from_pretrained("distilbert-base-multilingual-cased")
|
| 15 |
+
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-multilingual-cased",max_length=512)
|
|
|
|
| 16 |
|
| 17 |
def sentence_to_vector(sentence, context):
|
| 18 |
|