Added truncation for long sequences

BertForPrefixMarking.py

@@ -159,7 +159,7 @@ class BertForPrefixMarking(BertPreTrainedModel):
 
 
 def encode_sentences_for_bert_for_prefix_marking(tokenizer: BertTokenizerFast, sentences: List[str], padding='longest'):
-    inputs = tokenizer(sentences, padding=padding, return_tensors='pt')
+    inputs = tokenizer(sentences, padding=padding, truncation=True, return_tensors='pt')
 
     # create our prefix_id_options array which will be like the input ids shape but with an addtional
     # dimension containing for each prefix whether it can be for that word