add max_length
Browse files- run_mlm_flax_stream.py +1 -0
run_mlm_flax_stream.py
CHANGED
|
@@ -504,6 +504,7 @@ if __name__ == "__main__":
|
|
| 504 |
def tokenize_function(examples):
|
| 505 |
return tokenizer(
|
| 506 |
examples[data_args.text_column_name],
|
|
|
|
| 507 |
return_special_tokens_mask=True
|
| 508 |
)
|
| 509 |
|
|
|
|
| 504 |
def tokenize_function(examples):
|
| 505 |
return tokenizer(
|
| 506 |
examples[data_args.text_column_name],
|
| 507 |
+
max_length=512,
|
| 508 |
return_special_tokens_mask=True
|
| 509 |
)
|
| 510 |
|