add requirements
Browse files- run_mlm_flax_stream.py +0 -1
run_mlm_flax_stream.py
CHANGED
@@ -487,7 +487,6 @@ if __name__ == "__main__":
|
|
487 |
def tokenize_function(examples):
|
488 |
return tokenizer(
|
489 |
examples[data_args.text_column_name],
|
490 |
-
max_length=512,
|
491 |
return_special_tokens_mask=True
|
492 |
)
|
493 |
|
|
|
487 |
def tokenize_function(examples):
|
488 |
return tokenizer(
|
489 |
examples[data_args.text_column_name],
|
|
|
490 |
return_special_tokens_mask=True
|
491 |
)
|
492 |
|