add max_length

run_mlm_flax_stream.py

@@ -308,7 +308,7 @@ def advance_iter_and_group_samples(train_iterator, num_samples, max_seq_length):
     while i < num_total_tokens:
         tokenized_samples = next(train_iterator)
         i += len(tokenized_samples["input_ids"])
-
+        print(tokenized_samples)
         # concatenate tokenized samples to list
         samples = {k: samples[k] + tokenized_samples[k] for k in tokenized_samples.keys()}
 
@@ -505,6 +505,7 @@ if __name__ == "__main__":
         return tokenizer(
             examples[data_args.text_column_name],
             max_length=512,
+            truncation=True,
             return_special_tokens_mask=True
         )