Spaces:
No application file
No application file
Update training.py
Browse files- training.py +8 -1
training.py
CHANGED
@@ -21,8 +21,15 @@ data_collator = DataCollatorWithPadding(tokenizer=tokenizer, return_tensors="tf"
|
|
21 |
|
22 |
tf_train_dataset = tokenized_dataset["train"].to_tf_dataset(
|
23 |
columns=["attention_mask", "input_ids", "token_type_ids"],
|
24 |
-
label_cols=["
|
25 |
shuffle=True,
|
26 |
collate_fn=data_collator,
|
27 |
batch_size=8,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
28 |
)
|
|
|
21 |
|
22 |
tf_train_dataset = tokenized_dataset["train"].to_tf_dataset(
|
23 |
columns=["attention_mask", "input_ids", "token_type_ids"],
|
24 |
+
label_cols=["answer"],
|
25 |
shuffle=True,
|
26 |
collate_fn=data_collator,
|
27 |
batch_size=8,
|
28 |
+
)
|
29 |
+
tf_validation_dataset = tokenized_datasets["validation"].to_tf_dataset(
|
30 |
+
columns=["attention_mask", "input_ids", "token_type_ids"],
|
31 |
+
label_cols=["answer"],
|
32 |
+
shuffle=False,
|
33 |
+
collate_fn=data_collator,
|
34 |
+
batch_size=8,
|
35 |
)
|