AnkitKr commited on
Commit
053c4f0
1 Parent(s): 83f7278

Update training.py

Browse files
Files changed (1) hide show
  1. training.py +8 -1
training.py CHANGED
@@ -21,8 +21,15 @@ data_collator = DataCollatorWithPadding(tokenizer=tokenizer, return_tensors="tf"
21
 
22
  tf_train_dataset = tokenized_dataset["train"].to_tf_dataset(
23
  columns=["attention_mask", "input_ids", "token_type_ids"],
24
- label_cols=["labels"],
25
  shuffle=True,
26
  collate_fn=data_collator,
27
  batch_size=8,
 
 
 
 
 
 
 
28
  )
 
21
 
22
  tf_train_dataset = tokenized_dataset["train"].to_tf_dataset(
23
  columns=["attention_mask", "input_ids", "token_type_ids"],
24
+ label_cols=["answer"],
25
  shuffle=True,
26
  collate_fn=data_collator,
27
  batch_size=8,
28
+ )
29
+ tf_validation_dataset = tokenized_datasets["validation"].to_tf_dataset(
30
+ columns=["attention_mask", "input_ids", "token_type_ids"],
31
+ label_cols=["answer"],
32
+ shuffle=False,
33
+ collate_fn=data_collator,
34
+ batch_size=8,
35
  )