deve

app.py

@@ -190,7 +190,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                 dataset['train'] = second_third
                 del dataset['test']
                 tokenized_sh_fq_dataset = dataset.map(tokenize_function, batched=True,)
-                dataset['validation'] = tokenize_function(dataset['validation'])
+                dataset['validation'] = dataset['validation'].map(tokenize_function, batched=True)
                 saved_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset') 
                 dataset['train'] = concatenate_datasets([saved_dataset, tokenized_sh_fq_dataset['train']])
                 dataset['train'].save_to_disk(f'/data/{hub_id.strip()}_train_dataset2')