Kevin Fink
commited on
Commit
·
e89a055
1
Parent(s):
f1ce0f3
deve
Browse files
app.py
CHANGED
@@ -207,7 +207,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
|
|
207 |
del dataset['validation']
|
208 |
tokenized_first_third = dataset.map(tokenize_function, batched=True,)
|
209 |
|
210 |
-
tokenized_first_third.save_to_disk(f'/data/{hub_id.strip()}_train_dataset')
|
211 |
print('DONE')
|
212 |
return 'RUN AGAIN TO LOAD REST OF DATA'
|
213 |
dataset = load_dataset(dataset_name.strip())
|
|
|
207 |
del dataset['validation']
|
208 |
tokenized_first_third = dataset.map(tokenize_function, batched=True,)
|
209 |
|
210 |
+
tokenized_first_third['train'].save_to_disk(f'/data/{hub_id.strip()}_train_dataset')
|
211 |
print('DONE')
|
212 |
return 'RUN AGAIN TO LOAD REST OF DATA'
|
213 |
dataset = load_dataset(dataset_name.strip())
|