dev

app.py

@@ -113,7 +113,7 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
         #max_length = 512
         # Load the dataset        
         max_length = model.get_input_embeddings().weight.shape[0]
-        print(f"TESTING: {os.access(f'/data/{hub_id.strip()}_train_dataset', os.R_OK)}")
+        
         try:
             saved_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset')
             if os.access(f'/data/{hub_id.strip()}_test_dataset', os.R_OK):
@@ -147,11 +147,11 @@ def fine_tune_model(model, dataset_name, hub_id, api_key, num_epochs, batch_size
                 print("FOUND VALIDATION")
                 saved_dataset = load_from_disk(f'/data/{hub_id.strip()}_train_dataset2')     
                 third_third = dataset['train'].select(range(third_size*2, train_size))
-                #dataset['train'] = third_third
+                dataset['train'] = third_third
                 print(dataset)
                 print(dataset.keys())
-                tokenized_second_half = tokenize_function(third_third)
-                #tokenized_second_half = dataset.map(tokenize_function)
+                #tokenized_second_half = tokenize_function(third_third)
+                tokenized_second_half = dataset.map(tokenize_function)
                 dataset['train'] = concatenate_datasets([saved_dataset['train'], tokenized_second_half['train']])
                 dataset['train'].save_to_disk(f'/data/{hub_id.strip()}_train_dataset3')
                 return 'THIRD THIRD LOADED'