File size: 783 Bytes
cbff41a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 |
import os
from datasets import load_from_disk, concatenate_datasets
def load_and_merge_datasets(directories):
datasets = []
for directory in directories:
dataset = load_from_disk(directory)
datasets.append(dataset)
merged_dataset = concatenate_datasets(datasets)
return merged_dataset
data_directories = ["/bask/projects/p/phwq4930-gbm/Zeyu/WSI_Dataset/WVLMdata_part0",
"/bask/projects/p/phwq4930-gbm/Zeyu/WSI_Dataset/WVLMdata_part1",
"/bask/projects/p/phwq4930-gbm/Zeyu/WSI_Dataset/WVLMdata_part2",
"/bask/projects/p/phwq4930-gbm/Zeyu/WSI_Dataset/WVLMdata_part3"]
merged_dataset = load_and_merge_datasets(data_directories)
merged_dataset.push_to_hub("CNX-PathLLM/TCGA-WSI-Text")
|