File size: 1,752 Bytes
973de77
f44b91f
 
1f2b32e
973de77
38481ea
0d45303
6419e69
 
973de77
cfb4b98
05df1af
 
 
 
 
 
 
 
348fb48
 
973de77
348fb48
d811903
 
 
 
 
 
973de77
 
 
 
 
 
d811903
 
 
 
 
 
6cc96bf
 
 
 
 
 
 
ea723aa
 
 
 
6cc96bf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
# grab a dataset, prove we can save it
from datasets import load_dataset
raw_datasets = load_dataset("allocine")
raw_datasets.save_to_disk("my-arrow-datasets")
# load dataset from disk - prove we can reload it
from datasets import load_from_disk
arrow_datasets_reloaded = load_from_disk("my-arrow-datasets")
arrow_datasets_reloaded

# prove we can save in CSV
for split, dataset in raw_datasets.items():
  dataset.to_csv(f"my-dataset-{split}.csv", index=None)
data_files = { 
  "train": "my-dataset-train.csv",
  "validation": "my-dataset-validation.csv",
  "test": "my-dataset-test.csv",
}
csv_datasets_reloaded = load_dataset("csv", data_files=data_files)
csv_datasets_reloaded


# prove we can save in JSON
for split, dataset in raw_datasets.items():
  dataset.to_json(f"my-dataset-{split}.jsonl")
json_data_files = { 
  "train": "my-dataset-train.jsonl",
  "validation": "my-dataset-validation.jsonl",
  "test": "my-dataset-test.jsonl",
}
json_datasets_reloaded = load_dataset("json", data_files=json_data_files)
json_datasets_reloaded 

# prove we can save in Parquet
for split, dataset in raw_datasets.items():
  dataset.to_parquet(f"my-dataset-{split}.parquet")
parquet_data_files = { 
  "train": "my-dataset-train.parquet",
  "validation": "my-dataset-validation.parquet",
  "test": "my-dataset-test.parquet",
}
parquet_datasets_reloaded = load_dataset("parquet", data_files=parquet_data_files)
parquet_datasets_reloaded 



# prove we can save and load public local dataset on huggingface spaces
raw_datasets.save_to_disk("awacke1/my-arrow-datasets")
arrow_datasets_reloaded = load_from_disk("awacke1/my-arrow-datasets")
arrow_datasets_reloaded 

#awacke1_public_datasets = load_dataset("awacke1/my-arrow-datasets")
#awacke1_public_datasets