File size: 556 Bytes
e829671
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
#!/usr/bin/env python3

import glob
import os

import random
import requests

from datasets import load_dataset, Dataset, DatasetDict

folder_path = 'isp-data-json/'  # Replace with your folder path

# Search for all .json files in the folder
json_files = glob.glob(os.path.join(folder_path, '*.jsonl'))

path = 'pminervini/inverse-scaling'

for json_path in json_files:
    base_name = os.path.basename(json_path)
    name = base_name.split("_")[0]

    ds = load_dataset("json", data_files={'data': json_path})
    ds.push_to_hub(path, config_name=name)