File size: 894 Bytes
be473e6
 
3573a39
 
 
be473e6
 
0607989
be473e6
 
 
 
 
3573a39
be473e6
 
0607989
be473e6
 
3573a39
 
 
be473e6
 
 
 
 
 
3573a39
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
import logging

import datasets


def check_dataset_and_get_config(dataset_id):
    try:
        configs = datasets.get_dataset_config_names(dataset_id, trust_remote_code=True)
        return configs
    except Exception:
        # Dataset may not exist
        return None


def check_dataset_and_get_split(dataset_id, dataset_config):
    try:
        ds = datasets.load_dataset(dataset_id, dataset_config, trust_remote_code=True)
    except Exception as e:
        # Dataset may not exist
        logging.warning(
            f"Failed to load dataset {dataset_id} with config {dataset_config}: {e}"
        )
        return None
    try:
        splits = list(ds.keys())
        return splits
    except Exception as e:
        # Dataset has no splits
        logging.warning(
            f"Dataset {dataset_id} with config {dataset_config} has no splits: {e}"
        )
        return None