hf_data_loader.py

from datasets import load_dataset

import config


def load_raw_dataset_as_pandas():
    return load_dataset(config.HF_RAW_DATASET_NAME,
                        split=config.HF_RAW_DATASET_SPLIT,
                        token=config.HF_TOKEN,
                        cache_dir=config.CACHE_DIR).to_pandas()


load_raw_dataset_as_pandas()