tests/test_coco_dataset_script.py

import csv
import json
import os

import datasets
import pandas as pd
import numpy as np

ds = datasets.load_dataset('./coco_dataset_script.py', data_dir='/home/33611/caption/')
ds = ds['train']


def transform(example):

    example['pixel_values'] = np.load(example['pixels_file'])
    return example


# ds = ds.map(transform)

n = 0
for x in ds:
    n += 1
    assert os.path.isfile(x['image_file'])
    if n == 10:
        print(x)
        break