File size: 470 Bytes
0f90f73
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#!/usr/bin/env python3


import tqdm
import webdataset as wds


def main(args):
    input_dataset = wds.Dataset(args.infile)
    output_dataset = wds.ShardWriter(args.outpattern)
    for rec in tqdm.tqdm(input_dataset):
        output_dataset.write(rec)


if __name__ == '__main__':
    import argparse

    aparser = argparse.ArgumentParser()
    aparser.add_argument('infile', type=str)
    aparser.add_argument('outpattern', type=str)

    main(aparser.parse_args())