p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] /home/franz.srambical/crowd-pilot/crowd_pilot/read_dataset.py (python)
#!/usr/bin/env python3
import sys
from pathlib import Path
import pandas as pd

input_dir = sys.argv[1]
for parquet_file in sorted(Path(input_dir).glob("shard_*.parquet")):
    df = pd.read_parquet(parquet_file)
    for text in df['text']:
        print(text)