p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] /home/franz.srambical/crowd-pilot/crowd_pilot/read_dataset.py (python)
def main():
    input_dir = sys.argv[1]
    for parquet_file in sorted(Path(input_dir).glob("shard_*.parquet")):
        df = pd.read_parquet(parquet_file)
        for text in df['text']:
            print(text)