p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] utils/dataloader.py (python)
    arrayrecord_files = []
    
    for tfrecord_file in tfrecord_files:
        # Create output filename based on input filename
        output_filename = tfrecord_file.stem + ".arrayrecord"
        output_file = os.path.join(output_folder, output_filename)
        
        # Create dataset from single TFRecord file
        dataset = tf.data.TFRecordDataset(str(tfrecord_file))
        
        def parse_tfrecord(example_proto):
            """Parse a single TFRecord example."""
            parsed_features = tf.io.parse_single_example(example_proto, feature_description)
            return {
                'raw_video': parsed_features['raw_video'].numpy(),
                'sequence_length': int(parsed_features['sequence_length'].numpy())
            }
        
        # Process records and write to single ArrayRecord file
        record_count = 0
        with ArrayRecordWriter(output_file) as writer:
            for record in dataset:
                parsed_record = parse_tfrecord(record)
                writer.write(parsed_record)
                record_count += 1
[content] experiments/convert_knoms_to_arrayrecords.py (python)


convert_tfrecords_to_arrayrecords(
    tfrecord_folder="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards",
    output_folder="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_arrayrecords_500_shards",
)