p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] calculate_avg_frames.py (python)
#!/usr/bin/env python3
"""
Script to calculate the average number of frames per video in an ArrayRecord file.
"""

from array_record.python.array_record_module import ArrayRecordReader
import pickle
import sys

def calculate_avg_frames(array_record_path):
    """
    Calculate the average number of frames per video in an ArrayRecord file.
    
    Args:
        array_record_path: Path to the ArrayRecord file
        
    Returns:
        Average number of frames per video
    """
    total_frames = 0
    num_videos = 0
    
    print(f"Reading ArrayRecord file: {array_record_path}")
    
    reader = ArrayRecordReader(array_record_path)
    for record in reader:
        data = pickle.loads(record)
        total_frames += data["sequence_length"]
        num_videos += 1
        
        # Print progress every 1000 videos
        if num_videos % 1000 == 0:
            print(f"Processed {num_videos} videos...")
    
    reader.close()
    
    if num_videos > 0:
        avg_frames = total_frames / num_videos
        print(f"
Results:")
        print(f"Total videos: {num_videos}")
        print(f"Total frames: {total_frames}")
        print(f"Average frames per video: {avg_frames:.2f}")
        return avg_frames
    else:
        print("No videos found in the ArrayRecord file.")
        return 0

if __name__ == "__main__":
    # Default path to the ArrayRecord file
    array_record_path = "/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/dummy_arrayrecords_600_shards/videos_0010.array_record"
    
    # Allow command line argument to override the path
    if len(sys.argv) > 1:
        array_record_path = sys.argv[1]
    
    try:
        calculate_avg_frames(array_record_path)
    except Exception as e:
        print(f"Error: {e}")
        sys.exit(1)
[content] calculate_avg_frames.py (python)
#!/usr/bin/env python3
"""
Script to calculate the average number of frames per video in an ArrayRecord file.
"""

import grain
import pickle
import sys

def calculate_avg_frames(array_record_path):
    """
    Calculate the average number of frames per video in an ArrayRecord file.
    
    Args:
        array_record_path: Path to the ArrayRecord file
        
    Returns:
        Average number of frames per video
    """
    total_frames = 0
    num_videos = 0
    
    print(f"Reading ArrayRecord file: {array_record_path}")
    
    # Use Grain's ArrayRecordDataSource to read the file
    data_source = grain.sources.ArrayRecordDataSource([array_record_path])
    num_records = len(data_source)
    
    print(f"Found {num_records} records in the file")
    
    for i in range(num_records):
        record_bytes = data_source[i]
        data = pickle.loads(record_bytes)
        total_frames += data["sequence_length"]
        num_videos += 1
        
        # Print progress every 1000 videos
        if num_videos % 1000 == 0:
            print(f"Processed {num_videos}/{num_records} videos...")
    
    if num_videos > 0:
        avg_frames = total_frames / num_videos
        print(f"
Results:")
        print(f"Total videos: {num_videos}")
        print(f"Total frames: {total_frames}")
        print(f"Average frames per video: {avg_frames:.2f}")
        return avg_frames
    else:
        print("No videos found in the ArrayRecord file.")
        return 0

if __name__ == "__main__":
    # Default path to the ArrayRecord file
    array_record_path = "/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/dummy_arrayrecords_600_shards/videos_0010.array_record"
    
    # Allow command line argument to override the path
    if len(sys.argv) > 1:
        array_record_path = sys.argv[1]
    
    try:
        calculate_avg_frames(array_record_path)
    except Exception as e:
        print(f"Error: {e}")
        sys.exit(1)