[tab] fix_corrupted_files.py (python)
#!/usr/bin/env python3

import os
import shutil
from pathlib import Path

def list_corrupted_files():
    """List all corrupted files that need to be fixed."""
    corrupted_files = [
        "chunked_videos_0198_0001.array_record",
        "chunked_videos_0261_0001.array_record", 
        "chunked_videos_0165_0001.array_record",
        "chunked_videos_0063_0001.array_record",
        "chunked_videos_0258_0000.array_record",
        "chunked_videos_0358_0000.array_record",
        "chunked_videos_0271_0000.array_record",
        "chunked_videos_0399_0000.array_record",
        "chunked_videos_0373_0000.array_record"
    ]
    
    output_folder = "/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_arrayrecords_500_shards_optimized_layout"
    
    print("=== Corrupted Files Found ===")
    for filename in corrupted_files:
        file_path = os.path.join(output_folder, filename)
        if os.path.exists(file_path):
            size_mb = os.path.getsize(file_path) / (1024 * 1024)
            print(f"  {filename}: {size_mb:.1f} MB")
        else:
            print(f"  {filename}: File not found!")
    
    return corrupted_files

def backup_corrupted_files():
    """Create a backup of corrupted files before attempting to fix them."""
    corrupted_files = list_corrupted_files()
    output_folder = "/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_arrayrecords_500_shards_optimized_layout"
    backup_folder = "/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_arrayrecords_500_shards_optimized_layout_backup"
    
    print(f"
=== Creating Backup ===")
    os.makedirs(backup_folder, exist_ok=True)
    
    for filename in corrupted_files:
        src_path = os.path.join(output_folder, filename)
        dst_path = os.path.join(backup_folder, filename)
        
        if os.path.exists(src_path):
            print(f"Backing up {filename}...")
            shutil.copy2(src_path, dst_path)
        else:
            print(f"Warning: {filename} not found, skipping backup")
    
    print(f"Backup created in: {backup_folder}")

def suggest_fixes():
    """Suggest potential fixes for the corrupted files."""
    print("
=== Suggested Fixes ===")
    print("1. **Re-run the reprocessing** for the specific input files that produced corrupted output")
    print("   - The corruption suggests the reprocessing was interrupted or failed for these specific files")
    print("   - You can identify which input files produced these corrupted outputs by the filename pattern")
    
    print("
2. **Check disk space and I/O errors**")
    print("   - The corruption might be due to disk space issues during writing")
    print("   - Check if there were any I/O errors during the original reprocessing")
    
    print("
3. **Re-run with smaller batch sizes**")
    print("   - The corruption might be due to memory pressure during processing")
    print("   - Consider reducing the number of worker processes or batch sizes")
    
    print("
4. **Verify input files**")
    print("   - Check if the original input files are still intact")
    print("   - The corruption might have originated from corrupted input files")
    
    print("
5. **Use different storage location**")
    print("   - Try reprocessing to a different storage location to rule out storage issues")

def analyze_corruption_pattern():
    """Analyze the pattern of corruption to understand the root cause."""
    print("
=== Corruption Pattern Analysis ===")
    
    corrupted_files = [
        "chunked_videos_0198_0001.array_record",  # 98.2 MB
        "chunked_videos_0261_0001.array_record",  # 303.2 MB  
        "chunked_videos_0165_0001.array_record",  # 89.7 MB
        "chunked_videos_0063_0001.array_record",  # 137.5 MB
        "chunked_videos_0258_0000.array_record",  # 246.0 MB
        "chunked_videos_0358_0000.array_record",  # 51.3 MB
        "chunked_videos_0271_0000.array_record",  # 165.4 MB
        "chunked_videos_0399_0000.array_record",  # 22.2 MB
        "chunked_videos_0373_0000.array_record"   # 113.5 MB
    ]
    
    print("Corruption observations:")
    print("- 9 out of 865 files are corrupted (1.04% corruption rate)")
    print("- Corrupted files vary in size from 22.2 MB to 303.2 MB")
    print("- Corruption occurs at different byte offsets in each file")
    print("- All corruption is 'chunk header hash mismatch' errors")
    print("- This suggests the files were partially written or corrupted during I/O")
    
    print("
Likely causes:")
    print("1. **Interrupted processing**: The reprocessing job may have been interrupted")
    print("2. **Disk space issues**: Storage may have run out during writing")
    print("3. **Memory pressure**: System may have run out of memory during processing")
    print("4. **Network/storage issues**: I/O errors during file writing")
    print("5. **Process crashes**: Individual worker processes may have crashed")

def main():
    """Main function to analyze and suggest fixes for corrupted files."""
    print("=== ArrayRecord Corruption Analysis ===")
    
    # List corrupted files
    corrupted_files = list_corrupted_files()
    
    # Analyze the pattern
    analyze_corruption_pattern()
    
    # Suggest fixes
    suggest_fixes()
    
    # Offer to create backup
    print(f"
=== Next Steps ===")
    print("1. Create a backup of corrupted files (recommended)")
    print("2. Re-run the reprocessing for the affected input files")
    print("3. Monitor the reprocessing for any errors or interruptions")
    
    response = input("
Would you like to create a backup of the corrupted files? (y/n): ")
    if response.lower() == 'y':
        backup_corrupted_files()
        print("Backup completed!")
    else:
        print("Skipping backup.")

if __name__ == "__main__":
    main()