p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] scripts/file_duplicate_checker.py (python)
import os
from collections import defaultdict
from tqdm import tqdm

def find_duplicate_filenames(root_dir):
    filenames = defaultdict(list)
    file_count = 0

    # Use tqdm with manual update and no percentage/ETA bar
    pbar = tqdm(desc="Files scanned", unit="file", dynamic_ncols=True, bar_format="{desc}: {n_fmt}")

    # Walk the directory recursively
    for dirpath, _, files in os.walk(root_dir):
        for file in files:
            full_path = os.path.join(dirpath, file)
            if os.path.isfile(full_path):
                filenames[file].append(full_path)
                file_count += 1
                pbar.update(1)

    pbar.close()

    # Print duplicates
    duplicates = {name: paths for name, paths in filenames.items() if len(paths) > 1}
    if duplicates:
        print("
Duplicate filenames found:
")
        for name, paths in duplicates.items():
            print(f"Filename: {name}")
            for path in paths:
                print(f" - {path}")
            print()
    else:
        print("
No duplicate filenames found.")

if __name__ == "__main__":
    import sys
    if len(sys.argv) < 2:
        print("Usage: python find_duplicates.py <directory_path>")
    else:
        find_duplicate_filenames(sys.argv[1])