[content] process_dataset.py (python)
import csv
import re
import math
import argparse
import spacy
from collections import Counter

# Load the spaCy model for Named Entity Recognition (NER)
nlp = spacy.load("en_core_web_sm")

# --- Anonymization Configuration ---

# Regex patterns for various secrets
SECRET_REGEXES = {
    "api_key": r"(?i)(api_key|secret_key|token)[\s=:'\"]+([a-zA-Z0-9\-_]{20,})",
    "google_api_key": r"AIza[0-9A-Za-z\-_]{35}",
    "aws_access_key": r"AKIA[0-9A-Z]{16}",
    "private_key": r"-----BEGIN[A-Z\s]+PRIVATE KEY-----"
}

# Entropy threshold for detecting random strings (potential secrets)
ENTROPY_THRESHOLD = 4.0

# Placeholders for redacted information
PLACEHOLDERS = {
    "name": "[REDACTED_NAME]",
    "secret": "[REDACTED_SECRET]"
}

# --- Detection Functions ---

def shannon_entropy(s):
    """Calculates the Shannon entropy of a string."""
    if not s:
        return 0
    entropy = 0
    for char_code in range(256):
        char = chr(char_code)
        p_char = s.count(char) / len(s)
        if p_char > 0:
            entropy += -p_char * math.log2(p_char)
    return entropy

def find_high_entropy_strings(text):
    """Finds and replaces high-entropy strings in text."""
    words = text.split()
    redacted_words = []
    for word in words:
        if shannon_entropy(word) > ENTROPY_THRESHOLD:
            redacted_words.append(PLACEHOLDERS["secret"])
        else:
            redacted_words.append(word)
    return " ".join(redacted_words)

def redact_names(text):
    """Identifies and redacts names from text using NER."""
    doc = nlp(text)
    redacted_text = text
    for ent in doc.ents:
        if ent.label_ == "PERSON":
            redacted_text = redacted_text.replace(ent.text, PLACEHOLDERS["name"])
    return redacted_text

def apply_regex_redaction(text):
    """Applies regex patterns to find and redact secrets."""
    for key_type, pattern in SECRET_REGEXES.items():
        text = re.sub(pattern, f"{key_type}='{PLACEHOLDERS['secret']}'", text)
    return text

# --- Main Anonymization Logic ---

def anonymize_text(text):
    """Runs a single piece of text through all anonymization steps."""
    if not isinstance(text, str):
        return text

    # 1. Redact names using NER
    text = redact_names(text)
    
    # 2. Apply regex for known secret patterns
    text = apply_regex_redaction(text)

    # 3. Find and redact high-entropy strings
    text = find_high_entropy_strings(text)

    return text

def anonymize_csv(input_file, output_file):
    """
    Reads a CSV file, anonymizes its content, and writes to a new file.
    """
    try:
        with open(input_file, mode='r', encoding='utf-8') as infile, \
             open(output_file, mode='w', encoding='utf-8', newline='') as outfile:
            
            reader = csv.reader(infile)
            writer = csv.writer(outfile)
            
            # Process header
            header = next(reader, None)
            if header:
                writer.writerow(header)

            # Process rows
            for row in reader:
                anonymized_row = [anonymize_text(cell) for cell in row]
                writer.writerow(anonymized_row)
        
        print(f"
Anonymization complete. Anonymized data saved to '{output_file}'.")

    except FileNotFoundError:
        print(f"Error: The file '{input_file}' was not found.")
    except Exception as e:
        print(f"An unexpected error occurred: {e}")

# --- Command-Line Interface ---

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Anonymize a CSV file by redacting names, passwords, API keys, and other secrets."
    )
    parser.add_argument(
        "input_file",
        help="The path to the input CSV file to be anonymized."
    )
    args = parser.parse_args()
    
    # Define the output filename
    output_filename = args.input_file.replace('.csv', '_anonymized.csv')

    print(f"Starting anonymization for '{args.input_file}'...")
    anonymize_csv(args.input_file, output_filename)