[tab] /fast/home/mihir.mahajan/Projects/crowd-code-processing/process_dataset.py (python)
import csv
import re
import math
import argparse
import spacy
import os
from collections import Counter

# --- Configuration ---

# Load spaCy model for Named Entity Recognition (NER)
try:
    nlp = spacy.load("en_core_web_trf")
except OSError:
    print("Spacy model 'en_core_web_trf' not found. Please run:
python -m spacy download en_core_web_trf")
    exit()

# Placeholders for redacted information
PLACEHOLDERS = {
    "name": "[REDACTED_NAME]",
    "secret": "[REDACTED_SECRET]"
}

# --- 1. High-Confidence & Context-Aware Regex Patterns ---

# Specific patterns for known API keys and secrets (low false positives)
HIGH_CONFIDENCE_REGEXES = {
    "google_api_key": r"AIza[0-9A-Za-z\-_]{35}",
    "aws_access_key_id": r"AKIA[0-9A-Z]{16}",
    "github_token": r"ghp_[a-zA-Z0-9]{36}",
    "slack_token": r"xox[baprs]-[0-9a-zA-Z]{10,48}",
    "private_key": r"-----BEGIN[A-Z\s]+PRIVATE KEY-----",
    # Common email addresses including gmail, yahoo, outlook, icloud, etc.
    "email_address": r"\b[a-zA-Z0-9._%+-]+@(gmail|yahoo|outlook|icloud|hotmail|protonmail|aol|live)\.(com|net|org|de|fr|co\.uk|ca|edu|info|ru|it|es|in|br|ch)\b"
}

# Patterns for keywords in assignment contexts
CONTEXT_AWARE_REGEX = re.compile(
    r"""
    (?i) # Case-insensitive
    \b(password|secret|token|api_key|auth|credentials|access_key) # Keywords
    \s*[:=]\s* # Assignment operator
    ['"]([a-zA-Z0-9\-_.~!@#$%^&*()+=]{8,})['"] # The secret value in quotes
    """,
    re.VERBOSE
)

# --- 2. Heuristic Patterns to Exclude from Entropy Checks ---

# Regex to identify common non-secret patterns that may have high entropy
EXCLUSION_REGEXES = {
    "file_path": re.compile(r'(\/|\\)[\w\-\.\/\\ ]+'),
    "uuid": re.compile(r'[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}'),
    "git_hash": re.compile(r'\b[a-f0-9]{7,40}\b'),
    # Simple check for variable names (snake_case or camelCase)
    "variable_name": re.compile(r'\b[a-z_][a-zA-Z0-9_]{5,}\b|\b[a-z][a-zA-Z0-9]{5,}\b'),
    "code_files": re.compile(r'[\w\-\.\/\\]+\.(json|py|sh|yml|yaml|js|java|cpp|c|h|go|rs|cs|.gif|rb|php|swift|scala|ts|html|css|log|sbatch)\b', re.IGNORECASE),
    "time_stamp": re.compile(r'\b\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}\b'),
    # Match HH:MM:SS time (24-hour)
    "simple_time_hms": re.compile(r'\b([01]?\d|2[0-3]):[0-5]\d:[0-5]\d\b'),
}

# --- 3. Entropy-Based Detection Logic ---

ENTROPY_THRESHOLD = 4.0

def shannon_entropy(s):
    """Calculates the Shannon entropy of a string."""
    if not s or len(s) < 8:  # Ignore short strings
        return 0
    return -sum(p * math.log2(p) for p in (count / len(s) for count in Counter(s).values()))

# --- Anonymization Workflow Functions ---

def anonymize_text(text, redaction_log):
    """
    Runs text through a multi-pass anonymization workflow.
    Returns the anonymized text.
    """
    if not isinstance(text, str) or not text.strip():
        return text

    replacements = set()

    # --- Pass 1: High-confidence and Context-aware Detections ---
    # NER for names
    doc = nlp(text)
    for ent in doc.ents:
        if ent.label_ == "PERSON":
            replacements.add((ent.text, PLACEHOLDERS["name"]))

    # High-confidence secret patterns
    for pattern in HIGH_CONFIDENCE_REGEXES.values():
        for match in re.finditer(pattern, text):
            replacements.add((match.group(0), PLACEHOLDERS["secret"]))

    # Context-aware keyword assignments
    for match in re.finditer(CONTEXT_AWARE_REGEX, text):
        # The secret is in group 2
        replacements.add((match.group(2), PLACEHOLDERS["secret"]))

    # --- Pass 2: Heuristic Filtering & Entropy Analysis ---
    # Create a temporary text with already found secrets replaced to avoid re-checking
    temp_text = text
    for original, _ in replacements:
        temp_text = temp_text.replace(original, "")

    # Tokenize the remaining text
    tokens = re.split(r'[\s\'"(),;=:[\]{}]', temp_text)
    for token in tokens:
        if not token or len(token) < 10:  # Skip empty or short tokens
            continue

        # Check if the token matches any exclusion patterns
        is_excluded = False
        for pattern in EXCLUSION_REGEXES.values():
            if pattern.fullmatch(token):
                is_excluded = True
                break
        if is_excluded:
            continue

        # If not excluded, check entropy
        # if shannon_entropy(token) > ENTROPY_THRESHOLD:
            # replacements.add((token, PLACEHOLDERS["secret"]))

    # --- Pass 3: Apply all replacements ---
    if not replacements:
        return text

    # Sort replacements by length of the original string (desc) to replace longer matches first
    sorted_replacements = sorted(list(replacements), key=lambda x: len(x[0]), reverse=True)

    final_text = text
    for original, placeholder in sorted_replacements:
        if original in final_text:
            final_text = final_text.replace(original, placeholder)
            # Add to the log for the markdown file
            redaction_log.append((original, placeholder))
            
    return final_text

def anonymize_csv(input_file, output_csv_file, output_md_file):
    """
    Reads a CSV, anonymizes it, writes the result to a new CSV,
    and logs all redactions to a Markdown file.
    """
    redaction_log = []
    try:
        with open(input_file, mode='r', encoding='utf-8') as infile, \
             open(output_csv_file, mode='w', encoding='utf-8', newline='') as outfile:
            
            reader = csv.reader(infile)
            writer = csv.writer(outfile)
            
            header = next(reader, None)
            if header:
                writer.writerow(header)

            for row_num, row in enumerate(reader, 1):
                anonymized_row = [anonymize_text(cell, redaction_log) for cell in row]
                writer.writerow(anonymized_row)
                if row_num % 100 == 0:
                    print(f"Processed {row_num} rows...")
        
        print(f"
Anonymization complete. Anonymized data saved to '{output_csv_file}'.")

        # Write the redaction log to a Markdown file
        if redaction_log:
            # Remove duplicates for a cleaner log
            unique_log = sorted(list(set(redaction_log)), key=lambda x: x[0])
            with open(output_md_file, 'w', encoding='utf-8') as md_file:
                md_file.write("# Anonymization Redaction Log

")
                md_file.write(f"A total of **{len(unique_log)}** unique values were redacted.

")
                md_file.write("| Original Value | Replaced With |
")
                md_file.write("|---|---|
")
                for original, placeholder in unique_log:
                    # Escape pipe characters for markdown table compatibility
                    original_escaped = original.replace('|', '\|')
                    md_file.write(f"| `{original_escaped}` | {placeholder} |
")
            print(f"Redaction log saved to '{output_md_file}'.")
        else:
            print("No sensitive data was found to redact.")

    except FileNotFoundError:
        print(f"Error: The file '{input_file}' was not found.")
    except Exception as e:
        print(f"An unexpected error occurred: {e}")

# --- Command-Line Interface ---

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Anonymize a CSV file using a nuanced, multi-pass strategy."
    )
    parser.add_argument(
        "input_file",
        help="The path to the input CSV file to be anonymized."
    )
    args = parser.parse_args()
    
    base_filename = os.path.splitext(args.input_file)[0]
    output_csv_filename = f"{base_filename}_anonymized.csv"
    output_md_filename = f"{base_filename}_redaction_log.md"

    print(f"Starting anonymization for '{args.input_file}'...")
    anonymize_csv(args.input_file, output_csv_filename, output_md_filename)