CLIP-SAE-TypoAttack-Robust-ViT-L-14

A model trained with SAE-guided attention head hinge loss

ℹ️ Goal: Reduction of SAE-derived 'text-reading' directions contribution (typographic attack vulnerability).
💡 Extremely robust (>90% typographic attack benchmark accuracy) model with clean features, attention maps.
💡 Mitigation of high-norm outlier tokens; lower-norm spurious global aggregation makes it a quantization / distillation candidate.
💡 Model can still 'read text'; it just won't prefer text, if there's a better (visual salience) choice.
👉 You can find the code for the SAE and to fine-tune CLIP on github.com/zer0int/CLIP-SAE-hinge-fine-tune

Patch norms, Attention Heatmaps, Feature Act Max, Caveat:

A fine-tune of CLIP-L. Original model: openai/clip-vit-large-patch14

Love ❤️ this CLIP?

ᐅ Buy me a coffee on Ko-Fi ☕

Or click here for address to send 🪙₿ BTC

3PscBrWYvrutXedLmvpcnQbE12Py8qLqMK

---

Evaluation Results

TA = Typographic Attack ZS dataset

Section	Measurement / Task	Pre-Trained	Regression	SAE-Typo ✮
zer0int / RTA-100 (TA)	NoRTA	0.9880	0.9920	0.9880
HF Datasets	RTA	0.4310	0.7880	0.9550
	SynthRTA	0.3890	0.8050	0.9550
BLISS-e-V / SCAM (TA)	NoSCAM	0.9905	0.9897	0.9819
HF Datasets	SCAM	0.4191	0.8046	0.9053
	SynthSCAM	0.3227	0.8029	0.9355
ILSVRC2012 Linear Probe	Top-1	72.35%	70.94%	64.95%
git/zer0int	Top-5	93.42%	93.29%	90.26%
ObjectNet MVT (ZS)	Accuracy	0.8652	0.8717	0.7690
git/zer0int
ImageNet 1k (ZS)	acc1	0.32696	0.4566	0.4022
LAION/CLIP-Benchmark	acc5	0.52997	0.6817	0.6342
	mean_per_class_recall	0.32609	0.4547	0.4005
VoC-2007 (ZS)	mAP	0.7615	0.8523	0.8548
LAION/CLIP-Benchmark
mscoco ZS Retrieval	image_retrieval_recall@5	0.2196	0.3510	0.3434
LAION/CLIP-Benchmark	text_retrieval_recall@5	0.3032	0.5042	0.4916
xm3600 ZS Retrieval	image_retrieval_recall@5	0.30593	0.4254	0.4131
LAION/CLIP-Benchmark	text_retrieval_recall@5	0.24293	0.4091	0.4007
Sugar_Crepe (PT)	add_obj: acc	0.7842	0.9627	0.9646
git/zer0int	add_att: acc	0.7168	0.9205	0.8974
	replace_obj: acc	0.9407	0.9752	0.9764
	replace_att: acc	0.7919	0.8579	0.8490
	replace_rel: acc	0.6529	0.7752	0.7368
	swap_obj: acc	0.6041	0.7224	0.6653
	swap_att: acc	0.6261	0.7282	0.7252
Flickr-8k Cross-modal	Euclidean Gap (center)	0.8299	0.6788	0.6377
git/zer0int	Geometry:Image cone_R	0.7481	0.4514	0.4256
	Geometry:Text cone_R	0.5908	0.4481	0.4263
	Image-Text Cos Sim (mean)	0.2754	0.3555	0.3507
	Text-Text Cos Sim (mean)	0.6762	0.6591	0.6567
	Image-Image Cos Sim (mean)	0.5594	0.2034	0.1809

SAE-Typo: This model. CLIP-SAE-TypoAttack-Robust-ViT-L-14
Pretrained: openai/clip-vit-large-patch14
Regression: CLIP-Regression-ViT-L-14

👉 CLICK to expand code for reproducing ZERO-SHOT typographic attack benchmarks ⚡💻

from __future__ import annotations
import argparse
import json
from collections import defaultdict
from pathlib import Path
from typing import Any
import torch
import torch.nn.functional as F
from datasets import load_dataset
from PIL import Image
from tqdm import tqdm
from transformers import CLIPModel, CLIPProcessor


MODELS = [
    ("pretrained-clip", "ViT-L/14"),
    ("regression-clip", "zer0int/CLIP-Regression-ViT-L-14"),
    ("sae-robust-clip", "zer0int/CLIP-SAE-TypoAttack-Robust-ViT-L-14"), 
]


def normalize_feature(x: torch.Tensor, eps: float = 1e-12) -> torch.Tensor:
    return x / x.norm(dim=-1, keepdim=True).clamp_min(eps)


def unwrap_feature_output(x: Any, kind: str) -> torch.Tensor:
    """
    Robustly extract CLIP embedding tensors from transformers outputs.
    """
    if isinstance(x, torch.Tensor):
        return x

    preferred_attrs = ["pooler_output", "last_hidden_state"]
    if kind == "image":
        preferred_attrs = ["image_embeds"] + preferred_attrs
    elif kind == "text":
        preferred_attrs = ["text_embeds"] + preferred_attrs

    for attr in preferred_attrs:
        if hasattr(x, attr):
            v = getattr(x, attr)
            if isinstance(v, torch.Tensor):
                if attr == "last_hidden_state":
                    return v[:, 0, :]
                return v

    if isinstance(x, (tuple, list)):
        tensors = [v for v in x if isinstance(v, torch.Tensor)]
        if tensors:
            for t in tensors:
                if t.ndim == 2:
                    return t
            if tensors[0].ndim == 3:
                return tensors[0][:, 0, :]
            return tensors[0]

    raise TypeError(f"Could not unwrap {kind} feature output of type {type(x)}")


def batched(iterable, batch_size: int):
    for start in range(0, len(iterable), batch_size):
        yield start, iterable[start : start + batch_size]


def get_image(x: Any) -> Image.Image:
    """
    datasets.Image usually gives PIL.Image directly.
    Keep fallback for dict/path variants.
    """
    if isinstance(x, Image.Image):
        return x.convert("RGB")
    if isinstance(x, dict):
        if x.get("bytes") is not None:
            import io
            return Image.open(io.BytesIO(x["bytes"])).convert("RGB")
        if x.get("path") is not None:
            return Image.open(x["path"]).convert("RGB")
    if isinstance(x, (str, Path)):
        return Image.open(x).convert("RGB")
    raise TypeError(f"Unsupported image field type: {type(x)}")


@torch.inference_mode()
def evaluate_model(
    ds,
    model_alias: str,
    model_name_or_path: str,
    device: torch.device,
    batch_size: int,
    fp16: bool,
) -> dict[str, Any]:
    print(f"\n[load] {model_alias}: {model_name_or_path}")

    processor = CLIPProcessor.from_pretrained(model_name_or_path)
    model = CLIPModel.from_pretrained(model_name_or_path)
    model = model.eval().to(device)

    if fp16:
        model = model.half()

    totals = defaultdict(int)
    corrects = defaultdict(int)
    margins = defaultdict(list)

    indices = list(range(len(ds)))

    for _, batch_indices in tqdm(list(batched(indices, batch_size)), desc=f"eval {model_alias}"):
        examples = [ds[i] for i in batch_indices]

        images = [get_image(ex["image"]) for ex in examples]
        object_labels = [str(ex["object_label"]) for ex in examples]
        attack_words = [str(ex["attack_word"]) for ex in examples]
        types = [str(ex["type"]) for ex in examples]

        # Two prompts per image: object prompt and attack-word prompt.
        texts = []
        for obj, atk in zip(object_labels, attack_words):
            texts.append(f"a photo of a {obj}")
            texts.append(f"a photo of a {atk}")

        image_inputs = processor(images=images, return_tensors="pt")
        text_inputs = processor(text=texts, return_tensors="pt", padding=True, truncation=True)

        image_inputs = {k: v.to(device) for k, v in image_inputs.items()}
        text_inputs = {k: v.to(device) for k, v in text_inputs.items()}

        if fp16:
            image_inputs = {
                k: (v.half() if torch.is_floating_point(v) else v)
                for k, v in image_inputs.items()
            }

        image_features_raw = model.get_image_features(**image_inputs)
        text_features_raw = model.get_text_features(**text_inputs)

        image_features = unwrap_feature_output(image_features_raw, kind="image")
        text_features = unwrap_feature_output(text_features_raw, kind="text")

        image_features = normalize_feature(image_features.float())
        text_features = normalize_feature(text_features.float())

        text_features = text_features.view(len(examples), 2, -1)
        object_sims = (image_features * text_features[:, 0, :]).sum(dim=-1)
        attack_sims = (image_features * text_features[:, 1, :]).sum(dim=-1)
        batch_margins = object_sims - attack_sims
        batch_preds = batch_margins > 0

        for typ, ok, margin in zip(types, batch_preds.tolist(), batch_margins.tolist()):
            totals[typ] += 1
            corrects[typ] += int(ok)
            margins[typ].append(float(margin))

    total_all = sum(totals.values())
    correct_all = sum(corrects.values())

    results = {
        "model_alias": model_alias,
        "model_name_or_path": model_name_or_path,
        "by_type": {},
        "all": {
            "n": total_all,
            "correct": correct_all,
            "accuracy": correct_all / total_all if total_all else None,
            "mean_margin_object_minus_attack": (
                sum(m for vals in margins.values() for m in vals) / total_all if total_all else None
            ),
        },
    }

    for typ in sorted(totals):
        vals = margins[typ]
        results["by_type"][typ] = {
            "n": totals[typ],
            "correct": corrects[typ],
            "accuracy": corrects[typ] / totals[typ] if totals[typ] else None,
            "mean_margin_object_minus_attack": sum(vals) / len(vals) if vals else None,
            "min_margin": min(vals) if vals else None,
            "max_margin": max(vals) if vals else None,
        }

    del model
    if torch.cuda.is_available():
        torch.cuda.empty_cache()

    return results


def print_results(results: list[dict[str, Any]]) -> None:
    print("\n=== ZERO-SHOT RESULTS ===")

    for res in results:
        print(f"\n[{res['model_alias']}] {res['model_name_or_path']}")
        for typ in ["NoRTA", "RTA", "SynthRTA"]:
            item = res["by_type"].get(typ)
            if item is None:
                print(f"  {typ:8s}: missing")
                continue
            print(
                f"  {typ:8s}: "
                f"acc={item['accuracy']:.4f} "
                f"correct={item['correct']}/{item['n']} "
                f"mean_margin={item['mean_margin_object_minus_attack']:+.4f}"
            )

        all_item = res["all"]
        print(
            f"  {'ALL':8s}: "
            f"acc={all_item['accuracy']:.4f} "
            f"correct={all_item['correct']}/{all_item['n']} "
            f"mean_margin={all_item['mean_margin_object_minus_attack']:+.4f}"
        )


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="ZS binary-choice eval for local RTA-100-Triplet HF dataset.")
    parser.add_argument("--dataset-path", type=str, default="zer0int/RTA-100-Triplet", help="Local HF dataset repo path or HF repo id.")
    parser.add_argument("--split", default="train")
    parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
    parser.add_argument("--batch-size", type=int, default=64)
    parser.add_argument("--fp16", action="store_true")
    parser.add_argument("--output-json", default=None)
    parser.add_argument("--trust-remote-code", action="store_true")
    return parser.parse_args()


def main() -> None:
    args = parse_args()
    device = torch.device(args.device)

    print(f"[load_dataset] {args.dataset_path} split={args.split}")
    ds = load_dataset(args.dataset_path, split=args.split, trust_remote_code=args.trust_remote_code)

    print("\n[dataset]")
    print(ds)

    print("[features]")
    print(ds.features)

    results = []
    for alias, model_name_or_path in MODELS:
        res = evaluate_model(
            ds=ds,
            model_alias=alias,
            model_name_or_path=model_name_or_path,
            device=device,
            batch_size=args.batch_size,
            fp16=args.fp16,
        )
        results.append(res)

    print_results(results)

    if args.output_json:
        out = Path(args.output_json)
        out.parent.mkdir(parents=True, exist_ok=True)
        out.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
        print(f"\n[wrote] {out}")


if __name__ == "__main__":
    main()

---

Fun summary of the model, in a nutshell. For code, check my github. :)