Upload 6 files

detree/utils/detectors/Fast_DetectGPT_evaluation.py

@@ -0,0 +1,137 @@
+import logging
+import random
+import numpy as np
+import torch
+import argparse
+import json
+from tqdm import tqdm
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from ..utils import evaluate_metrics
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+
+def get_sampling_discrepancy_analytic(logits_ref, logits_score, labels):
+    assert logits_ref.shape[0] == 1
+    assert logits_score.shape[0] == 1
+    assert labels.shape[0] == 1
+    if logits_ref.size(-1) != logits_score.size(-1):
+        # print(f"WARNING: vocabulary size mismatch {logits_ref.size(-1)} vs {logits_score.size(-1)}.")
+        vocab_size = min(logits_ref.size(-1), logits_score.size(-1))
+        logits_ref = logits_ref[:, :, :vocab_size]
+        logits_score = logits_score[:, :, :vocab_size]
+
+    labels = labels.unsqueeze(-1) if labels.ndim == logits_score.ndim - 1 else labels
+    lprobs_score = torch.log_softmax(logits_score, dim=-1)
+    probs_ref = torch.softmax(logits_ref, dim=-1)
+    log_likelihood = lprobs_score.gather(dim=-1, index=labels).squeeze(-1)
+    mean_ref = (probs_ref * lprobs_score).sum(dim=-1)
+    var_ref = (probs_ref * torch.square(lprobs_score)).sum(dim=-1) - torch.square(mean_ref)
+    discrepancy = (log_likelihood.sum(dim=-1) - mean_ref.sum(dim=-1)) / var_ref.sum(dim=-1).sqrt()
+    discrepancy = discrepancy.mean()
+    return discrepancy.item()
+
+def get_text_crit(text, args, model_config):
+    tokenized = model_config["scoring_tokenizer"](text, return_tensors="pt",
+                                  return_token_type_ids=False)
+    labels = tokenized.input_ids[:, 1:]
+    with torch.no_grad():
+        logits_score = model_config["scoring_model"](**tokenized).logits[:, :-1]
+        if args.reference_model == args.scoring_model:
+            logits_ref = logits_score
+        else:
+            tokenized = model_config["reference_tokenizer"](text, return_tensors="pt",
+                                       return_token_type_ids=False)
+            assert torch.all(tokenized.input_ids[:, 1:] == labels), "Tokenizer is mismatch."
+            logits_ref = model_config["reference_model"](**tokenized).logits[:, :-1]
+        text_crit = get_sampling_discrepancy_analytic(logits_ref, logits_score, labels)
+
+    return text_crit
+
+def load_jsonl(file_path):
+    out = []
+    with open(file_path, mode='r', encoding='utf-8') as jsonl_file:
+        for line in jsonl_file:
+            item = json.loads(line)
+            out.append(item)
+    print(f"Loaded {len(out)} examples from {file_path}")
+    return out
+
+def dict2str(metrics):
+    out_str=''
+    for key in metrics.keys():
+        out_str+=f"{key}:{metrics[key]} "
+    return out_str
+
+def experiment(args):
+    # load model
+    logging.info(f"Loading reference model of type {args.reference_model}...")
+    reference_tokenizer = AutoTokenizer.from_pretrained(args.reference_model)
+    reference_model = AutoModelForCausalLM.from_pretrained(args.reference_model,device_map="auto")
+    reference_model.eval()
+    reference_model
+    scoring_tokenizer = AutoTokenizer.from_pretrained(args.scoring_model)
+    scoring_model = AutoModelForCausalLM.from_pretrained(args.scoring_model,device_map="auto")
+    scoring_model.eval()
+    scoring_model
+
+    model_config = {
+        "reference_tokenizer": reference_tokenizer,
+        "reference_model": reference_model,
+        "scoring_tokenizer": scoring_tokenizer,
+        "scoring_model": scoring_model,
+    }
+
+    logging.info(f"Test in {args.test_data_path}")
+    test_data = load_jsonl(args.test_data_path)
+    random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    np.random.seed(args.seed)
+    random.shuffle(test_data)
+    predictions = []
+    labels = []
+    st = time.time()
+    for i, item in tqdm(enumerate(test_data), total=len(test_data)):
+        if i>=100:  # for debugging, only use the first 100 samples
+            break
+        text = item["text"]
+        label = item["label"]
+        src = item["src"]
+        text_crit = get_text_crit(text, args, model_config)
+        if text_crit is None or np.isnan(text_crit) or np.isinf(text_crit):
+            text_crit = 0
+        if 'human' in src:
+            labels.append(0)
+        else:
+            labels.append(1)
+        predictions.append(text_crit)
+    ed = time.time()
+    print((ed - st) / 100)
+    # metric = evaluate_metrics(labels, predictions)
+    # print(dict2str(metric))
+    # with open("runs/val-other_detector.txt",'a+') as f:
+    #     f.write(f"Fast DetectGPT {args.test_data_path} {args.scoring_model} {args.reference_model}\n")
+    #     f.write(f"{dict2str(metric)}\n")
+
+
+
+    # logging.info(f"{result}")
+    # with open(filename.split(".json")[0] + "_Fast_DetectGPT_data.json", "w") as f:
+    #     json.dump(test_data, f, indent=4)
+
+    # with open(filename.split(".json")[0] + "_Fast_DetectGPT_result.json", "w") as f:
+    #     json.dump(result, f, indent=4)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--test_data_path', type=str, default='/path/to/RealBench/Beemo/Llama_edited/test.jsonl',
+                        help="Path to the test data. could be several files with ','. "
+                             "Note that the data should have been perturbed.")
+    parser.add_argument('--reference_model', type=str, default="EleutherAI/gpt-neo-2.7B")
+    parser.add_argument('--scoring_model', type=str, default="EleutherAI/gpt-j-6B")
+    parser.add_argument('--DEVICE0', default="cuda:0", type=str, required=False)
+    parser.add_argument('--DEVICE1', default="cuda:1", type=str, required=False)
+    parser.add_argument('--seed', default=2023, type=int, required=False)
+    args = parser.parse_args()
+
+    experiment(args)

detree/utils/detectors/MAGE.py

@@ -0,0 +1,384 @@
+import argparse
+import json
+import logging
+import random
+import re
+from itertools import chain
+from pathlib import Path
+from typing import Sequence
+
+import numpy as np
+import regex
+import torch
+from cleantext import clean
+from tqdm import tqdm
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+from ..utils import evaluate_metrics
+
+_LOG_PATH = Path(__file__).resolve().parents[3] / "runs" / "val-other_detector.txt"
+_LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
+
+class MosesPunctNormalizer:
+
+    EXTRA_WHITESPACE = [  # lines 21 - 30
+        (r"\r", r""),
+        (r"\(", r" ("),
+        (r"\)", r") "),
+        (r" +", r" "),
+        (r"\) ([.!:?;,])", r")\g<1>"),
+        (r"\( ", r"("),
+        (r" \)", r")"),
+        (r"(\d) %", r"\g<1>%"),
+        (r" :", r":"),
+        (r" ;", r";"),
+    ]
+
+    NORMALIZE_UNICODE_IF_NOT_PENN = [(r"`", r"'"), (r"''", r' " ')]  # lines 33 - 34
+
+    NORMALIZE_UNICODE = [  # lines 37 - 50
+        ("„", r'"'),
+        ("“", r'"'),
+        ("”", r'"'),
+        ("–", r"-"),
+        ("—", r" - "),
+        (r" +", r" "),
+        ("´", r"'"),
+        ("([a-zA-Z])‘([a-zA-Z])", r"\g<1>'\g<2>"),
+        ("([a-zA-Z])’([a-zA-Z])", r"\g<1>'\g<2>"),
+        ("‘", r"'"),
+        ("‚", r"'"),
+        ("’", r"'"),
+        (r"''", r'"'),
+        ("´´", r'"'),
+        ("…", r"..."),
+    ]
+
+    FRENCH_QUOTES = [  # lines 52 - 57
+        ("\u00A0«\u00A0", r'"'),
+        ("«\u00A0", r'"'),
+        ("«", r'"'),
+        ("\u00A0»\u00A0", r'"'),
+        ("\u00A0»", r'"'),
+        ("»", r'"'),
+    ]
+
+    HANDLE_PSEUDO_SPACES = [  # lines 59 - 67
+        ("\u00A0%", r"%"),
+        ("nº\u00A0", "nº "),
+        ("\u00A0:", r":"),
+        ("\u00A0ºC", " ºC"),
+        ("\u00A0cm", r" cm"),
+        ("\u00A0\\?", "?"),
+        ("\u00A0\\!", "!"),
+        ("\u00A0;", r";"),
+        (",\u00A0", r", "),
+        (r" +", r" "),
+    ]
+
+    EN_QUOTATION_FOLLOWED_BY_COMMA = [(r'"([,.]+)', r'\g<1>"')]
+
+    DE_ES_FR_QUOTATION_FOLLOWED_BY_COMMA = [
+        (r',"', r'",'),
+        (r'(\.+)"(\s*[^<])', r'"\g<1>\g<2>'),  # don't fix period at end of sentence
+    ]
+
+    DE_ES_CZ_CS_FR = [
+        ("(\\d)\u00A0(\\d)", r"\g<1>,\g<2>"),
+    ]
+
+    OTHER = [
+        ("(\\d)\u00A0(\\d)", r"\g<1>.\g<2>"),
+    ]
+
+    # Regex substitutions from replace-unicode-punctuation.perl
+    # https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/replace-unicode-punctuation.perl
+    REPLACE_UNICODE_PUNCTUATION = [
+        ("，", ","),
+        (r"。\s*", ". "),
+        ("、", ","),
+        ("”", '"'),
+        ("“", '"'),
+        ("∶", ":"),
+        ("：", ":"),
+        ("？", "?"),
+        ("《", '"'),
+        ("》", '"'),
+        ("）", ")"),
+        ("！", "!"),
+        ("（", "("),
+        ("；", ";"),
+        ("」", '"'),
+        ("「", '"'),
+        ("０", "0"),
+        ("１", "1"),
+        ("２", "2"),
+        ("３", "3"),
+        ("４", "4"),
+        ("５", "5"),
+        ("６", "6"),
+        ("７", "7"),
+        ("８", "8"),
+        ("９", "9"),
+        (r"．\s*", ". "),
+        ("～", "~"),
+        ("’", "'"),
+        ("…", "..."),
+        ("━", "-"),
+        ("〈", "<"),
+        ("〉", ">"),
+        ("【", "["),
+        ("】", "]"),
+        ("％", "%"),
+    ]
+
+    def __init__(
+        self,
+        lang="en",
+        penn=True,
+        norm_quote_commas=True,
+        norm_numbers=True,
+        pre_replace_unicode_punct=False,
+        post_remove_control_chars=False,
+    ):
+        """
+        :param language: The two-letter language code.
+        :type lang: str
+        :param penn: Normalize Penn Treebank style quotations.
+        :type penn: bool
+        :param norm_quote_commas: Normalize quotations and commas
+        :type norm_quote_commas: bool
+        :param norm_numbers: Normalize numbers
+        :type norm_numbers: bool
+        """
+        self.substitutions = [
+            self.EXTRA_WHITESPACE,
+            self.NORMALIZE_UNICODE,
+            self.FRENCH_QUOTES,
+            self.HANDLE_PSEUDO_SPACES,
+        ]
+
+        if penn:  # Adds the penn substitutions after extra_whitespace regexes.
+            self.substitutions.insert(1, self.NORMALIZE_UNICODE_IF_NOT_PENN)
+
+        if norm_quote_commas:
+            if lang == "en":
+                self.substitutions.append(self.EN_QUOTATION_FOLLOWED_BY_COMMA)
+            elif lang in ["de", "es", "fr"]:
+                self.substitutions.append(self.DE_ES_FR_QUOTATION_FOLLOWED_BY_COMMA)
+
+        if norm_numbers:
+            if lang in ["de", "es", "cz", "cs", "fr"]:
+                self.substitutions.append(self.DE_ES_CZ_CS_FR)
+            else:
+                self.substitutions.append(self.OTHER)
+
+        self.substitutions = list(chain(*self.substitutions))
+
+        self.pre_replace_unicode_punct = pre_replace_unicode_punct
+        self.post_remove_control_chars = post_remove_control_chars
+
+    def normalize(self, text):
+        """
+        Returns a string with normalized punctuation.
+        """
+        # Optionally, replace unicode puncts BEFORE normalization.
+        if self.pre_replace_unicode_punct:
+            text = self.replace_unicode_punct(text)
+
+        # Actual normalization.
+        for regexp, substitution in self.substitutions:
+            # print(regexp, substitution)
+            text = re.sub(regexp, substitution, str(text))
+            # print(text)
+
+        # Optionally, replace unicode puncts BEFORE normalization.
+        if self.post_remove_control_chars:
+            text = self.remove_control_chars(text)
+
+        return text.strip()
+
+    def replace_unicode_punct(self, text):
+        for regexp, substitution in self.REPLACE_UNICODE_PUNCTUATION:
+            text = re.sub(regexp, substitution, str(text))
+        return text
+
+    def remove_control_chars(self, text):
+        return regex.sub(r"\p{C}", "", text)
+
+def _tokenization_norm(text):
+    text = text.replace(
+        ' ,', ',').replace(
+        ' .', '.').replace(
+        ' ?', '?').replace(
+        ' !', '!').replace(
+        ' ;', ';').replace(
+        ' \'', '\'').replace(
+        ' ’ ', '\'').replace(
+        ' :', ':').replace(
+        '<newline>', '\n').replace(
+        '`` ', '"').replace(
+        ' \'\'', '"').replace(
+        '\'\'', '"').replace(
+        '.. ', '... ').replace(
+        ' )', ')').replace(
+        '( ', '(').replace(
+        ' n\'t', 'n\'t').replace(
+        ' i ', ' I ').replace(
+        ' i\'', ' I\'').replace(
+        '\\\'', '\'').replace(
+        '\n ', '\n').strip()
+    return text
+
+
+def _clean_text(text):
+    # remove PLM special tokens
+    plm_special_tokens = r'(\<pad\>)|(\<s\>)|(\<\/s\>)|(\<unk\>)|(\<\|endoftext\|\>)'
+    text = re.sub(plm_special_tokens, "", text)
+
+    # normalize puncuations
+    moses_norm = MosesPunctNormalizer()
+    text = moses_norm.normalize(text)
+    
+    # normalize tokenization
+    text = _tokenization_norm(text)
+    
+    # remove specific text patterns, e.g,, url, email and phone number
+    text = clean(text,
+        fix_unicode=True,               # fix various unicode errors
+        to_ascii=True,                  # transliterate to closest ASCII representation
+        lower=False,                     # lowercase text
+        no_line_breaks=True,           # fully strip line breaks as opposed to only normalizing them
+        no_urls=True,                  # replace all URLs with a special token
+        no_emails=True,                # replace all email addresses with a special token
+        no_phone_numbers=True,         # replace all phone numbers with a special token
+        no_numbers=False,               # replace all numbers with a special token
+        no_digits=False,                # replace all digits with a special token
+        no_currency_symbols=False,      # replace all currency symbols with a special token
+        no_punct=False,                 # remove punctuations
+        replace_with_punct="",          # instead of removing punctuations you may replace them
+        replace_with_url="",
+        replace_with_email="",
+        replace_with_phone_number="",
+        replace_with_number="<NUMBER>",
+        replace_with_digit="<DIGIT>",
+        replace_with_currency_symbol="<CUR>",
+        lang="en"                       # set to 'de' for German special handling
+    )
+    
+    # keep common puncts only
+    punct_pattern = r'[^ A-Za-z0-9.?!,:;\-\[\]\{\}\(\)\'\"]'
+    text = re.sub(punct_pattern, '', text)
+    # remove specific patterns
+    spe_pattern = r'[-\[\]\{\}\(\)\'\"]{2,}'
+    text = re.sub(spe_pattern, '', text)
+    # remove redundate spaces
+    text = " ".join(text.split())
+    return text
+
+def _rm_line_break(text):
+    text = text.replace("\n","\\n")
+    text = re.sub(r'(?:\\n)*\\n', r'\\n', text)
+    text = re.sub(r'^.{0,3}\\n', '', text)
+    text = text.replace("\\n"," ")
+    return text
+
+def preprocess(text):
+    text = _rm_line_break(text)
+    text = _clean_text(text)
+    return text
+
+
+def detect(input_text, tokenizer, model, device='cuda', th=-3.08583984375):
+    # Tokenize input text
+    tokenize_input = tokenizer(input_text, padding=True, truncation=True, max_length=512, return_tensors="pt")
+    tensor_input = torch.tensor(tokenize_input["input_ids"]).to(device)
+    
+    # Get model output
+    outputs = model(tensor_input)
+    
+    # Calculate score (probability for AI-generated text)
+    score = -outputs.logits[0][0].item()  # Negative logit for AI-generated probability
+
+    return score
+
+def load_jsonl(file_path):
+    out = []
+    with open(file_path, mode='r', encoding='utf-8') as jsonl_file:
+        for line in jsonl_file:
+            item = json.loads(line)
+            out.append(item)
+    print(f"Loaded {len(out)} examples from {file_path}")
+    return out
+
+def dict2str(metrics):
+    out_str=''
+    for key in metrics.keys():
+        out_str+=f"{key}:{metrics[key]} "
+    return out_str
+
+def experiment(args):
+    # Initialize MAGE model
+    model_dir = "yaful/MAGE"
+    tokenizer = AutoTokenizer.from_pretrained(model_dir)
+    model = AutoModelForSequenceClassification.from_pretrained(model_dir).cuda()
+
+    logging.info(f"Test in {args.test_data_path}")
+    test_data = load_jsonl(args.test_data_path)
+
+    random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    np.random.seed(args.seed)
+    random.shuffle(test_data)
+
+    predictions = []
+    labels = []
+
+    for i, item in tqdm(enumerate(test_data), total=len(test_data)):
+        text = item["text"]
+        label = item["label"]
+        src = item["src"]
+        
+        # preprocess the text
+        text = preprocess(text)
+        
+        # MAGE detection
+        score = detect(text, tokenizer, model)
+
+        # Determine the label and append to predictions and labels
+        if 'human' in src:
+            labels.append(1)
+        else:
+            labels.append(0)
+
+        predictions.append(score)
+
+    # Compute metrics
+    metric = evaluate_metrics(labels, predictions)
+    print(dict2str(metric))
+
+    # Save results
+    with _LOG_PATH.open("a+", encoding="utf-8") as f:
+        f.write(f"MAGE  {args.test_data_path}\n")
+        f.write(f"{dict2str(metric)}\n")
+
+def build_argument_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--test_data_path',
+        type=str,
+        default='/path/to/RealBench/DetectRL/Multi_Attack/all_attacks_llm_test.jsonl',
+        help="Path to the test data. could be several files with ','. Note that the data should have been perturbed.",
+    )
+    parser.add_argument('--seed', default=2023, type=int, required=False)
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> None:
+    parser = build_argument_parser()
+    args = parser.parse_args(argv)
+    experiment(args)
+
+
+if __name__ == '__main__':
+    main()

detree/utils/detectors/RADAR.py

@@ -0,0 +1,100 @@
+import argparse
+import json
+import logging
+import random
+from pathlib import Path
+from typing import Sequence
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+import transformers
+from tqdm import tqdm
+
+from ..utils import evaluate_metrics
+
+_LOG_PATH = Path(__file__).resolve().parents[3] / "runs" / "val-other_detector.txt"
+_LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
+
+def load_jsonl(file_path):
+    out = []
+    with open(file_path, mode='r', encoding='utf-8') as jsonl_file:
+        for line in jsonl_file:
+            item = json.loads(line)
+            out.append(item)
+    print(f"Loaded {len(out)} examples from {file_path}")
+    return out
+
+def dict2str(metrics):
+    out_str=''
+    for key in metrics.keys():
+        out_str+=f"{key}:{metrics[key]} "
+    return out_str
+
+def experiment(args):
+    # Initialize RADAR detector model
+    detector = transformers.AutoModelForSequenceClassification.from_pretrained("TrustSafeAI/RADAR-Vicuna-7B",device_map="auto")
+    tokenizer = transformers.AutoTokenizer.from_pretrained("TrustSafeAI/RADAR-Vicuna-7B")
+    detector.eval()
+
+    logging.info(f"Test in {args.test_data_path}")
+    test_data = load_jsonl(args.test_data_path)
+
+    random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    np.random.seed(args.seed)
+    random.shuffle(test_data)
+
+    predictions = []
+    labels = []
+    
+    for i, item in tqdm(enumerate(test_data), total=len(test_data)):
+        text = item["text"]
+        label = item["label"]
+        src = item["src"]
+        
+        # Tokenize input text
+        inputs = tokenizer(text, padding=True, truncation=True, max_length=512, return_tensors="pt")
+        inputs = {k: v.cuda() for k, v in inputs.items()}
+        
+        # Get model output
+        with torch.no_grad():
+            output_probs = F.log_softmax(detector(**inputs).logits, -1)[:, 0].exp().tolist()
+
+        # Determine the label and append to predictions and labels
+        if 'human' in src:
+            labels.append(0)
+        else:
+            labels.append(1)
+
+        predictions.append(output_probs[0])  # Probabilities for AI-generated text
+    
+    # Compute metrics
+    metric = evaluate_metrics(labels, predictions)
+    print(dict2str(metric))
+    
+    # Save results
+    with _LOG_PATH.open("a+", encoding="utf-8") as f:
+        f.write(f"RADAR  {args.test_data_path}\n")
+        f.write(f"{dict2str(metric)}\n")
+
+def build_argument_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--test_data_path',
+        type=str,
+        default='/path/to/RealBench/DetectRL/Multi_Domain/all_multi_domains_test.jsonl',
+        help="Path to the test data. could be several files with ','. Note that the data should have been perturbed.",
+    )
+    parser.add_argument('--seed', default=2023, type=int, required=False)
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> None:
+    parser = build_argument_parser()
+    args = parser.parse_args(argv)
+    experiment(args)
+
+
+if __name__ == '__main__':
+    main()

detree/utils/detectors/UAR_eval.py

@@ -0,0 +1,126 @@
+import argparse
+import json
+import os
+from pathlib import Path
+from typing import Sequence
+
+import numpy as np; np.random.seed(43)
+import torch
+import torch.nn.functional as F
+from torch.utils.data import DataLoader, Dataset
+from tqdm import tqdm
+from transformers import AutoModel, AutoTokenizer
+
+from ..utils import evaluate_metrics
+os.environ["TOKENIZERS_PARALLELISM"] = "true"
+
+_LOG_PATH = Path(__file__).resolve().parents[3] / "runs" / "val-other_detector.txt"
+_LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
+class PassagesDataset(Dataset):
+    def __init__(self, data):
+        
+        self.passages = data
+
+    def __len__(self):
+        return len(self.passages)
+         
+    def __getitem__(self, idx):
+        data_now = self.passages[idx]
+        text = data_now['text']
+        label = int(data_now['label'])==0
+        ids = data_now['id']
+        return text, int(label), int(ids)
+
+def load_jsonl(file_path,need_human=True):
+    out = []
+    with open(file_path, mode='r', encoding='utf-8') as jsonl_file:
+        for line in jsonl_file:
+            item = json.loads(line)
+            if item['src'] =='human' and need_human==False:
+                continue
+            out.append(item)
+    print(f"Loaded {len(out)} examples from {file_path}")
+    return out
+
+def dict2str(metrics):
+    out_str=''
+    for key in metrics.keys():
+        out_str+=f"{key}:{metrics[key]} "
+    return out_str
+
+def gen_embeddings(data, model, tokenizer):
+    device = torch.device("cuda")
+    dataset = PassagesDataset(data)
+    dataloder = DataLoader(dataset, batch_size=32, num_workers=8, shuffle=False)
+    labels, embeddings = [], []
+    with torch.no_grad():
+        for batch in tqdm(dataloder,total=len(dataloder)):
+            texts,label,ids= batch
+            encoded_batch = tokenizer.batch_encode_plus(
+                                texts,
+                                return_tensors="pt",
+                                max_length=512,
+                                padding='max_length',
+                                truncation=True,
+                            )
+            for key in encoded_batch:
+                encoded_batch[key] = encoded_batch[key].unsqueeze(1).to(device)
+
+
+            now_embeddings = model(**encoded_batch)
+            now_embeddings = F.normalize(now_embeddings, p=2, dim=-1)
+            embeddings.append(now_embeddings.cpu())
+            labels.append(label.cpu())
+        labels = torch.cat(labels, dim=0).numpy()
+        embeddings = torch.cat(embeddings, dim=0).numpy()
+
+    return embeddings, labels
+
+def run(opt):
+    device = torch.device("cuda")
+    model = AutoModel.from_pretrained("rrivera1849/LUAR-CRUD", trust_remote_code=True)
+    model.to(device)
+    model.eval()
+    tokenizer = AutoTokenizer.from_pretrained("rrivera1849/LUAR-CRUD")
+    database_data = load_jsonl(opt.database_path,need_human=False)
+    test_data = load_jsonl(opt.test_dataset_path)
+    print("Database Data Size:", len(database_data), "Test Data Size:", len(test_data))
+    database_embeddings, database_labels = gen_embeddings(database_data, model, tokenizer)
+    test_embeddings, test_labels = gen_embeddings(test_data, model, tokenizer)
+    dis = test_embeddings @ database_embeddings.T
+    dis = dis.min(axis=1)
+    metric = evaluate_metrics(test_labels, dis)
+    print(dict2str(metric))
+    with _LOG_PATH.open("a+", encoding="utf-8") as f:
+        f.write(f"UAR  {opt.test_dataset_path}\n")
+        f.write(f"{dict2str(metric)}\n")
+
+def build_argument_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--database_path", type=str, default="/path/to/RealBench/MAGE_Unseen/Unseen/5shot/train_0.jsonl")
+    parser.add_argument("--test_dataset_path", type=str, default="/path/to/RealBench/MAGE_Unseen/Unseen/5shot/test_0.jsonl")
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> None:
+    parser = build_argument_parser()
+    opt = parser.parse_args(argv)
+    run(opt)
+
+
+if __name__ == "__main__":
+    main()
+# text = ['The quick brown fox jumps over the lazy dog.','There is a cat on the roof.']
+# encoded_batch = tokenizer.batch_encode_plus(
+#                         text,
+#                         return_tensors="pt",
+#                         max_length=512,
+#                         padding='max_length',
+#                         truncation=True,
+#                     )
+# for key in encoded_batch:
+#     encoded_batch[key] = encoded_batch[key].unsqueeze(1).to(device)
+
+# with torch.no_grad():
+#     embeddings = model(**encoded_batch)
+#     print(embeddings.shape)

detree/utils/detectors/binoculars_detector.py

@@ -0,0 +1,166 @@
+from typing import Union
+import os
+import numpy as np
+import torch
+import transformers
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
+import transformers
+
+ce_loss_fn = torch.nn.CrossEntropyLoss(reduction="none")
+softmax_fn = torch.nn.Softmax(dim=-1)
+
+
+torch.set_grad_enabled(False)
+
+huggingface_config = {
+    # Only required for private models from Huggingface (e.g. LLaMA models)
+    "TOKEN": os.environ.get("HF_TOKEN", None)
+}
+
+# selected using Falcon-7B and Falcon-7B-Instruct at bfloat16
+BINOCULARS_ACCURACY_THRESHOLD = 0.9015310749276843  # optimized for f1-score
+BINOCULARS_FPR_THRESHOLD = 0.8536432310785527  # optimized for low-fpr [chosen at 0.01%]
+
+DEVICE_1 = "cuda:0" if torch.cuda.is_available() else "cpu"
+DEVICE_2 = "cuda:1" if torch.cuda.device_count() > 1 else DEVICE_1
+
+
+def assert_tokenizer_consistency(model_id_1, model_id_2):
+    identical_tokenizers = (
+            AutoTokenizer.from_pretrained(model_id_1).vocab
+            == AutoTokenizer.from_pretrained(model_id_2).vocab
+    )
+    if not identical_tokenizers:
+        raise ValueError(f"Tokenizers are not identical for {model_id_1} and {model_id_2}.")
+
+
+def perplexity(encoding: transformers.BatchEncoding,
+               logits: torch.Tensor,
+               median: bool = False,
+               temperature: float = 1.0):
+    shifted_logits = logits[..., :-1, :].contiguous() / temperature
+    shifted_labels = encoding.input_ids[..., 1:].contiguous()
+    shifted_attention_mask = encoding.attention_mask[..., 1:].contiguous()
+
+    if median:
+        ce_nan = (ce_loss_fn(shifted_logits.transpose(1, 2), shifted_labels).
+                  masked_fill(~shifted_attention_mask.bool(), float("nan")))
+        ppl = np.nanmedian(ce_nan.cpu().float().numpy(), 1)
+
+    else:
+        ppl = (ce_loss_fn(shifted_logits.transpose(1, 2), shifted_labels) *
+               shifted_attention_mask).sum(1) / shifted_attention_mask.sum(1)
+        ppl = ppl.to("cpu").float().numpy()
+
+    return ppl
+
+
+def entropy(p_logits: torch.Tensor,
+            q_logits: torch.Tensor,
+            encoding: transformers.BatchEncoding,
+            pad_token_id: int,
+            median: bool = False,
+            sample_p: bool = False,
+            temperature: float = 1.0):
+    vocab_size = p_logits.shape[-1]
+    total_tokens_available = q_logits.shape[-2]
+    p_scores, q_scores = p_logits / temperature, q_logits / temperature
+
+    p_proba = softmax_fn(p_scores).view(-1, vocab_size)
+
+    if sample_p:
+        p_proba = torch.multinomial(p_proba.view(-1, vocab_size), replacement=True, num_samples=1).view(-1)
+
+    q_scores = q_scores.view(-1, vocab_size)
+
+    ce = ce_loss_fn(input=q_scores, target=p_proba).view(-1, total_tokens_available)
+    padding_mask = (encoding.input_ids != pad_token_id).type(torch.uint8)
+
+    if median:
+        ce_nan = ce.masked_fill(~padding_mask.bool(), float("nan"))
+        agg_ce = np.nanmedian(ce_nan.cpu().float().numpy(), 1)
+    else:
+        agg_ce = (((ce * padding_mask).sum(1) / padding_mask.sum(1)).to("cpu").float().numpy())
+
+    return agg_ce
+
+
+class Binoculars(object):
+    def __init__(self,
+                 observer_name_or_path: str = "tiiuae/falcon-7b",
+                 performer_name_or_path: str = "tiiuae/falcon-7b-instruct",
+                 use_bfloat16: bool = True,
+                 max_token_observed: int = 512,
+                 mode: str = "low-fpr",
+                 ) -> None:
+        assert_tokenizer_consistency(observer_name_or_path, performer_name_or_path)
+
+        self.change_mode(mode)
+        self.observer_model = AutoModelForCausalLM.from_pretrained(observer_name_or_path,
+                                                                   device_map={"": DEVICE_1},
+                                                                   trust_remote_code=True,
+                                                                   torch_dtype=torch.bfloat16 if use_bfloat16
+                                                                   else torch.float32,
+                                                                   token=huggingface_config["TOKEN"]
+                                                                   )
+        self.performer_model = AutoModelForCausalLM.from_pretrained(performer_name_or_path,
+                                                                    device_map={"": DEVICE_2},
+                                                                    trust_remote_code=True,
+                                                                    torch_dtype=torch.bfloat16 if use_bfloat16
+                                                                    else torch.float32,
+                                                                    token=huggingface_config["TOKEN"]
+                                                                    )
+        self.observer_model.eval()
+        self.performer_model.eval()
+
+        self.tokenizer = AutoTokenizer.from_pretrained(observer_name_or_path)
+        if not self.tokenizer.pad_token:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+        self.max_token_observed = max_token_observed
+
+    def change_mode(self, mode: str) -> None:
+        if mode == "low-fpr":
+            self.threshold = BINOCULARS_FPR_THRESHOLD
+        elif mode == "accuracy":
+            self.threshold = BINOCULARS_ACCURACY_THRESHOLD
+        else:
+            raise ValueError(f"Invalid mode: {mode}")
+
+    def _tokenize(self, batch: list[str]) -> transformers.BatchEncoding:
+        batch_size = len(batch)
+        encodings = self.tokenizer(
+            batch,
+            return_tensors="pt",
+            padding="longest" if batch_size > 1 else False,
+            truncation=True,
+            max_length=self.max_token_observed,
+            return_token_type_ids=False).to(self.observer_model.device)
+        return encodings
+
+    @torch.inference_mode()
+    def _get_logits(self, encodings: transformers.BatchEncoding) -> torch.Tensor:
+        observer_logits = self.observer_model(**encodings.to(DEVICE_1)).logits
+        performer_logits = self.performer_model(**encodings.to(DEVICE_2)).logits
+        if DEVICE_1 != "cpu":
+            torch.cuda.synchronize()
+        return observer_logits, performer_logits
+
+    def compute_score(self, input_text: Union[list[str], str]) -> Union[float, list[float]]:
+        batch = [input_text] if isinstance(input_text, str) else input_text
+        encodings = self._tokenize(batch)
+        observer_logits, performer_logits = self._get_logits(encodings)
+        ppl = perplexity(encodings, performer_logits)
+        x_ppl = entropy(observer_logits.to(DEVICE_1), performer_logits.to(DEVICE_1),
+                        encodings.to(DEVICE_1), self.tokenizer.pad_token_id)
+        binoculars_scores = ppl / x_ppl
+        binoculars_scores = binoculars_scores.tolist()
+        return binoculars_scores[0] if isinstance(input_text, str) else binoculars_scores
+
+    def predict(self, input_text: Union[list[str], str]) -> Union[list[str], str]:
+        binoculars_scores = np.array(self.compute_score(input_text))
+        pred = np.where(binoculars_scores < self.threshold,
+                        "Most likely AI-generated",
+                        "Most likely human-generated"
+                        ).tolist()
+        return pred

detree/utils/detectors/binoculars_evaluation.py

@@ -0,0 +1,89 @@
+import argparse
+import json
+import logging
+import random
+from pathlib import Path
+from typing import Sequence
+
+import numpy as np
+import torch
+from tqdm import tqdm
+
+from .binoculars_detector import Binoculars
+from ..utils import evaluate_metrics
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+
+_LOG_PATH = Path(__file__).resolve().parents[3] / "runs" / "val-other_detector.txt"
+_LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
+
+def load_jsonl(file_path):
+    out = []
+    with open(file_path, mode='r', encoding='utf-8') as jsonl_file:
+        for line in jsonl_file:
+            item = json.loads(line)
+            out.append(item)
+    print(f"Loaded {len(out)} examples from {file_path}")
+    return out
+
+def dict2str(metrics):
+    out_str=''
+    for key in metrics.keys():
+        out_str+=f"{key}:{metrics[key]} "
+    return out_str
+
+def experiment(args):
+    # Initialize Binoculars (experiments in paper use the "accuracy" mode threshold wherever applicable)
+    bino = Binoculars(mode="accuracy", max_token_observed=args.tokens_seen)
+
+    logging.info(f"Test in {args.test_data_path}")
+    test_data = load_jsonl(args.test_data_path)
+
+    random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    np.random.seed(args.seed)
+    random.shuffle(test_data)
+    predictions = []
+    labels = []
+    for i, item in tqdm(enumerate(test_data), total=len(test_data)):
+        text = item["text"]
+        label = item["label"]
+        src = item["src"]
+        bino_score = -bino.compute_score(text)
+        
+        if bino_score is None or np.isnan(bino_score) or np.isinf(bino_score):
+            bino_score = 0
+        if 'human' in src:
+            labels.append(0)
+        else:
+            labels.append(1)
+        predictions.append(bino_score)
+    metric = evaluate_metrics(labels, predictions)
+    print(dict2str(metric))
+    with _LOG_PATH.open("a+", encoding="utf-8") as f:
+        f.write(f"binoculars  {args.test_data_path}\n")
+        f.write(f"{dict2str(metric)}\n")
+
+
+def build_argument_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--test_data_path',
+        type=str,
+        default='/path/to/RealBench/Deepfake/no_attack/test.jsonl',
+        help="Path to the test data. could be several files with ','. Note that the data should have been perturbed.",
+    )
+    parser.add_argument("--tokens_seen", type=int, default=512, help="Number of tokens seen by the model")
+    parser.add_argument('--DEVICE', default="cuda", type=str, required=False)
+    parser.add_argument('--seed', default=2023, type=int, required=False)
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> None:
+    parser = build_argument_parser()
+    args = parser.parse_args(argv)
+    experiment(args)
+
+
+if __name__ == '__main__':
+    main()