Upload 9 files

README.md

@@ -1,8 +1,8 @@
 ---
-title: MMD MP Text Dection
-emoji: 🌖
-colorFrom: purple
-colorTo: indigo
+title: MMD-MP Text Dection
+emoji: 💻
+colorFrom: blue
+colorTo: gray
 sdk: gradio
 sdk_version: 3.45.1
 app_file: app.py

app.py

@@ -0,0 +1,55 @@
+import torch
+from utils_MMD import MMD_batch2
+from api_init import api_init
+import gradio as gr
+
+base_model, base_tokenizer, net, feature_ref, sigma, sigma0_u, ep, loaded_model, DEVICE = api_init()
+
+def single_instance_detection(sentence=None):
+
+	with torch.no_grad():
+		if sentence is None:
+			sentence = "Now that you've built a demo, you'll probably want to share it with others. Gradio demos can be shared in two ways: using a temporary share link or permanent hosting on Spaces."
+		sentence = sentence.strip()
+		inputs = base_tokenizer([sentence], padding='max_length', truncation=True, max_length=100,return_tensors="pt").to(DEVICE)
+		outputs = base_model(**inputs)
+		hidden_states_all = outputs[2]
+		hidden_states = hidden_states_all[-1]
+		token_mask_10 = inputs['attention_mask'].unsqueeze(-1)
+		hidden_states_mask_10 = hidden_states * token_mask_10
+
+		feature_for_one_sample = net(hidden_states_mask_10)
+		mmd_feature_for_one_sample = MMD_batch2(torch.cat([feature_ref,feature_for_one_sample],dim=0), feature_ref.shape[0], 0, sigma, sigma0_u, ep,is_smooth=False).to('cpu')
+
+		y_pred_loaded = loaded_model.predict(mmd_feature_for_one_sample.detach().numpy().reshape(-1, 1))
+		print("y_pred_loaded:", y_pred_loaded)
+
+		prediction = int(y_pred_loaded[0])
+		if prediction == 0:
+			return "Human"
+		elif prediction == 1:
+			return "AI"
+
+
+description = "Please input a sentence in English to classify whether it's generated by a human or AI.\n\nFor example: the texts below the box are generated by ChatGPT with a prompt: Write an essay in English titled by 'A Walk in the Woods'. You can directly click on the box or copy some texts into the input field for testing."
+
+AI_examples = [
+    ["The gentle rustling of leaves, the symphony of birdsong, and the dappled sunlight filtering through the towering trees create a serene tapestry of nature's beauty. A walk in the woods is a timeless and profound experience, one that invites us to disconnect from the hustle and bustle of modern life and reconnect with the natural world. It is a journey that nourishes the soul, rejuvenates the spirit, and reminds us of our deep-rooted connection to the earth."],
+    ["The woods have always held a special place in human history and imagination. They are places of mystery and enchantment, where ancient trees bear witness to centuries of change and growth. The woods are where stories of knights and dragons, fairies and gnomes, have been woven into the fabric of our culture. They are places where poets have found inspiration, artists have discovered their muse, and philosophers have contemplated the mysteries of existence."],
+    ["A walk in the woods offers a unique opportunity to escape the relentless pace of modern life. As we step onto the forest path, the cares and worries of the world seem to melt away. The woods have a way of grounding us, reminding us of our place in the grand scheme of things. In the presence of towering trees that have weathered countless storms and witnessed the passage of time, our own problems and concerns can feel insignificant, allowing us to gain a broader perspective on life."],
+    ["The sensory delights of the woods are unparalleled. The air is rich with the earthy scent of leaves and moss, and the soft, mossy ground beneath our feet cradles each step. The chorus of birdsong provides a soothing soundtrack, while the play of light and shadow through the foliage creates a mesmerizing dance of colors and patterns."],
+    ["But perhaps the most profound aspect of a walk in the woods is the opportunity for introspection and self-discovery. In the stillness of the forest, away from the distractions of screens and schedules, we have a chance to listen to the whispers of our own thoughts. It is a time for reflection, for sorting through the jumble of ideas and emotions that often crowd our minds."]
+]
+
+# Create the Gradio interface
+iface = gr.Interface(
+	fn=single_instance_detection,
+	inputs="text",
+	outputs="text",
+	title="GPT-Inspector: Text generated by Human or AI?",
+	description=description,
+	examples=AI_examples,
+)
+
+# Launch the Gradio interface
+iface.launch()

dataTST.py

@@ -0,0 +1,63 @@
+import numpy as np
+import torch
+import random
+from meta_train import mmdPreModel
+from collections import namedtuple
+import joblib
+from transformers import RobertaTokenizer, RobertaModel
+
+
+def api_init():
+
+	random.seed(0)
+	np.random.seed(0)
+	torch.manual_seed(0)
+	torch.cuda.manual_seed(0)
+	torch.cuda.manual_seed_all(0)
+	torch.backends.cudnn.benchmark = False
+	torch.backends.cudnn.deterministic = True
+
+	model_name = 'roberta-base-openai-detector'
+	model_path_api = f'.'
+	token_num, hidden_size = 100, 768
+
+	Config = namedtuple('Config', ['in_dim', 'hid_dim', 'dropout', 'out_dim', 'token_num'])
+	config = Config(
+			in_dim=hidden_size,
+			token_num=token_num,
+			hid_dim=512,
+			dropout=0.2,
+			out_dim=300,)
+	
+	net = mmdPreModel(config=config, num_mlp=0, transformer_flag=True, num_hidden_layers=1)
+
+	# load the features and models
+	feature_ref_for_test_filename = f'{model_path_api}/feature_ref_for_test.pt'
+	model_filename = f'{model_path_api}/logistic_regression_model.pkl'
+	net_filename = f'{model_path_api}/net.pt'
+	
+	load_ref_data =  torch.load(feature_ref_for_test_filename,map_location=torch.device('cpu')) # cpu
+	loaded_model = joblib.load(model_filename) # cpu
+	checkpoint = torch.load(net_filename,map_location=torch.device('cpu'))
+	net.load_state_dict(checkpoint['net'])
+	sigma, sigma0_u, ep  = checkpoint['sigma'], checkpoint['sigma0_u'], checkpoint['ep']
+
+	# generic generative model
+	cache_dir = ".cache"
+	base_tokenizer = RobertaTokenizer.from_pretrained(model_name, cache_dir=cache_dir)
+	base_model = RobertaModel.from_pretrained(model_name, output_hidden_states=True, cache_dir=cache_dir)
+
+	# whether load the model to gpu
+	gpu_using = False
+
+	DEVICE = torch.device("cpu")
+	if gpu_using:
+		DEVICE = torch.device("cuda:0")
+	net = net.to(DEVICE)
+	sigma, sigma0_u, ep  = sigma.to(DEVICE), sigma0_u.to(DEVICE), ep.to(DEVICE)
+	load_ref_data = load_ref_data.to(DEVICE)
+	base_model = base_model.to(DEVICE)	
+	num_ref = 5000
+	feature_ref = load_ref_data[np.random.permutation(load_ref_data.shape[0])][:num_ref].to(DEVICE)
+
+	return base_model, base_tokenizer, net, feature_ref, sigma, sigma0_u, ep, loaded_model, DEVICE

feature_ref_for_test.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f690f4244c0b245823f847a75d1fd25a151ad96075744156e628c0c1dda65302
+size 42240786

logistic_regression_model.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce9afa92d745bff3bbfb4ae028db369cb35710a9099159a12ac1048fed3760e6
+size 756

meta_train.py

@@ -0,0 +1,131 @@
+import  torch
+from  torch import nn
+import math
+
+
+from pytorch_transformers.modeling_bert import(
+	BertEncoder,
+	BertPreTrainedModel,
+	BertConfig
+)
+
+class GeLU(nn.Module):
+	"""Implementation of the gelu activation function.
+		For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
+		0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
+		Also see https://arxiv.org/abs/1606.08415
+	"""
+
+	def __init__(self):
+		super().__init__()
+
+	def forward(self, x):
+		return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
+
+class BertLayerNorm(nn.Module):
+	def __init__(self, hidden_size, eps=1e-12):
+		"""Construct a layernorm module in the TF style (epsilon inside the square root).
+		"""
+		super(BertLayerNorm, self).__init__()
+		self.weight = nn.Parameter(torch.ones(hidden_size))
+		self.bias = nn.Parameter(torch.zeros(hidden_size))
+		self.variance_epsilon = eps
+
+	def forward(self, x):
+		u = x.mean(-1, keepdim=True)
+		s = (x - u).pow(2).mean(-1, keepdim=True)
+		x = (x - u) / torch.sqrt(s + self.variance_epsilon)
+		return self.weight * x + self.bias
+
+class mlp_meta(nn.Module):
+	def __init__(self, config):
+		super().__init__()
+		self.mlp = nn.Sequential(
+			nn.Linear(config.hid_dim, config.hid_dim),
+			GeLU(),
+			BertLayerNorm(config.hid_dim, eps=1e-12),
+			nn.Dropout(config.dropout),
+		)
+
+	def forward(self, x):
+		return self.mlp(x)
+	
+class Bert_Transformer_Layer(BertPreTrainedModel):
+	def __init__(self,fusion_config):
+		super().__init__(BertConfig(**fusion_config))
+		bertconfig_fusion = BertConfig(**fusion_config)
+		self.encoder = BertEncoder(bertconfig_fusion)
+		self.init_weights()
+	
+	def forward(self,input, mask=None):
+		"""
+		input:(bs, 4, dim)
+		"""
+		batch, feats, dim = input.size()
+		if mask is not None:
+			mask_ = torch.ones(size=(batch,feats), device=mask.device)
+			mask_[:,1:] = mask
+			mask_ = torch.bmm(mask_.view(batch,1,-1).transpose(1,2), mask_.view(batch,1,-1))
+			mask_ = mask_.unsqueeze(1)
+		
+		else:
+			mask = torch.Tensor([1.0]).to(input.device)
+			mask_ = mask.repeat(batch,1,feats, feats)
+		
+		extend_mask = (1- mask_) * -10000
+		assert not extend_mask.requires_grad
+		head_mask = [None] * self.config.num_hidden_layers
+
+		enc_output = self.encoder(
+			input,extend_mask,head_mask=head_mask
+		)
+		output = enc_output[0]
+		all_attention = enc_output[1]
+
+		return output,all_attention
+	
+class mmdPreModel(nn.Module):
+	def __init__(self, config, num_mlp=0, transformer_flag=False, num_hidden_layers=1, mlp_flag=True):
+		super(mmdPreModel, self).__init__()
+		self.num_mlp = num_mlp
+		self.transformer_flag = transformer_flag
+		self.mlp_flag = mlp_flag
+		token_num = config.token_num
+		self.mlp = nn.Sequential(
+			nn.Linear(config.in_dim, config.hid_dim),
+			GeLU(),
+			BertLayerNorm(config.hid_dim, eps=1e-12),
+			nn.Dropout(config.dropout),
+			# nn.Linear(config.hid_dim, config.out_dim),
+		)
+		self.fusion_config = {
+			'hidden_size': config.in_dim,
+			'num_hidden_layers':num_hidden_layers,
+			'num_attention_heads':4,
+			'output_attentions':True
+			}
+		if self.num_mlp>0:
+			self.mlp2 = nn.ModuleList([mlp_meta(config) for _ in range(self.num_mlp)])
+		if self.transformer_flag:
+			self.transformer = Bert_Transformer_Layer(self.fusion_config)
+		self.feature = nn.Linear(config.hid_dim * token_num, config.out_dim)
+
+	def forward(self, features):
+		"""
+		input: [batch, token_num, hidden_size], output: [batch, token_num * config.out_dim]
+		"""
+
+		if self.transformer_flag:
+			features,_ = self.transformer(features)
+		if self.mlp_flag:
+			features = self.mlp(features)
+		
+		if self.num_mlp>0:
+			# features = self.mlp2(features)
+			for _ in range(1):
+				for mlp in self.mlp2:
+					features = mlp(features)
+
+		features = self.feature(features.view(features.shape[0], -1))
+		return features #features.view(features.shape[0], -1)
+

net.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51eddf445c758f5d42e8f4783b1e8ecc5833db5a9bc170203478d73a761ddd6a
+size 91379438

requirements.txt

@@ -0,0 +1,6 @@
+torch
+transformers
+numpy
+joblib
+pytorch_transformers
+scikit-learn

utils_MMD.py

@@ -0,0 +1,58 @@
+import torch
+
+def Pdist2(x, y):
+	"""compute the paired distance between x and y."""
+	x_norm = (x ** 2).sum(1).view(-1, 1)
+	if y is not None:
+		y_norm = (y ** 2).sum(1).view(1, -1)
+	else:
+		y = x
+		y_norm = x_norm.view(1, -1)
+	Pdist = x_norm + y_norm - 2.0 * torch.mm(x, torch.transpose(y, 0, 1))
+	Pdist[Pdist<0]=0
+	return Pdist
+
+def MMD_batch2(Fea, len_s, Fea_org, sigma, sigma0=0.1, epsilon = 10**(-10), is_smooth=True, is_var_computed=True, use_1sample_U=True, coeff_xy=2):
+	X = Fea[0:len_s, :] 
+	Y = Fea[len_s:, :] 
+	if is_smooth:
+		X_org = Fea_org[0:len_s, :] 
+		Y_org = Fea_org[len_s:, :]  
+	L = 1 # generalized Gaussian (if L>1)
+
+	nx = X.shape[0] 
+	ny = Y.shape[0] 
+	Dxx = Pdist2(X, X) 
+	Dyy = torch.zeros(Fea.shape[0] - len_s, 1).to(Dxx.device) 
+	# Dyy = Pdist2(Y, Y) 
+	Dxy = Pdist2(X, Y).transpose(0,1) 
+	if is_smooth:
+		Dxx_org = Pdist2(X_org, X_org)
+		Dyy_org = torch.zeros(Fea.shape[0] - len_s, 1).to(Dxx.device)
+		# Dyy_org = Pdist2(Y_org, Y_org) # 1，1  0
+		Dxy_org = Pdist2(X_org, Y_org).transpose(0,1)  
+
+
+	if is_smooth:
+		Kx = (1-epsilon) * torch.exp(-(Dxx / sigma0)**L -Dxx_org / sigma) + epsilon * torch.exp(-Dxx_org / sigma) 
+		Ky = (1-epsilon) * torch.exp(-(Dyy / sigma0)**L -Dyy_org / sigma) + epsilon * torch.exp(-Dyy_org / sigma)
+		Kxy = (1-epsilon) * torch.exp(-(Dxy / sigma0)**L -Dxy_org / sigma) + epsilon * torch.exp(-Dxy_org / sigma)
+	else:
+		Kx = torch.exp(-Dxx / sigma0)
+		Ky = torch.exp(-Dyy / sigma0)
+		Kxy = torch.exp(-Dxy / sigma0)
+
+	nx = Kx.shape[0] 
+
+	is_unbiased = False
+	if 1:
+		xx = torch.div((torch.sum(Kx)), (nx * nx)) 
+		yy = Ky.reshape(-1) 
+
+		# one-sample U-statistic.
+
+		xy = torch.div(torch.sum(Kxy, dim = 1), (nx )) 
+		
+		mmd2 = xx - 2 * xy + yy
+	return mmd2
+