some FakeVD

FakeVD/code_test/C3D_Feature_Extractor/C3D_model.py

@@ -0,0 +1,74 @@
+# coding: utf-8
+
+import torch.nn as nn
+
+	
+class C3D(nn.Module):
+	"""
+	nb_classes: nb_classes in classification task, 101 for UCF101 dataset
+	"""
+
+	def __init__(self, nb_classes):
+		super(C3D, self).__init__()
+
+		self.conv1 = nn.Conv3d(3, 64, kernel_size=(3, 3, 3), padding=(1, 1, 1))
+		self.pool1 = nn.MaxPool3d(kernel_size=(1, 2, 2), stride=(1, 2, 2))
+
+		self.conv2 = nn.Conv3d(64, 128, kernel_size=(3, 3, 3), padding=(1, 1, 1))
+		self.pool2 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2))
+
+		self.conv3a = nn.Conv3d(128, 256, kernel_size=(3, 3, 3), padding=(1, 1, 1))
+		self.conv3b = nn.Conv3d(256, 256, kernel_size=(3, 3, 3), padding=(1, 1, 1))
+		self.pool3 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2))
+
+		self.conv4a = nn.Conv3d(256, 512, kernel_size=(3, 3, 3), padding=(1, 1, 1))
+		self.conv4b = nn.Conv3d(512, 512, kernel_size=(3, 3, 3), padding=(1, 1, 1))
+		self.pool4 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2))
+
+		self.conv5a = nn.Conv3d(512, 512, kernel_size=(3, 3, 3), padding=(1, 1, 1))
+		self.conv5b = nn.Conv3d(512, 512, kernel_size=(3, 3, 3), padding=(1, 1, 1))
+		self.pool5 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2), padding=(0, 1, 1))
+
+		self.fc6 = nn.Linear(8192, 4096)
+		self.fc7 = nn.Linear(4096, 4096)
+		self.fc8 = nn.Linear(4096, nb_classes)
+
+		self.dropout = nn.Dropout(p=0.5)
+
+		self.relu = nn.ReLU()
+
+	def forward(self, x, feature_layer):
+		
+		h = self.relu(self.conv1(x))
+		h = self.pool1(h)
+		h = self.relu(self.conv2(h))
+		h = self.pool2(h)
+
+		h = self.relu(self.conv3a(h))
+		h = self.relu(self.conv3b(h))
+		h = self.pool3(h)
+
+		h = self.relu(self.conv4a(h))
+		h = self.relu(self.conv4b(h))
+		h = self.pool4(h)
+
+		h = self.relu(self.conv5a(h))
+		h = self.relu(self.conv5b(h))
+		h = self.pool5(h)
+
+		h = h.reshape(-1, 8192)
+		out = h if feature_layer == 5 else None
+		h = self.relu(self.fc6(h))
+		out = h if feature_layer == 6 and out == None else out
+		h = self.dropout(h)
+		h = self.relu(self.fc7(h))
+		out = h if feature_layer == 7 and out == None else out
+		h = self.dropout(h)
+		logits = self.fc8(h)
+		return logits, out
+
+
+
+
+
+

FakeVD/code_test/C3D_Feature_Extractor/feature_extractor_frm.py

@@ -0,0 +1,119 @@
+# coding: utf-8
+# from data_provider import *
+from C3D_model import *
+import json
+import torchvision
+import torch.optim as optim
+import torch
+from torch.autograd import Variable
+import torch.nn.functional as F
+import argparse
+import os 
+from torch import save, load
+import pickle
+import time
+import numpy as np
+import PIL.Image as Image
+import collections
+#import imageio # read video
+import skimage.io as io
+from skimage.transform import resize
+import h5py
+import fnmatch
+from PIL import Image
+
+def feature_extractor():
+	#trainloader = Train_Data_Loader( VIDEO_DIR, resize_w=128, resize_h=171, crop_w = 112, crop_h = 112, nb_frames=16)
+	net = C3D(487)
+        print('net', net)
+	## Loading pretrained model from sports and finetune the last layer
+	net.load_state_dict(torch.load('/data1/miayuan/pretrained_models/c3d.pickle'))
+	if RUN_GPU : 
+		net.cuda(0)
+        net.eval()
+        print('net', net)
+	feature_dim = 4096 if EXTRACTED_LAYER != 5 else 8192
+	video_list = os.listdir(VIDEO_DIR)
+        print('video_list', video_list)
+        if not os.path.isdir(OUTPUT_DIR):
+            os.mkdir(OUTPUT_DIR)
+        f = h5py.File(os.path.join(OUTPUT_DIR, OUTPUT_NAME), 'w')
+        
+        def count_files(directory, prefix_list):
+            lst = os.listdir(directory)
+            cnt_list = [len(fnmatch.filter(lst, x+'*')) for x in prefix_list]
+            return cnt_list
+
+        
+	for video_name in video_list: 
+		video_path = os.path.join(VIDEO_DIR, video_name)
+                print('video_path', video_path)
+		#video = imageio.get_reader(video_path,  'ffmpeg')
+                #print('video', video)
+                all_cnt = count_files(video_path, ('image_'))
+		total_frames = all_cnt[0]
+                print 'Total frames: %d'%total_frames 
+		valid_frames = total_frames/nb_frames * nb_frames
+		print 'Total validated frames: %d'%valid_frames
+		index_w = np.random.randint(resize_w - crop_w) ## crop
+		index_h = np.random.randint(resize_h - crop_h) ## crop
+		#features = np.array((valid_frames/nb_frames, feature_dim))
+                features = []
+                #print('features', features)
+		print 'NB features: %d' %(valid_frames/nb_frames)
+                #print(io.imread(os.path.join(video_path, 'image_{:04d}.jpg'.format(1))).shape)		
+		for i in range(valid_frames/nb_frames) :   
+                        clip = np.array([resize(io.imread(os.path.join(video_path, 'image_{:04d}.jpg'.format(j))), output_shape=(resize_w, resize_h), preserve_range=True) for j in range(i * nb_frames+1, (i+1) * nb_frames+1)])
+			#clip = np.array([resize(video.get_data(j), output_shape=(resize_w, resize_h), preserve_range=True) for j in range(i * nb_frames, (i+1) * nb_frames)])
+			clip = clip[:, index_w: index_w+ crop_w, index_h: index_h+ crop_h, :]
+			clip = torch.from_numpy(np.float32(clip.transpose(3, 0, 1, 2)))
+			clip = Variable(clip).cuda() if RUN_GPU else Variable(clip)			
+			clip = clip.resize(1, 3, nb_frames, crop_w, crop_h)
+                        #print('clip', clip)
+			_, clip_output = net(clip, EXTRACTED_LAYER)
+                        #print('clip_output', clip_output)  
+			clip_feature  = (clip_output.data).cpu()  
+                        features.append(clip_feature)
+                        #features[i] = np.array(clip_feature)
+                features = torch.cat(features, 0)
+                features = features.numpy()
+                print('features', features)       
+               
+                fgroup = f.create_group(video_name)
+		fgroup.create_dataset('c3d_features', data=features)
+                fgroup.create_dataset('total_frames', data=np.array(total_frames))
+                fgroup.create_dataset('valid_frames', data=np.array(valid_frames))
+	
+		#with open(os.path.join(OUTPUT_DIR, video_name[:-4]), 'wb') as f :
+		#	pickle.dump( features, f )
+		print '%s has been processed...'%video_name
+
+if __name__ == "__main__":
+
+	parser = argparse.ArgumentParser()
+
+	print '******--------- Extract C3D features ------*******'
+	parser.add_argument('-o', '--OUTPUT_DIR', dest='OUTPUT_DIR', type=str, default='./output_frm/', help='Output file name')
+	parser.add_argument('-l', '--EXTRACTED_LAYER', dest='EXTRACTED_LAYER', type=int, choices=[5, 6, 7], default=5, help='Feature extractor layer')
+	parser.add_argument('-i', '--VIDEO_DIR', dest='VIDEO_DIR', type = str, help='Input Video directory')
+	parser.add_argument('-gpu', '--gpu', dest='GPU', action = 'store_true', help='Run GPU?')
+        parser.add_argument('--OUTPUT_NAME', default='c3d_features.hdf5', help='The output name of the hdf5 features')
+
+	args = parser.parse_args()
+	params = vars(args) # convert to ordinary dict
+	print 'parsed parameters:'
+	print json.dumps(params, indent = 2)
+
+	OUTPUT_DIR = params['OUTPUT_DIR']
+	EXTRACTED_LAYER = params['EXTRACTED_LAYER']
+	VIDEO_DIR = params['VIDEO_DIR']
+	RUN_GPU = params['GPU']
+        OUTPUT_NAME = params['OUTPUT_NAME']
+	crop_w = 112
+	resize_w = 128
+	crop_h = 112
+	resize_h = 171
+	nb_frames = 16
+	feature_extractor()
+
+

FakeVD/code_test/C3D_Feature_Extractor/feature_extractor_vid.py

@@ -0,0 +1,209 @@
+# coding: utf-8
+# from data_provider import *
+from .C3D_model import *
+import torchvision
+import torch
+from torch.autograd import Variable
+import torch.nn.functional as F
+import argparse
+import os 
+from torch import save, load
+import pickle
+import time
+import numpy as np
+import PIL.Image as Image
+import skimage.io as io
+from skimage.transform import resize
+import h5py
+from PIL import Image
+
+def load_model_c3d(RUN_GPU = False):
+	net = C3D(487)
+	# print('net', net)
+	## Loading pretrained model from sports and finetune the last layer
+	net.load_state_dict(torch.load('./FakeVD/code_test/C3D_Feature_Extractor/c3d.pickle'))
+	if RUN_GPU : 
+		net.cuda(0)
+	net.eval()
+		# print('net', net)
+	
+	return net
+
+
+def feature_extractor(net, OUTPUT_DIR,VIDEO_DIR,video_path=None):
+	crop_w = 112
+	resize_w = 128
+	crop_h = 112
+	resize_h = 171
+	nb_frames = 16	
+	BATCH_SIZE = 10
+	EXTRACTED_LAYER = 6
+	RUN_GPU = False
+	
+	#trainloader = Train_Data_Loader( VIDEO_DIR, resize_w=128, resize_h=171, crop_w = 112, crop_h = 112, nb_frames=16)
+
+
+	feature_dim = 4096 if EXTRACTED_LAYER != 5 else 8192
+
+	# read video list from the folder 
+	if video_path:
+		video_list = [video_path]
+	else:
+		video_list = [f for f in os.listdir(VIDEO_DIR) if os.path.isfile(os.path.join(VIDEO_DIR, f))]
+
+
+	if not os.path.isdir(OUTPUT_DIR):
+		os.mkdir(OUTPUT_DIR)
+	# f = h5py.File(os.path.join(OUTPUT_DIR, OUTPUT_NAME), 'w')
+
+	# current location
+	temp_path = os.path.join(os.getcwd(), 'temp')
+	if not os.path.exists(temp_path):
+		os.mkdir(temp_path)
+
+	error_fid = open('error.txt', 'w')
+	for video_name in video_list: 
+		output_file_name = video_name.split('.')[0] + '.hdf5'
+		print("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")
+		print(output_file_name)
+		f = h5py.File(os.path.join(OUTPUT_DIR, output_file_name), 'w')
+
+		video_path = os.path.join(VIDEO_DIR, video_name)
+		print('video_path', video_path)
+		frame_path = os.path.join(temp_path, video_name)
+		if not os.path.exists(frame_path):
+			os.mkdir(frame_path)
+
+
+		print('Extracting video frames ...')
+		# using ffmpeg to extract video frames into a temporary folder
+		# example: ffmpeg -i video_validation_0000051.mp4 -q:v 2 -f image2 output/image%5d.jpg
+		os.system('ffmpeg -i ' + video_path + ' -q:v 2 -f image2 ' + frame_path + '/image_%5d.jpg')
+
+
+		print('Extracting features ...')
+		total_frames = len(os.listdir(frame_path))
+		if total_frames == 0:
+			error_fid.write(video_name+'\n')
+			print('Fail to extract frames for video: %s'%video_name)
+			continue
+
+		valid_frames = total_frames // nb_frames * nb_frames
+		n_feat = valid_frames // nb_frames   # 可提取的特征数，每个特征由 nb_frames 帧组成
+		n_batch = n_feat // BATCH_SIZE 
+		if n_feat - n_batch*BATCH_SIZE > 0:
+			n_batch = n_batch + 1
+		print('n_frames: %d; n_feat: %d; n_batch: %d'%(total_frames, n_feat, n_batch))
+		
+		#print 'Total frames: %d'%total_frames 
+		#print 'Total validated frames: %d'%valid_frames
+		#print 'NB features: %d' %(valid_frames/nb_frames)
+		# 随机裁剪
+		index_w = np.random.randint(resize_w - crop_w) ## crop
+		index_h = np.random.randint(resize_h - crop_h) ## crop
+
+		features = []
+
+		for i in range(n_batch-1):
+			input_blobs = []
+			for j in range(BATCH_SIZE):
+				clip = []
+				clip = np.array([resize(io.imread(os.path.join(frame_path, 'image_{:05d}.jpg'.format(k))), output_shape=(resize_w, resize_h), preserve_range=True) for k in range((i*BATCH_SIZE+j) * nb_frames+1, min((i*BATCH_SIZE+j+1) * nb_frames+1, valid_frames+1))])
+				# print('clip_shape', clip.shape)
+				clip = clip[:, index_w: index_w+ crop_w, index_h: index_h+ crop_h, :]
+				#print('clip_shape',clip.shape)
+				#print('range', range((i*BATCH_SIZE+j) * nb_frames+1, min((i*BATCH_SIZE+j+1) * nb_frames+1, valid_frames+1)))	
+				input_blobs.append(clip)
+			input_blobs = np.array(input_blobs, dtype='float32')
+			#print('input_blobs_shape', input_blobs.shape)
+			input_blobs = torch.from_numpy(np.float32(input_blobs.transpose(0, 4, 1, 2, 3)))
+			input_blobs = Variable(input_blobs).cuda() if RUN_GPU else Variable(input_blobs)
+			_, batch_output = net(input_blobs, EXTRACTED_LAYER)   # 输入后提取某一层	
+			batch_feature  = (batch_output.data).cpu()
+			features.append(batch_feature)
+
+		# The last batch
+		input_blobs = []
+		for j in range(n_feat-(n_batch-1)*BATCH_SIZE):
+			clip = []
+			clip = np.array([resize(io.imread(os.path.join(frame_path, 'image_{:05d}.jpg'.format(k))), output_shape=(resize_w, resize_h), preserve_range=True) for k in range(((n_batch-1)*BATCH_SIZE+j) * nb_frames+1, min(((n_batch-1)*BATCH_SIZE+j+1) * nb_frames+1, valid_frames+1))])
+
+			clip = clip[:, index_w: index_w+ crop_w, index_h: index_h+ crop_h, :]
+			#print('range', range(((n_batch-1)*BATCH_SIZE+j) * nb_frames+1, min(((n_batch-1)*BATCH_SIZE+j+1) * nb_frames+1, valid_frames+1)))
+			input_blobs.append(clip)
+		input_blobs = np.array(input_blobs, dtype='float32')
+		#print('input_blobs_shape', input_blobs.shape)
+		input_blobs = torch.from_numpy(np.float32(input_blobs.transpose(0, 4, 1, 2, 3)))
+		input_blobs = Variable(input_blobs).cuda() if RUN_GPU else Variable(input_blobs)
+		_, batch_output = net(input_blobs, EXTRACTED_LAYER)
+		batch_feature  = (batch_output.data).cpu()
+		features.append(batch_feature)
+
+		features = torch.cat(features, 0)
+		features = features.numpy()
+		print('features', features.shape)
+		fgroup = f.create_group(video_name.split('.')[0])
+		fgroup.create_dataset('c3d_features', data=features)
+		# fgroup.create_dataset('total_frames', data=np.array(total_frames))
+		# fgroup.create_dataset('valid_frames', data=np.array(valid_frames))
+
+		print('%s has been processed...'%video_name)
+
+
+		# clear temp frame folders
+		try: 
+			os.system('rm -rf ' + frame_path)
+		except: 
+			pass
+
+
+#		for i in range(valid_frames/nb_frames) :   
+#			clip = np.array([resize(io.imread(os.path.join(video_path, 'image_{:05d}.jpg'.format(j))), output_shape=(resize_w, resize_h), preserve_range=True) for j in range(i * nb_frames+1, (i+1) * nb_frames+1)])
+#			#clip = np.array([resize(video.get_data(j), output_shape=(resize_w, resize_h), preserve_range=True) for j in range(i * nb_frames, (i+1) * nb_frames)])
+#			clip = clip[:, index_w: index_w+ crop_w, index_h: index_h+ crop_h, :]
+#			clip = torch.from_numpy(np.float32(clip.transpose(3, 0, 1, 2)))
+#			clip = Variable(clip).cuda() if RUN_GPU else Variable(clip)			
+#			clip = clip.resize(1, 3, nb_frames, crop_w, crop_h)
+#			#print('clip', clip)
+#			_, clip_output = net(clip, EXTRACTED_LAYER)
+#			#print('clip_output', clip_output)  
+#			clip_feature  = (clip_output.data).cpu()  
+#			features.append(clip_feature)
+#			#features[i] = np.array(clip_feature)
+#		features = torch.cat(features, 0)
+#		features = features.numpy()
+#		print('features', features)       
+#		fgroup = f.create_group(video_name)
+#		fgroup.create_dataset('c3d_features', data=features)
+#		fgroup.create_dataset('total_frames', data=np.array(total_frames))
+#		fgroup.create_dataset('valid_frames', data=np.array(valid_frames))
+#	
+#		#with open(os.path.join(OUTPUT_DIR, video_name[:-4]), 'wb') as f :
+#		#	pickle.dump( features, f )
+#		print '%s has been processed...'%video_name
+
+
+if __name__ == "__main__":
+
+	# parser = argparse.ArgumentParser()
+	# parser.add_argument('-o', '--OUTPUT_DIR', dest='OUTPUT_DIR', type=str, default='./output_frm/', help='Output file name')
+	# parser.add_argument('-l', '--EXTRACTED_LAYER', dest='EXTRACTED_LAYER', type=int, choices=[5, 6, 7], default=6, help='Feature extractor layer')
+	# parser.add_argument('-i', '--VIDEO_DIR', dest='VIDEO_DIR', type = str, default='./raw_video/', help='Input Video directory')
+	# parser.add_argument('-v', '--VIDEO_PATH', dest='VIDEO_PATH', type=str, help='Path to a single video file to process')
+	# parser.add_argument('-gpu', '--gpu', dest='GPU', action = 'store_true', help='Run GPU?')
+	# # parser.add_argument('--OUTPUT_NAME', default='c3d_features.hdf5', help='The output name of the hdf5 features')
+	# parser.add_argument('-b', '--BATCH_SIZE', default=10, help='the batch size')
+	# parser.add_argument('-id', '--gpu_id', default=0, type=int)
+	# args = parser.parse_args()
+	# params = vars(args) # convert to ordinary dict
+	# print('parsed parameters:')
+
+	OUTPUT_DIR = './FakeVD/code/C3D_Feature_Extractor/output_frm'
+	VIDEO_DIR = './FakeVD/code/C3D_Feature_Extractor/raw_video'
+	VIDEO_PATH = 'douyin_6571001202379590925.mp4'
+	# OUTPUT_NAME = params['OUTPUT_NAME']
+
+	net = load_model_c3d()
+	feature_extractor(net, OUTPUT_DIR, VIDEO_DIR, video_path=VIDEO_PATH)
+
+

FakeVD/code_test/main.py

@@ -0,0 +1,67 @@
+
+import argparse
+import os
+import random
+import warnings
+warnings.filterwarnings('ignore')
+import numpy as np
+import torch
+from run import Run
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--model_name', default='SVFEND', help='SVFEND/FANVM/C3D/VGG/Bbox/Vggish/Bert/TextCNN/Comments/TikTec')
+parser.add_argument('--mode_eval', default= 'nocv', help='nocv/cv/temporal')
+parser.add_argument('--fold', type=int, default= 1, help='needed when model_eval=nocv') 
+
+parser.add_argument('--epoches', type=int, default=30)
+parser.add_argument('--batch_size', type = int, default=128)
+parser.add_argument('--num_workers', type=int, default=0)
+parser.add_argument('--epoch_stop', type=int, default=5) 
+parser.add_argument('--seed', type=int, default=2022)
+parser.add_argument('--gpu', type=int, required=True)
+parser.add_argument('--lr', type=float, default=0.0001)
+parser.add_argument('--lambd', type=float, default=0.1)
+parser.add_argument('--dropout', type=float, default=0.1) 
+parser.add_argument('--weight_decay', type=float, default=5e-5)
+
+parser.add_argument('--path_param', default= './checkpoints/')
+parser.add_argument('--path_tensorboard', default= './tb/')
+
+args = parser.parse_args()
+
+# os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu)
+
+seed = args.seed
+random.seed(seed)
+np.random.seed(seed)
+torch.manual_seed(seed)
+torch.cuda.manual_seed(seed)
+torch.backends.cudnn.benchmark = False
+torch.backends.cudnn.deterministic = True
+
+print (args)
+
+config = {
+        'model_name': args.model_name,
+        'mode_eval':args.mode_eval,
+        'fold':args.fold,
+
+        'epoches': args.epoches,
+        'batch_size': args.batch_size,
+        'num_workers': args.num_workers,
+        'epoch_stop': args.epoch_stop,
+        'seed': args.seed,
+        'device': args.gpu,
+        'lr': args.lr,
+        'lambd': args.lambd,
+        'dropout': args.dropout,
+        'weight_decay': args.weight_decay,
+
+        'path_param': args.path_param,
+        'path_tensorboard': args.path_tensorboard,
+        }
+
+
+if __name__ == '__main__':
+    Run(config = config
+        ).main()

FakeVD/code_test/models/Baselines.py

@@ -0,0 +1,160 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import rearrange
+from transformers import BertModel
+from .layers import Attention
+
+
+class bBbox(torch.nn.Module):
+    def __init__(self,fea_dim):
+        super(bBbox, self).__init__()
+        self.img_dim = 4096
+        self.attention1 = Attention(dim=128,heads=4)
+        self.attention2 = Attention(dim=128,heads=4)
+
+        self.linear_img = nn.Sequential(torch.nn.Linear(self.img_dim, fea_dim),torch.nn.ReLU())
+
+        self.classifier = nn.Linear(fea_dim,2)
+    
+    def forward(self,  **kwargs):
+        frames=kwargs['bbox_vgg']
+        fea_img = self.linear_img(frames) 
+        fea_img = torch.reshape(fea_img, (-1, 45, 128)) 
+        fea_img = self.attention1(fea_img)
+        fea_img = torch.mean(fea_img, -2) 
+        fea_img = torch.reshape(fea_img, (-1, 83, 128))
+        fea_img = self.attention2(fea_img)
+        fea_img = torch.mean(fea_img, -2) 
+        output = self.classifier(fea_img)
+        return output, fea_img
+
+class bC3D(torch.nn.Module):
+    def __init__(self,fea_dim):
+        super(bC3D, self).__init__()
+        # self.video_dim = 4096
+        self.video_dim = 2048
+        self.attention = Attention(dim=128,heads=4)
+
+        self.linear_video = nn.Sequential(torch.nn.Linear(self.video_dim, fea_dim),torch.nn.ReLU())
+
+        self.classifier = nn.Linear(fea_dim,2)
+    
+    def forward(self,  **kwargs):
+        c3d = kwargs['c3d'] 
+        fea_video = self.linear_video(c3d)
+        fea_video = self.attention(fea_video)
+        fea_video = torch.mean(fea_video, -2)
+        output = self.classifier(fea_video)
+        return output
+
+class bVGG(torch.nn.Module):
+    def __init__(self,fea_dim):
+        super(bVGG, self).__init__()
+        # self.img_dim = 4096
+        self.img_dim = 2048
+        self.attention = Attention(dim=128,heads=4)
+
+        self.linear_img = nn.Sequential(torch.nn.Linear(self.img_dim, fea_dim),torch.nn.ReLU())
+
+        self.classifier = nn.Linear(fea_dim,2)
+    
+    def forward(self,  **kwargs):
+        frames=kwargs['frames'] 
+        fea_img = self.linear_img(frames) 
+        fea_img = self.attention(fea_img)
+        fea_img = torch.mean(fea_img, -2)
+        output = self.classifier(fea_img)
+        return output
+
+class bVggish(torch.nn.Module):
+    def __init__(self,fea_dim):
+        super(bVggish, self).__init__()
+        # self.audio_dim = 128
+        self.attention = Attention(dim=128,heads=4)
+
+        self.vggish_layer = torch.hub.load('./torchvggish/', 'vggish', source = 'local')        
+        net_structure = list(self.vggish_layer.children())      
+        self.vggish_modified = nn.Sequential(*net_structure[-2:-1])
+
+        self.classifier = nn.Linear(fea_dim,2)
+    
+    def forward(self,  **kwargs):
+        audioframes=kwargs['audioframes']
+        fea_audio = self.vggish_modified(audioframes)
+        fea_audio = self.attention(fea_audio)
+        fea_audio = torch.mean(fea_audio, -2)
+        print (fea_audio.shape)
+        output = self.classifier(fea_audio)
+        return output, fea_audio
+
+
+class bBert(torch.nn.Module):
+    def __init__(self,bert_model,fea_dim, dropout):
+        super(bBert, self).__init__()
+        self.text_dim = 768
+
+        self.bert = BertModel.from_pretrained(bert_model).requires_grad_(False)
+
+        self.linear_text = nn.Sequential(torch.nn.Linear(self.text_dim, fea_dim),torch.nn.ReLU())
+        self.classifier = nn.Linear(fea_dim,2)
+    
+    def forward(self,  **kwargs):
+        title_inputid = kwargs['title_inputid']
+        title_mask=kwargs['title_mask']
+        fea_text=self.bert(title_inputid,attention_mask=title_mask)[1]
+        fea_text=self.linear_text(fea_text) 
+        output = self.classifier(fea_text)
+        return output,fea_text
+
+class bTextCNN(nn.Module):
+    def __init__(self, fea_dim, vocab_size):
+        super(bTextCNN, self).__init__()
+        self.vocab_size = vocab_size
+        self.fea_dim=fea_dim
+
+        self.channel_in = 1
+        self.filter_num = 14
+        self.window_size = [3,4,5]
+
+        self.textcnn =nn.ModuleList([nn.Conv2d(self.channel_in, self.filter_num, (K,self.vocab_size)) for K in self.window_size])
+        self.linear = nn.Sequential(torch.nn.Linear(len(self.window_size) * self.filter_num, self.fea_dim),torch.nn.ReLU())
+        self.classifier = nn.Linear(self.fea_dim,2)
+
+    def forward(self, **kwargs):
+        title_w2v = kwargs['title_w2v'] 
+        text = title_w2v.unsqueeze(1)
+        text = [F.relu(conv(text)).squeeze(3) for conv in self.textcnn]
+        text = [F.max_pool1d(i.squeeze(2), i.shape[-1]).squeeze(2) for i in text]
+        fea_text = torch.cat(text, 1)
+        fea_text = self.linear(fea_text) 
+
+        output = self.classifier(fea_text)
+        
+        return output
+
+class bComments(torch.nn.Module):
+    def __init__(self,bert_model,fea_dim):
+        super(bComments, self).__init__()
+        self.comment_dim = 768
+        self.bert = BertModel.from_pretrained(bert_model).requires_grad_(False)
+        self.attention = Attention(dim=128,heads=4)
+        self.linear_comment = nn.Sequential(torch.nn.Linear(self.comment_dim, fea_dim),torch.nn.ReLU())
+        self.classifier = nn.Linear(fea_dim,2)
+    
+    def forward(self,  **kwargs):
+        comments_inputid = kwargs['comments_inputid']
+        comments_mask=kwargs['comments_mask']
+        comments_feature=[]
+        for i in range(comments_inputid.shape[0]):
+            bert_fea=self.bert(comments_inputid[i], attention_mask=comments_mask[i])[1]
+            comments_feature.append(bert_fea)
+        comments_feature=torch.stack(comments_feature)
+        fea_comments=self.linear_comment(comments_feature)
+        print (fea_comments.shape)
+        fea_comments = self.attention(fea_comments)
+        fea_comments = torch.mean(fea_comments, -2)
+        output = self.classifier(fea_comments)
+        return output
+
+

FakeVD/code_test/models/FANVM.py

@@ -0,0 +1,133 @@
+
+
+import os
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import tqdm
+from sklearn.metrics import *
+from transformers import BertModel
+from zmq import device
+
+from .layers import *
+
+
+class TextCNN(nn.Module):
+    def __init__(self, fea_dim, vocab_size):
+        super(TextCNN, self).__init__()
+        self.vocab_size = vocab_size
+        self.fea_dim=fea_dim
+
+        self.channel_in = 1
+        self.filter_num = 14
+        self.window_size = [3,4,5]
+
+        self.textcnn =nn.ModuleList([nn.Conv2d(self.channel_in, self.filter_num, (K,self.vocab_size)) for K in self.window_size])
+        self.linear = nn.Sequential(torch.nn.Linear(len(self.window_size) * self.filter_num, self.fea_dim),torch.nn.ReLU())
+
+    def forward(self, inputs):
+        text = inputs.unsqueeze(1)
+        text = [F.relu(conv(text)).squeeze(3) for conv in self.textcnn]
+        text = [F.max_pool1d(i.squeeze(2), i.shape[-1]).squeeze(2) for i in text]
+        fea_text = torch.cat(text, 1)
+        fea_text = self.linear(fea_text) 
+        
+        return fea_text
+
+
+class VideoEncoder(nn.Module):
+    def __init__(self,emb_dim,fea_dim):
+        super(VideoEncoder, self).__init__()
+
+        self.emb_dim = emb_dim
+        self.linear1 = torch.nn.Linear(self.emb_dim, self.emb_dim, bias=False)
+        self.linear2 = nn.Sequential(torch.nn.Linear(self.emb_dim, fea_dim),torch.nn.ReLU())
+
+    def forward(self, input_thumb, input_L):
+        input_ALL = torch.cat((input_L, input_thumb),1) #（bs,len+1,4096）
+        fea_A = torch.bmm(input_thumb,self.linear1(input_ALL).permute(0,2,1)) # (bs, 1, len+1)
+        fea_alpha = F.softmax(fea_A) # (bs, 1, len+1)
+        fea_V = torch.matmul(fea_alpha,input_ALL).squeeze() # (bs, 4096)
+        fea = self.linear2(fea_V)
+        return fea
+
+class ReverseLayerF(Function):
+    #@staticmethod
+    def forward(self, x):
+        self.lambd = 1
+        return x.view_as(x)
+
+    #@staticmethod
+    def backward(self, grad_output):
+        return (grad_output * -self.lambd)
+
+def grad_reverse(x):
+    return ReverseLayerF.apply(x)
+
+
+class FANVMModel(torch.nn.Module):
+    def __init__(self,bert_model,fea_dim):
+        super(FANVMModel, self).__init__()
+        self.text_dim = 768
+        self.img_dim = 4096
+        self.topic_dim = 15
+
+        self.bert = BertModel.from_pretrained(bert_model).requires_grad_(False)
+        self.title_encoder = TextCNN(fea_dim, self.text_dim)
+        # self.comments_encoder = BiLSTM(self.text_dim,300,fea_dim)
+        self.video_encoder = VideoEncoder(self.img_dim,fea_dim)
+
+        self.gate_m1 = torch.nn.Linear(fea_dim*2,1)
+        self.gate_m2 = torch.nn.Linear(fea_dim*2,1)
+
+        self.classifier = nn.Linear(fea_dim*2,2)
+        self.classifier_topic = nn.Linear(fea_dim*3,self.topic_dim)
+    
+    def forward(self, **kwargs):
+        title_inputid = kwargs['title_inputid']#(batch,512)
+        title_mask = kwargs['title_mask']#(batch,512)
+        fea_text = self.bert(title_inputid,attention_mask=title_mask)[0]  #(bs,seq,768)
+        fea_text = self.title_encoder(fea_text)
+        fea_R = fea_text # (bs, 128)
+
+        comments_inputid = kwargs['comments_inputid']#(batch,20,250)
+        comments_mask=kwargs['comments_mask']#(batch,20,250)
+        comments_like=kwargs['comments_like']
+        comments_feature=[]
+        for i in range(comments_inputid.shape[0]):
+            bert_fea=self.bert(comments_inputid[i], attention_mask=comments_mask[i])[0]
+            comments_feature.append(self.comments_encoder(bert_fea))
+        comments_feature=torch.stack(comments_feature) #(batch,seq,fea_dim)
+        fea_comments =[]
+        for v in range(comments_like.shape[0]): # batch内循环
+            # print (reviews_like[v])
+            comments_weight=torch.stack([torch.true_divide((i+1),(comments_like[v].shape[0]+comments_like[v].sum())) for i in comments_like[v]])
+            comments_fea_reweight = torch.sum(comments_feature[v]*(comments_weight.reshape(comments_weight.shape[0],1)),dim=0)
+            fea_comments.append(comments_fea_reweight)
+        fea_comments = torch.stack(fea_comments)
+        fea_H = fea_comments # (bs, 600)
+
+        frames = kwargs['frames'] # (bs, 30, 4096)
+        frame_thumb = kwargs['frame_thmub'] # (bs,1,4096)
+        fea_video = self.video_encoder(frame_thumb, frames)
+        fea_V = fea_video # (bs, 128)
+
+        s = kwargs['s']
+        
+        ## fusion: title, frames 
+        m1 = self.gate_m1(torch.cat((fea_V, fea_R),1))
+        fea_P = torch.add(torch.mul(m1,fea_V),torch.mul((1-m1),fea_R)) 
+        ## fusion: comments, title
+        m2 = s.reshape((s.shape[0],1))
+        fea_E = torch.add(torch.mul(fea_H,m2),torch.mul(fea_R,(1-m2)))
+
+        fea_fnd = torch.cat((fea_P,fea_E),1).to(torch.float32)
+        output = self.classifier(fea_fnd)
+
+        fea_topic = torch.cat((fea_H, fea_R, fea_V),1)
+        fea_reverse = grad_reverse(fea_topic)
+        output_topic = self.classifier_topic(fea_reverse)
+
+        return output,output_topic,fea_fnd

FakeVD/code_test/models/SVFEND.py

@@ -0,0 +1,110 @@
+import copy
+import json
+import os
+import time
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torchvision.transforms as transforms
+import tqdm
+from sklearn.metrics import *
+from tqdm import tqdm
+from transformers import AutoConfig, BertModel
+from transformers.models.bert.modeling_bert import BertLayer
+from zmq import device
+
+from .coattention import *
+from .layers import *
+from FakeVD.code_test.utils.metrics import *
+
+
+class SVFENDModel(torch.nn.Module):
+    def __init__(self,bert_model,fea_dim,dropout):
+        super(SVFENDModel, self).__init__()
+        self.bert = BertModel.from_pretrained("./FakeVD/Models/bert-base-chinese/").requires_grad_(False)
+
+        self.text_dim = 768
+        self.comment_dim = 768
+        self.img_dim = 4096
+        self.video_dim = 4096
+        self.num_frames = 83
+        self.num_audioframes = 50
+        self.num_comments = 23
+        self.dim = fea_dim
+        self.num_heads = 4
+
+        self.dropout = dropout
+
+        self.vggish_layer = torch.hub.load('./FakeVD/Models/torchvggish/', 'vggish', source = 'local')        
+        net_structure = list(self.vggish_layer.children())      
+        self.vggish_modified = nn.Sequential(*net_structure[-2:-1])
+
+        self.co_attention_ta = co_attention(d_k=fea_dim, d_v=fea_dim, n_heads=self.num_heads, dropout=self.dropout, d_model=fea_dim,
+                                        visual_len=self.num_audioframes, sen_len=512, fea_v=self.dim, fea_s=self.dim, pos=False)
+        self.co_attention_tv = co_attention(d_k=fea_dim, d_v=fea_dim, n_heads=self.num_heads, dropout=self.dropout, d_model=fea_dim,
+                                        visual_len=self.num_frames, sen_len=512, fea_v=self.dim, fea_s=self.dim, pos=False)
+        self.trm = nn.TransformerEncoderLayer(d_model = self.dim, nhead = 2, batch_first = True)
+
+
+        self.linear_text = nn.Sequential(torch.nn.Linear(self.text_dim, fea_dim), torch.nn.ReLU(),nn.Dropout(p=self.dropout))
+        self.linear_comment = nn.Sequential(torch.nn.Linear(self.comment_dim, fea_dim), torch.nn.ReLU(),nn.Dropout(p=self.dropout))
+        self.linear_img = nn.Sequential(torch.nn.Linear(self.img_dim, fea_dim), torch.nn.ReLU(),nn.Dropout(p=self.dropout))
+        self.linear_video = nn.Sequential(torch.nn.Linear(self.video_dim, fea_dim), torch.nn.ReLU(),nn.Dropout(p=self.dropout))
+        self.linear_intro = nn.Sequential(torch.nn.Linear(self.text_dim, fea_dim),torch.nn.ReLU(),nn.Dropout(p=self.dropout))
+        self.linear_audio = nn.Sequential(torch.nn.Linear(fea_dim, fea_dim), torch.nn.ReLU(),nn.Dropout(p=self.dropout))
+
+        self.classifier = nn.Linear(fea_dim,2)
+
+    def forward(self,  **kwargs):
+
+        ### User Intro ###
+        
+
+        ### Title ###
+        title_inputid = kwargs['title_inputid']#(batch,512)
+        title_mask=kwargs['title_mask']#(batch,512)
+
+        fea_text=self.bert(title_inputid,attention_mask=title_mask)['last_hidden_state']#(batch,sequence,768)
+        fea_text=self.linear_text(fea_text) 
+
+        ### Audio Frames ###
+        audioframes=kwargs['audioframes']#(batch,36,12288)
+        audioframes_masks = kwargs['audioframes_masks']
+        fea_audio = self.vggish_modified(audioframes) #(batch, frames, 128)
+        fea_audio = self.linear_audio(fea_audio) 
+        fea_audio, fea_text = self.co_attention_ta(v=fea_audio, s=fea_text, v_len=fea_audio.shape[1], s_len=fea_text.shape[1])
+        fea_audio = torch.mean(fea_audio, -2)
+
+        ### Image Frames ###
+        frames=kwargs['frames']#(batch,30,4096)
+        frames_masks = kwargs['frames_masks']
+        fea_img = self.linear_img(frames) 
+        fea_img, fea_text = self.co_attention_tv(v=fea_img, s=fea_text, v_len=fea_img.shape[1], s_len=fea_text.shape[1])
+        fea_img = torch.mean(fea_img, -2)
+
+        fea_text = torch.mean(fea_text, -2)
+
+        ### C3D ###
+        c3d = kwargs['c3d'] # (batch, 36, 4096)
+        c3d_masks = kwargs['c3d_masks']
+        fea_video = self.linear_video(c3d) #(batch, frames, 128)
+        fea_video = torch.mean(fea_video, -2)
+
+        ### Comment ###
+        
+        fea_text = fea_text.unsqueeze(1)
+        # fea_comments = fea_comments.unsqueeze(1)
+        fea_img = fea_img.unsqueeze(1)
+        fea_audio = fea_audio.unsqueeze(1)
+        fea_video = fea_video.unsqueeze(1)
+        # fea_intro = fea_intro.unsqueeze(1)
+
+        fea=torch.cat((fea_text, fea_audio, fea_video,fea_img),1) # (bs, 6, 128)
+        fea = self.trm(fea)
+        fea = torch.mean(fea, -2)
+        
+        output = self.classifier(fea)
+
+        return output, fea

FakeVD/code_test/models/TikTec.py

@@ -0,0 +1,140 @@
+import torch
+from torch import nn
+
+class MLP(nn.Module):
+    def __init__(self, input_dim, hidden_dims, output_dim, dropout):
+        super(MLP, self).__init__()
+        layers = list()
+        curr_dim = input_dim
+        for hidden_dim in hidden_dims:
+            layers.append(nn.Linear(curr_dim, hidden_dim))
+            layers.append(nn.BatchNorm1d(hidden_dim))
+            layers.append(nn.ReLU())
+            layers.append(nn.Dropout(p=dropout))
+            curr_dim = hidden_dim
+        layers.append(nn.Linear(curr_dim, output_dim))
+        self.mlp = nn.Sequential(*layers)
+
+    def forward(self, input):
+        return self.mlp(input)
+
+class MaskAvg(nn.Module):
+    def __init__(self):
+        super(MaskAvg, self).__init__()
+
+    def forward(self, input, mask):
+        score = torch.ones((input.shape[0], input.shape[1]), device=input.device)
+        score = score.masked_fill(mask == 0, float('-inf'))
+        score = torch.softmax(score, dim=-1).unsqueeze(1)
+        output = torch.matmul(score, input).squeeze(1)
+        return output
+
+class CVRL(nn.Module):
+    def __init__(self, d_w, d_f, obj_num, gru_dim):
+        super(CVRL, self).__init__()
+        self.gru = nn.GRU(d_w, gru_dim, batch_first=True, bidirectional=True)
+
+        self.linear_r = nn.Linear(d_f, 1)
+        self.linear_h = nn.Linear(2*gru_dim, obj_num)
+
+    def forward(self, caption_feature, visual_feature):
+        # IN: caption_feature: (bs, K, S, d_w), visual_feature: (bs, K, obj_num, d_f)
+        # OUT: frame_visual_rep: (bs, K, d_f)
+        encoded_caption, _ = self.gru(caption_feature.view(-1, caption_feature.shape[-2], caption_feature.shape[-1]))  # (bs*K, S, 2*gru_dim)
+        encoded_caption = encoded_caption.view(-1, caption_feature.shape[-3], caption_feature.shape[-2], encoded_caption.shape[-1])  # (bs, K, S, 2*gru_dim)
+        frame_caption_rep = encoded_caption.max(dim=2).values  # (bs, K, 2*gru_dim)
+
+        alpha = self.linear_r(visual_feature).squeeze() + self.linear_h(frame_caption_rep)  # (bs, K, obj_num)
+        alpha = torch.softmax(torch.tanh(alpha), dim=-1).unsqueeze(dim=-2)  # (bs, K, 1, obj_num)
+        frame_visual_rep = alpha.matmul(visual_feature)  # (bs, K, 1, d_f)
+        frame_visual_rep = frame_visual_rep.squeeze()  # (bs, K, d_f)
+        return frame_visual_rep
+
+class ASRL(nn.Module):
+    def __init__(self, d_w, gru_dim):
+        super(ASRL, self).__init__()
+        self.gru = nn.GRU(d_w, gru_dim, batch_first=True, bidirectional=True)
+
+    def forward(self, asr_feature):
+        # IN: asr_feature: (bs, N, d_w)
+        # OUT: text_audio_rep: (bs, N, 2*gru_dim)
+        text_audio_rep, _ = self.gru(asr_feature)
+        return text_audio_rep
+
+class VCIF(nn.Module):
+    def __init__(self, d_f, d_w, d_H, gru_f_dim, gru_w_dim, dropout):
+        super(VCIF, self).__init__()
+
+        self.param_D = nn.Parameter(torch.empty((d_f, d_w)))
+        self.param_Df = nn.Parameter(torch.empty((d_f, d_H)))
+        self.param_Dw = nn.Parameter(torch.empty((d_w, d_H)))
+        self.param_df = nn.Parameter(torch.empty(d_H))
+        self.param_dw = nn.Parameter(torch.empty(d_H))
+
+        self.gru_f = nn.GRU(d_f, gru_f_dim, batch_first=True)
+        self.gru_w = nn.GRU(d_w, gru_w_dim, batch_first=True)
+        self.mask_avg = MaskAvg()
+        self.dropout = nn.Dropout(p=dropout)
+
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        nn.init.xavier_uniform_(self.param_D)
+        nn.init.xavier_uniform_(self.param_Df)
+        nn.init.xavier_uniform_(self.param_Dw)
+        nn.init.uniform_(self.param_df)
+        nn.init.uniform_(self.param_dw)
+
+    def forward(self, frame_visual_rep, text_audio_rep, mask_K, mask_N):
+        # IN: frame_visual_rep: (bs, K, d_f), text_audio_rep: (bs, N, d_w)
+        # OUT: video_rep: (bs, gru_f_dim + gru_w_dim)
+        affinity_matrix = torch.tanh(frame_visual_rep.matmul(self.param_D).matmul(text_audio_rep.transpose(-1, -2)))
+        affinity_matrix = self.dropout(affinity_matrix)
+
+        frame_co_att_map = torch.tanh(frame_visual_rep.matmul(self.param_Df) + affinity_matrix.matmul(text_audio_rep).matmul(self.param_Dw))
+        word_co_att_map = torch.tanh(text_audio_rep.matmul(self.param_Dw) + affinity_matrix.transpose(-1, -2).matmul(frame_visual_rep).matmul(self.param_Df))
+        frame_co_att_map = self.dropout(frame_co_att_map)
+        word_co_att_map = self.dropout(word_co_att_map)
+
+        frame_att_weight = torch.softmax(frame_co_att_map.matmul(self.param_df), dim=-1)
+        word_att_weight = torch.softmax(word_co_att_map.matmul(self.param_dw), dim=-1)
+
+        frame_visual_weighted_rep = frame_att_weight.unsqueeze(dim=-1) * frame_visual_rep
+        text_audio_weighted_rep = word_att_weight.unsqueeze(dim=-1) * text_audio_rep
+
+        encoded_visual_rep, _ = self.gru_f(frame_visual_weighted_rep)
+        encoded_speech_rep, _ = self.gru_w(text_audio_weighted_rep)
+
+        visual_rep = self.mask_avg(encoded_visual_rep, mask_K)  # (bs, gru_f_dim)
+        speech_rep = self.mask_avg(encoded_speech_rep, mask_N)  # (bs, gru_w_dim)
+
+        video_rep = torch.cat([visual_rep, speech_rep], dim=-1)
+        return video_rep
+
+class TikTecModel(nn.Module):
+    def __init__(self, word_dim=300, mfcc_dim=650, visual_dim=1000, obj_num=45, CVRL_gru_dim=200, ASRL_gru_dim=500, VCIF_d_H=200, VCIF_gru_f_dim=200, VCIF_gru_w_dim=100, VCIF_dropout=0.2, MLP_hidden_dims=[512], MLP_dropout=0.2):
+        super(TikTecModel, self).__init__()
+        self.CVRL = CVRL(d_w=word_dim, d_f=visual_dim, obj_num=obj_num, gru_dim=CVRL_gru_dim)
+        self.ASRL = ASRL(d_w=(word_dim + mfcc_dim), gru_dim=ASRL_gru_dim)
+        self.VCIF = VCIF(d_f=visual_dim, d_w=2*ASRL_gru_dim, d_H=VCIF_d_H, gru_f_dim=VCIF_gru_f_dim, gru_w_dim=VCIF_gru_w_dim, dropout=VCIF_dropout)
+        self.MLP = MLP(VCIF_gru_f_dim + VCIF_gru_w_dim, MLP_hidden_dims, 2, MLP_dropout)
+
+    def forward(self, **kwargs):
+        # IN:
+        #   caption_feature: (bs, K, S, word_dim) = (bs, 200, 100, 300)
+        #   visual_feature: (bs, K, obj_num, visual_dim) = (bs, 200, 45, 1000)
+        #   asr_feature: (bs, N, word_dim + mfcc_dim) = (bs, 500, 300 + 650)
+        #   mask_K: (bs, K) = (bs, 200)
+        #   mask_N: (bs, N) = (bs, 500)
+        # OUT: (bs, 2)
+        caption_feature = kwargs['caption_feature']
+        visual_feature = kwargs['visual_feature']
+        asr_feature = kwargs['asr_feature']
+        mask_K = kwargs['mask_K']
+        mask_N = kwargs['mask_N']
+
+        frame_visual_rep = self.CVRL(caption_feature, visual_feature)
+        text_audio_rep = self.ASRL(asr_feature)
+        video_rep = self.VCIF(frame_visual_rep, text_audio_rep, mask_K, mask_N)
+        output = self.MLP(video_rep)
+        return output

FakeVD/code_test/models/Trainer.py

@@ -0,0 +1,235 @@
+import copy
+import json
+import os
+import time
+from tkinter import E
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torchvision.transforms as transforms
+import tqdm
+from sklearn.metrics import *
+from tqdm import tqdm
+from transformers import BertModel
+from FakeVD.code_test.utils.metrics import *
+from zmq import device
+
+from .coattention import *
+from .layers import *
+
+
+
+class Trainer():
+    def __init__(self,
+                model, 
+                 device,
+                 lr,
+                 dropout,
+                 dataloaders,
+                 weight_decay,
+                 save_param_path,
+                 writer, 
+                 epoch_stop,
+                 epoches,
+                 mode,
+                 model_name, 
+                 event_num,
+                 save_threshold = 0.0, 
+                 start_epoch = 0,
+                 ):
+        
+        self.model = model
+        self.device = device
+        self.mode = mode
+        self.model_name = model_name
+        self.event_num = event_num
+
+        self.dataloaders = dataloaders
+        self.start_epoch = start_epoch
+        self.num_epochs = epoches
+        self.epoch_stop = epoch_stop
+        self.save_threshold = save_threshold
+        self.writer = writer
+
+        if os.path.exists(save_param_path):
+            self.save_param_path = save_param_path
+        else:
+            self.save_param_path = os.makedirs(save_param_path)
+            self.save_param_path= save_param_path
+
+        self.lr = lr
+        self.weight_decay = weight_decay
+        self.dropout = dropout
+    
+        self.criterion = nn.CrossEntropyLoss()
+        
+
+    def train(self):
+
+        since = time.time()
+
+        self.model.cuda()
+
+        best_model_wts_test = copy.deepcopy(self.model.state_dict())
+        best_acc_test = 0.0
+        best_epoch_test = 0
+        is_earlystop = False
+
+        if self.mode == "eann":
+            best_acc_test_event = 0.0
+            best_epoch_test_event = 0
+
+        for epoch in range(self.start_epoch, self.start_epoch+self.num_epochs):
+            if is_earlystop:
+                break
+            print('-' * 50)
+            print('Epoch {}/{}'.format(epoch+1, self.start_epoch+self.num_epochs))
+            print('-' * 50)
+
+            p = float(epoch) / 100
+            lr = self.lr / (1. + 10 * p) ** 0.75
+            self.optimizer = torch.optim.Adam(params=self.model.parameters(), lr=lr)
+            
+            for phase in ['train', 'test']:
+                if phase == 'train':
+                    self.model.train()  
+                else:
+                    self.model.eval()   
+                print('-' * 10)
+                print (phase.upper())
+                print('-' * 10)
+
+                running_loss_fnd = 0.0
+                running_loss = 0.0 
+                tpred = []
+                tlabel = []
+
+                if self.mode == "eann":
+                    running_loss_event = 0.0
+                    tpred_event = []
+                    tlabel_event = []
+
+                for batch in tqdm(self.dataloaders[phase]):
+                    batch_data=batch
+                    for k,v in batch_data.items():
+                        batch_data[k]=v.cuda()
+                    label = batch_data['label']
+                    if self.mode == "eann":
+                        label_event = batch_data['label_event']
+
+                
+                    with torch.set_grad_enabled(phase == 'train'):
+                        if self.mode == "eann":
+                            outputs, outputs_event,fea = self.model(**batch_data)
+                            loss_fnd = self.criterion(outputs, label)
+                            loss_event = self.criterion(outputs_event, label_event)
+                            loss = loss_fnd + loss_event
+                            _, preds = torch.max(outputs, 1)
+                            _, preds_event = torch.max(outputs_event, 1)
+                        else:
+                            outputs,fea = self.model(**batch_data)
+                            _, preds = torch.max(outputs, 1)
+                            loss = self.criterion(outputs, label)
+
+                        if phase == 'train':
+                            loss.backward()
+                            self.optimizer.step()
+                            self.optimizer.zero_grad()
+
+                    tlabel.extend(label.detach().cpu().numpy().tolist())
+                    tpred.extend(preds.detach().cpu().numpy().tolist())
+                    running_loss += loss.item() * label.size(0)
+
+                    if self.mode == "eann":
+                        tlabel_event.extend(label_event.detach().cpu().numpy().tolist())
+                        tpred_event.extend(preds_event.detach().cpu().numpy().tolist())
+                        running_loss_event += loss_event.item() * label_event.size(0)
+                        running_loss_fnd += loss_fnd.item() * label.size(0)
+                    
+                epoch_loss = running_loss / len(self.dataloaders[phase].dataset)
+                print('Loss: {:.4f} '.format(epoch_loss))
+                results = metrics(tlabel, tpred)
+                print (results)
+                self.writer.add_scalar('Loss/'+phase, epoch_loss, epoch+1)
+                self.writer.add_scalar('Acc/'+phase, results['acc'], epoch+1)
+                self.writer.add_scalar('F1/'+phase, results['f1'], epoch+1)
+
+                if self.mode == "eann":
+                    epoch_loss_fnd = running_loss_fnd / len(self.dataloaders[phase].dataset)
+                    print('Loss_fnd: {:.4f} '.format(epoch_loss_fnd))
+                    epoch_loss_event = running_loss_event / len(self.dataloaders[phase].dataset)
+                    print('Loss_event: {:.4f} '.format(epoch_loss_event))
+                    self.writer.add_scalar('Loss_fnd/'+phase, epoch_loss_fnd, epoch+1)
+                    self.writer.add_scalar('Loss_event/'+phase, epoch_loss_event, epoch+1)
+                
+                if phase == 'test':
+                    if results['acc'] > best_acc_test:
+                        best_acc_test = results['acc']
+                        best_model_wts_test = copy.deepcopy(self.model.state_dict())
+                        best_epoch_test = epoch+1
+                        if best_acc_test > self.save_threshold:
+                            torch.save(self.model.state_dict(), self.save_param_path + "_test_epoch" + str(best_epoch_test) + "_{0:.4f}".format(best_acc_test))
+                            print ("saved " + self.save_param_path + "_test_epoch" + str(best_epoch_test) + "_{0:.4f}".format(best_acc_test) )
+                    else:
+                        if epoch-best_epoch_test >= self.epoch_stop-1:
+                            is_earlystop = True
+                            print ("early stopping...")
+
+        time_elapsed = time.time() - since
+        print('Training complete in {:.0f}m {:.0f}s'.format(
+            time_elapsed // 60, time_elapsed % 60))
+        print("Best model on test: epoch" + str(best_epoch_test) + "_" + str(best_acc_test))
+
+        if self.mode == "eann":
+            print("Event: Best model on test: epoch" + str(best_epoch_test_event) + "_" + str(best_acc_test_event))
+
+        self.model.load_state_dict(best_model_wts_test)
+        return self.test()
+
+
+    def test(self):
+        since = time.time()
+
+        self.model.cuda()
+        self.model.eval()   
+
+        pred = []
+        label = []
+
+        if self.mode == "eann":
+            pred_event = []
+            label_event = []
+
+        for batch in tqdm(self.dataloaders['test']):
+            with torch.no_grad(): 
+                batch_data=batch
+                for k,v in batch_data.items():
+                    batch_data[k]=v.cuda()
+                batch_label = batch_data['label']
+
+                if self.mode == "eann":
+                    batch_label_event = batch_data['label_event']
+                    batch_outputs, batch_outputs_event, fea = self.model(**batch_data)
+                    _, batch_preds_event = torch.max(batch_outputs_event, 1)
+
+                    label_event.extend(batch_label_event.detach().cpu().numpy().tolist())
+                    pred_event.extend(batch_preds_event.detach().cpu().numpy().tolist())
+                else: 
+                    batch_outputs,fea = self.model(**batch_data) 
+
+                _, batch_preds = torch.max(batch_outputs, 1)
+
+                label.extend(batch_label.detach().cpu().numpy().tolist())
+                pred.extend(batch_preds.detach().cpu().numpy().tolist())
+
+        print (get_confusionmatrix_fnd(np.array(pred), np.array(label)))
+        print (metrics(label, pred))
+
+        if self.mode == "eann" and self.model_name != "FANVM":
+            print ("event:")
+            print (accuracy_score(np.array(label_event), np.array(pred_event)))
+
+        return metrics(label, pred)
+

FakeVD/code_test/models/Trainer_3set.py

@@ -0,0 +1,241 @@
+import copy
+import json
+import os
+import time
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torchvision.transforms as transforms
+import tqdm
+from sklearn.metrics import *
+from tqdm import tqdm
+from transformers import BertModel
+from FakeVD.code_test.utils.metrics import *
+from zmq import device
+
+from .coattention import *
+from .layers import *
+
+
+class Trainer3():
+    def __init__(self,
+                model, 
+                 device,
+                 lr,
+                 dropout,
+                 dataloaders,
+                 weight_decay,
+                 save_param_path,
+                 writer, 
+                 epoch_stop,
+                 epoches,
+                 mode,
+                 model_name, 
+                 event_num,
+                 save_threshold = 0.0, 
+                 start_epoch = 0,
+                 ):
+        
+        self.model = model
+
+        self.device = device
+        self.mode = mode
+        self.model_name = model_name
+        self.event_num = event_num
+
+        self.dataloaders = dataloaders
+        self.start_epoch = start_epoch
+        self.num_epochs = epoches
+        self.epoch_stop = epoch_stop
+        self.save_threshold = save_threshold
+        self.writer = writer
+
+        if os.path.exists(save_param_path):
+            self.save_param_path = save_param_path
+        else:
+            self.save_param_path = os.makedirs(save_param_path)
+            self.save_param_path= save_param_path
+
+        self.lr = lr
+        self.weight_decay = weight_decay
+        self.dropout = dropout
+    
+        self.criterion = nn.CrossEntropyLoss()
+        
+
+    def train(self):
+
+        since = time.time()
+
+        self.model.cuda()
+
+        best_model_wts_val = copy.deepcopy(self.model.state_dict())
+        best_acc_val = 0.0
+        best_epoch_val = 0
+
+        is_earlystop = False
+
+        if self.mode == "eann":
+            best_acc_val_event = 0.0
+            best_epoch_val_event = 0
+
+        for epoch in range(self.start_epoch, self.start_epoch+self.num_epochs):
+            if is_earlystop:
+                break
+            print('-' * 50)
+            print('Epoch {}/{}'.format(epoch+1, self.start_epoch+self.num_epochs))
+            print('-' * 50)
+
+            p = float(epoch) / 100
+            lr = self.lr / (1. + 10 * p) ** 0.75
+            self.optimizer = torch.optim.Adam(params=self.model.parameters(), lr=lr)
+            
+            for phase in ['train', 'val', 'test']:
+                if phase == 'train':
+                    self.model.train()  
+                else:
+                    self.model.eval()   
+                print('-' * 10)
+                print (phase.upper())
+                print('-' * 10)
+
+                running_loss_fnd = 0.0
+                running_loss = 0.0 
+                tpred = []
+                tlabel = []
+
+                if self.mode == "eann":
+                    running_loss_event = 0.0
+                    tpred_event = []
+                    tlabel_event = []
+
+                for batch in tqdm(self.dataloaders[phase]):
+                    batch_data=batch
+                    for k,v in batch_data.items():
+                        batch_data[k]=v.cuda()
+                    label = batch_data['label']
+                    if self.mode == "eann":
+                        label_event = batch_data['label_event']
+
+                    self.optimizer.zero_grad()
+
+                    with torch.set_grad_enabled(phase == 'train'):
+                        if self.mode == "eann":
+                            outputs, outputs_event,fea = self.model(**batch_data)
+                            loss_fnd = self.criterion(outputs, label)
+                            loss_event = self.criterion(outputs_event, label_event)
+                            loss = loss_fnd + loss_event
+                            _, preds = torch.max(outputs, 1)
+                            _, preds_event = torch.max(outputs_event, 1)
+                        else:
+                            outputs,fea = self.model(**batch_data)
+                            _, preds = torch.max(outputs, 1)
+                            loss = self.criterion(outputs, label)
+
+                        if phase == 'train':
+                            loss.backward()
+                            torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)
+                            self.optimizer.step()
+                            self.optimizer.zero_grad()
+
+                    tlabel.extend(label.detach().cpu().numpy().tolist())
+                    tpred.extend(preds.detach().cpu().numpy().tolist())
+                    running_loss += loss.item() * label.size(0)
+
+                    if self.mode == "eann":
+                        tlabel_event.extend(label_event.detach().cpu().numpy().tolist())
+                        tpred_event.extend(preds_event.detach().cpu().numpy().tolist())
+                        running_loss_event += loss_event.item() * label_event.size(0)
+                        running_loss_fnd += loss_fnd.item() * label.size(0)
+                    
+                epoch_loss = running_loss / len(self.dataloaders[phase].dataset)
+                print('Loss: {:.4f} '.format(epoch_loss))
+                results = metrics(tlabel, tpred)
+                print (results)
+                self.writer.add_scalar('Loss/'+phase, epoch_loss, epoch+1)
+                self.writer.add_scalar('Acc/'+phase, results['acc'], epoch+1)
+                self.writer.add_scalar('F1/'+phase, results['f1'], epoch+1)
+
+                if self.mode == "eann":
+                    epoch_loss_fnd = running_loss_fnd / len(self.dataloaders[phase].dataset)
+                    print('Loss_fnd: {:.4f} '.format(epoch_loss_fnd))
+                    epoch_loss_event = running_loss_event / len(self.dataloaders[phase].dataset)
+                    print('Loss_event: {:.4f} '.format(epoch_loss_event))
+                    self.writer.add_scalar('Loss_fnd/'+phase, epoch_loss_fnd, epoch+1)
+                    self.writer.add_scalar('Loss_event/'+phase, epoch_loss_event, epoch+1)
+
+                if phase == 'val' and results['acc'] > best_acc_val:
+                    best_acc_val = results['acc']
+                    best_model_wts_val = copy.deepcopy(self.model.state_dict())
+                    best_epoch_val = epoch+1
+                    if best_acc_val > self.save_threshold:
+                        torch.save(self.model.state_dict(), self.save_param_path + "_val_epoch" + str(best_epoch_val) + "_{0:.4f}".format(best_acc_val))
+                        print ("saved " + self.save_param_path + "_val_epoch" + str(best_epoch_val) + "_{0:.4f}".format(best_acc_val) )
+                    else:
+                        if epoch-best_epoch_val >= self.epoch_stop-1:
+                            is_earlystop = True
+                            print ("early stopping...")
+                
+        time_elapsed = time.time() - since
+        print('Training complete in {:.0f}m {:.0f}s'.format(
+            time_elapsed // 60, time_elapsed % 60))
+        print("Best model on val: epoch" + str(best_epoch_val) + "_" + str(best_acc_val))
+
+        if self.mode == "eann":
+            print("Event: Best model on val: epoch" + str(best_epoch_val_event) + "_" + str(best_acc_val_event))
+
+    
+        self.model.load_state_dict(best_model_wts_val)
+
+        print ("test result when using best model on val")
+        return self.test()   
+
+
+
+    def test(self):
+        since = time.time()
+
+        self.model.cuda()
+        self.model.eval()   
+
+        pred = []
+        label = []
+
+        if self.mode == "eann":
+            pred_event = []
+            label_event = []
+
+        for batch in tqdm(self.dataloaders['test']):
+            with torch.no_grad(): 
+                batch_data=batch
+                for k,v in batch_data.items():
+                    batch_data[k]=v.cuda()
+                batch_label = batch_data['label']
+
+                if self.mode == "eann":
+                    batch_label_event = batch_data['label_event']
+                    batch_outputs, batch_outputs_event, fea = self.model(**batch_data)
+                    _, batch_preds_event = torch.max(batch_outputs_event, 1)
+
+                    label_event.extend(batch_label_event.detach().cpu().numpy().tolist())
+                    pred_event.extend(batch_preds_event.detach().cpu().numpy().tolist())
+                else: 
+                    batch_outputs,fea = self.model(**batch_data) 
+
+                _, batch_preds = torch.max(batch_outputs, 1)
+
+                label.extend(batch_label.detach().cpu().numpy().tolist())
+                pred.extend(batch_preds.detach().cpu().numpy().tolist())
+
+
+        print (get_confusionmatrix_fnd(np.array(pred), np.array(label)))
+        print (metrics(label, pred))
+
+        if self.mode == "eann" and self.model_name != "FANVM":
+            print ("event:")
+            print (accuracy_score(np.array(label_event), np.array(pred_event)))
+
+        return metrics(label, pred)
+    

FakeVD/code_test/models/coattention.py

@@ -0,0 +1,122 @@
+
+import torch.nn as nn
+
+from .trm import *
+
+
+class _MultiHeadAttention(nn.Module):
+    def __init__(self, d_k, d_v, d_model, n_heads, dropout):
+        super(_MultiHeadAttention, self).__init__()
+        self.d_k = d_k
+        self.d_v = d_v
+        self.d_model = d_model
+        self.n_heads = n_heads
+
+        self.w_q = Linear(d_model, d_k * n_heads)
+        self.w_k = Linear(d_model, d_k * n_heads)
+        self.w_v = Linear(d_model, d_v * n_heads)
+
+    def forward(self, q, k, v):
+        # q: [b_size x len_q x d_model]
+        # k: [b_size x len_k x d_model]
+        # v: [b_size x len_k x d_model]
+        b_size = q.size(0)
+
+        # q_s: [b_size x n_heads x len_q x d_k]
+        # k_s: [b_size x n_heads x len_k x d_k]
+        # v_s: [b_size x n_heads x len_k x d_v]
+        q_s = self.w_q(q).view(b_size, -1, self.n_heads, self.d_k).transpose(1, 2)
+        k_s = self.w_k(k).view(b_size, -1, self.n_heads, self.d_k).transpose(1, 2)
+        v_s = self.w_v(v).view(b_size, -1, self.n_heads, self.d_v).transpose(1, 2)
+        return q_s, k_s, v_s
+
+class PoswiseFeedForwardNet(nn.Module):
+    def __init__(self, d_model, d_ff, dropout=0.1):
+        super(PoswiseFeedForwardNet, self).__init__()
+        self.relu = nn.ReLU()
+        self.conv1 = nn.Conv1d(in_channels=d_model, out_channels=d_ff, kernel_size=1)
+        self.conv2 = nn.Conv1d(in_channels=d_ff, out_channels=d_model, kernel_size=1)
+        self.dropout = nn.Dropout(dropout)
+        self.layer_norm = LayerNormalization(d_model)
+
+    def forward(self, inputs):
+        # inputs: [b_size x len_q x d_model]
+        residual = inputs
+        output = self.relu(self.conv1(inputs.transpose(1, 2)))
+
+        # outputs: [b_size x len_q x d_model]
+        output = self.conv2(output).transpose(1, 2)
+        output = self.dropout(output)
+
+        return self.layer_norm(residual + output)
+
+class MultiHeadAttention(nn.Module):
+    def __init__(self, d_k, d_v, n_heads, dropout, d_model, visual_len, sen_len, fea_v, fea_s, pos):
+        super(MultiHeadAttention, self).__init__()
+        self.n_heads = n_heads
+        self.multihead_attn_v = _MultiHeadAttention(d_k, d_v, d_model, n_heads, dropout)
+        self.multihead_attn_s = _MultiHeadAttention(d_k, d_v, d_model, n_heads, dropout)
+        self.pos_emb_v = PosEncoding(visual_len * 10, d_model)
+        self.pos_emb_s = PosEncoding(sen_len * 10, d_model)
+        self.linear_v = nn.Linear(in_features=fea_v, out_features=d_model)
+        self.linear_s = nn.Linear(in_features=fea_s, out_features=d_model)
+        self.proj_v = Linear(n_heads * d_v, d_model)
+        self.proj_s = Linear(n_heads * d_v, d_model)
+        self.d_v = d_v
+        self.dropout = nn.Dropout(dropout)
+        self.layer_norm_v = LayerNormalization(d_model)
+        self.layer_norm_s = LayerNormalization(d_model)
+        self.attention = ScaledDotProductAttention(d_k, dropout)
+        self.pos = pos
+
+    def forward(self, v, s, v_len, s_len):
+        b_size = v.size(0)
+        # q: [b_size x len_q x d_model]
+        # k: [b_size x len_k x d_model]
+        # v: [b_size x len_v x d_model] note (len_k == len_v)
+        v, s = self.linear_v(v), self.linear_s(s)
+        if self.pos:
+            pos_v, pos_s = self.pos_emb_v(v_len), self.pos_emb_s(s_len)
+            residual_v, residual_s = v + pos_v, s + pos_s
+        else:
+            residual_v, residual_s = v, s
+        # context: a tensor of shape [b_size x len_q x n_heads * d_v]
+        q_v, k_v, v_v = self.multihead_attn_v(v, v, v)
+        q_s, k_s, v_s = self.multihead_attn_s(s, s, s)
+        context_v, attn_v = self.attention(q_v, k_s, v_s)
+        context_s, attn_s = self.attention(q_s, k_v, v_v)
+        context_v = context_v.transpose(1, 2).contiguous().view(b_size, -1, self.n_heads * self.d_v)
+        context_s = context_s.transpose(1, 2).contiguous().view(b_size, -1, self.n_heads * self.d_v)
+        # project back to the residual size, outputs: [b_size x len_q x d_model]
+        output_v = self.dropout(self.proj_v(context_v))
+        output_s = self.dropout(self.proj_s(context_s))
+        return self.layer_norm_v(residual_v + output_v), self.layer_norm_s(residual_s + output_s)
+
+class co_attention(nn.Module):
+    def __init__(self, d_k, d_v, n_heads, dropout, d_model, visual_len, sen_len, fea_v, fea_s, pos):
+        super(co_attention, self).__init__()
+        # self.layer_num = layer_num
+        # self.multi_head = MultiHeadAttention(d_k=d_k, d_v=d_v, n_heads=n_heads, dropout=dropout, d_model=d_model,
+        #                                      visual_len=visual_len, sen_len=sen_len, fea_v=fea_v, fea_s=fea_s, pos=False)
+        # self.PoswiseFeedForwardNet_v = nn.ModuleList([PoswiseFeedForwardNet(d_model=d_model, d_ff=256)])
+        # self.PoswiseFeedForwardNet_s = nn.ModuleList([PoswiseFeedForwardNet(d_model=d_model, d_ff=256)])
+        # self.multi_head = nn.ModuleList([MultiHeadAttention(d_k=d_k, d_v=d_v, n_heads=n_heads, dropout=dropout, d_model=d_model,
+        #                                      visual_len=visual_len, sen_len=sen_len, fea_v=fea_v, fea_s=fea_s, pos=False)])
+        # for i in range(1, layer_num):
+        #     self.PoswiseFeedForwardNet_v.append(PoswiseFeedForwardNet(d_model=d_model, d_ff=256))
+        #     self.PoswiseFeedForwardNet_s.append(PoswiseFeedForwardNet(d_model=d_model, d_ff=256))
+        #     self.multi_head.append(MultiHeadAttention(d_k=d_k, d_v=d_v, n_heads=n_heads, dropout=dropout, d_model=d_model,
+        #                                      visual_len=visual_len, sen_len=sen_len, fea_v=d_model, fea_s=d_model, pos=True))
+        self.multi_head = MultiHeadAttention(d_k=d_k, d_v=d_v, n_heads=n_heads, dropout=dropout, d_model=d_model,
+                                             visual_len=visual_len, sen_len=sen_len, fea_v=fea_v, fea_s=fea_s, pos=pos)
+        self.PoswiseFeedForwardNet_v = PoswiseFeedForwardNet(d_model=d_model, d_ff=128, dropout=dropout)
+        self.PoswiseFeedForwardNet_s = PoswiseFeedForwardNet(d_model=d_model, d_ff=128,dropout=dropout)
+    def forward(self, v, s, v_len, s_len):
+        # for i in range(self.layer_num):
+        #     v, s = self.multi_head[i](v, s, v_len, s_len)
+        #     v = self.PoswiseFeedForwardNet_v[i](v)
+        #     s = self.PoswiseFeedForwardNet_s[i](s)
+        v, s = self.multi_head(v, s, v_len, s_len)
+        v = self.PoswiseFeedForwardNet_v(v)
+        s = self.PoswiseFeedForwardNet_s(s)
+        return v, s

FakeVD/code_test/models/layers.py

@@ -0,0 +1,54 @@
+import math
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from einops import rearrange
+from torch.autograd import Function
+
+
+class ReverseLayerF(Function):
+    @staticmethod
+    def forward(ctx, input_, alpha):
+        ctx.alpha = alpha
+        return input_
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        output = grad_output.neg() * ctx.alpha
+        return output, None
+
+
+class Attention(nn.Module):
+    def __init__(self, dim, heads = 2, dim_head = 64, dropout = 0.):
+        super().__init__()
+        inner_dim = dim_head *  heads
+        project_out = not (heads == 1 and dim_head == dim)
+
+        self.heads = heads
+        self.scale = dim_head ** -0.5
+
+        self.attend = nn.Softmax(dim = -1)
+        self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)
+
+        self.to_out = nn.Sequential(
+            nn.Linear(inner_dim, dim),
+            nn.Dropout(dropout)
+        ) if project_out else nn.Identity()
+
+    def forward(self, x):
+        qkv = self.to_qkv(x).chunk(3, dim = -1)
+        q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = self.heads), qkv)
+
+        dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale
+
+        attn = self.attend(dots)
+
+        out = torch.matmul(attn, v)
+        out = rearrange(out, 'b h n d -> b n (h d)')
+        return self.to_out(out)
+
+
+
+

FakeVD/code_test/models/trm.py

@@ -0,0 +1,80 @@
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.init as init
+
+
+class Linear(nn.Module):
+    def __init__(self, in_features, out_features, bias=True):
+        super(Linear, self).__init__()
+        self.linear = nn.Linear(in_features, out_features, bias=bias)
+        init.xavier_normal_(self.linear.weight)
+        init.zeros_(self.linear.bias)
+
+    def forward(self, inputs):
+        return self.linear(inputs)
+
+
+class ScaledDotProductAttention(nn.Module):
+    def __init__(self, d_k, dropout=.1):
+        super(ScaledDotProductAttention, self).__init__()
+        self.scale_factor = np.sqrt(d_k)
+        self.softmax = nn.Softmax(dim=-1)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, q, k, v, attn_mask=None):
+        # q: [b_size x n_heads x len_q x d_k]
+        # k: [b_size x n_heads x len_k x d_k]
+        # v: [b_size x n_heads x len_v x d_v] note: (len_k == len_v)
+
+        # attn: [b_size x n_heads x len_q x len_k]
+        scores = torch.matmul(q, k.transpose(-1, -2)) / self.scale_factor
+        if attn_mask is not None:
+            assert attn_mask.size() == scores.size()
+            scores.masked_fill_(attn_mask, -1e9)
+        attn = self.dropout(self.softmax(scores))
+
+        # outputs: [b_size x n_heads x len_q x d_v]
+        context = torch.matmul(attn, v)
+
+        return context, attn
+
+
+class LayerNormalization(nn.Module):
+    def __init__(self, d_hid, eps=1e-6):
+        super(LayerNormalization, self).__init__()
+        self.gamma = nn.Parameter(torch.ones(d_hid))
+        self.beta = nn.Parameter(torch.zeros(d_hid))
+        self.eps = eps
+
+    def forward(self, z):
+        mean = z.mean(dim=-1, keepdim=True,)
+        std = z.std(dim=-1, keepdim=True,)
+        ln_out = (z - mean) / (std + self.eps)
+        ln_out = self.gamma * ln_out + self.beta
+
+        return ln_out
+
+
+class PosEncoding(nn.Module):
+    def __init__(self, max_seq_len, d_word_vec):
+        super(PosEncoding, self).__init__()
+        pos_enc = np.array(
+            [[pos / np.power(10000, 2.0 * (j // 2) / d_word_vec) for j in range(d_word_vec)]
+            for pos in range(max_seq_len)])
+        pos_enc[:, 0::2] = np.sin(pos_enc[:, 0::2])
+        pos_enc[:, 1::2] = np.cos(pos_enc[:, 1::2])
+        pad_row = np.zeros([1, d_word_vec])
+        pos_enc = np.concatenate([pad_row, pos_enc]).astype(np.float32)
+
+        # additional single row for PAD idx
+        self.pos_enc = nn.Embedding(max_seq_len + 1, d_word_vec)
+        # fix positional encoding: exclude weight from grad computation
+        self.pos_enc.weight = nn.Parameter(torch.from_numpy(pos_enc), requires_grad=False)
+        self.max_len = int(max_seq_len/10)
+    def forward(self, input_len):
+        max_len = self.max_len            # torch.max(input_len)
+        tensor = torch.cuda.LongTensor if input_len.is_cuda else torch.LongTensor
+        input_pos = tensor([list(range(1, len+1)) + [0]*(max_len-len) for len in input_len])
+        return self.pos_enc(input_pos)

FakeVD/code_test/predict.py

@@ -0,0 +1,162 @@
+import os
+import torch
+import torch.nn.functional as F
+from torch.utils.data import DataLoader
+import numpy as np
+
+from tqdm import tqdm
+from FakeVD.code_test.utils.metrics import *
+
+from FakeVD.code_test.models.SVFEND import SVFENDModel
+from FakeVD.code_test.utils.dataloader import SVFENDDataset
+from FakeVD.code_test.run import _init_fn, SVFEND_collate_fn
+
+# from VGGish_Feature_Extractor.my_vggish_folder_fun import vggish_audio
+from FakeVD.code_test.VGGish_Feature_Extractor.my_vggish_fun import vggish_audio, load_model_vggish
+from FakeVD.code_test.VGG19_Feature_Extractor.vgg19_feature import process_video as vgg19_frame
+from FakeVD.code_test.VGG19_Feature_Extractor.vgg19_feature import load_model_vgg19
+from FakeVD.code_test.C3D_Feature_Extractor.feature_extractor_vid import feature_extractor as c3d_video
+from FakeVD.code_test.C3D_Feature_Extractor.feature_extractor_vid import load_model_c3d
+from FakeVD.code_test.Text_Feature_Extractor.main import video_work as asr_text
+from FakeVD.code_test.Text_Feature_Extractor.wav2text import wav2text
+
+def load_model(checkpoint_path):
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model = SVFENDModel(bert_model='bert-base-chinese', fea_dim=128,dropout=0.1)
+    # model.load_state_dict(torch.load(checkpoint_path))
+    model.load_state_dict(torch.load(checkpoint_path, map_location=device), False)
+    model.eval()
+    return model
+
+def get_model(checkpoint_path='./FakeVD/code_test/checkpoints/SVFEND/SVFEND/_test_epoch4_0.7943'):
+    # 加载检测模型  模型存放路径 checkpoint_path
+    model_main = load_model(checkpoint_path)
+    model_vggish = load_model_vggish()
+    model_vgg19 = load_model_vgg19()
+    model_c3d = load_model_c3d()
+    model_text = wav2text()
+
+    models = {
+    'model_main': model_main,
+    'model_vggish': model_vggish,
+    'model_vgg19': model_vgg19,
+    'model_c3d' : model_c3d,
+    'model_text' : model_text
+    }
+
+    return models
+
+
+
+# label = 0 if item['annotation']=='真' else 1
+def test(model, dataloader):
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model.to(device)
+    # model.cuda()
+    model.eval()
+
+    pred = []
+    label = []
+    prob = []
+
+    for batch in tqdm(dataloader):
+        with torch.no_grad():
+            batch_data = batch
+            for k, v in batch_data.items():
+                batch_data[k] = v.to(device)
+            batch_label = batch_data['label']
+
+            batch_outputs, fea = model(**batch_data)
+
+            _, batch_preds = torch.max(batch_outputs, 1)
+
+            softmax_probs = F.softmax(batch_outputs, dim=1)  # 计算softmax概率
+
+            label.extend(batch_label.detach().cpu().numpy().tolist())
+            pred.extend(batch_preds.detach().cpu().numpy().tolist())
+            prob.extend(softmax_probs.detach().cpu().numpy().tolist())  # 收集softmax概率
+
+    return (label, pred, prob)
+
+def main(models,
+         video_file_path, 
+         preprocessed_flag=False, 
+         feature_path='./FakeVD/code_test/preprocessed_feature'):
+    # 视频是否已经过预处理 preprocessed_flag
+    # 特征存放目录 feature_path
+    
+    # 获取模型
+    model_main = models['model_main']
+    model_vggish = models['model_vggish']
+    model_vgg19 = models['model_vgg19']
+    model_c3d = models['model_c3d']
+    model_text = models['model_text']
+
+    # 获取视频文件夹路径
+    video_folder_path = os.path.dirname(video_file_path)
+
+    # 获取视频文件名（包含扩展名）
+    video_file_name = os.path.basename(video_file_path)
+
+    # 提取视频文件名（不包括扩展名）作为视频ID
+    vids = []
+    vid = os.path.splitext(video_file_name)[0]
+    vids.append(vid)
+    # video_file_name = os.path.basename(video_file_path)
+    # vids.append(os.path.splitext(video_file_name)[0])
+    # # vids.append(video_file_name.split('_')[1].split('.')[0]
+    
+    # VGGish_audio特征目录
+    VGGish_audio_feature_path = os.path.join(feature_path, vid+'.pkl')
+    # C3D_video特征目录
+    C3D_video_feature_path = os.path.join(feature_path, 'C3D/')
+    # VGG19_frame特征目录
+    VGG19_frame_feature_path = os.path.join(feature_path, 'VGG19/')
+    # ASR_text特征目录
+    asr_text_feature_path = os.path.join(feature_path, 'ASR/'+vid+'.json')
+
+    # 特征提取
+    if not preprocessed_flag:
+        vggish_audio(model_vggish, video_file_path, VGGish_audio_feature_path)
+        vgg19_frame(model_vgg19, video_file_name, video_folder_path, VGG19_frame_feature_path)
+        c3d_video(model_c3d, C3D_video_feature_path, video_folder_path, video_file_name)
+        asr_text(model_text, model_vggish, video_file_path, asr_text_feature_path)
+
+    # 数据路径
+    data = vids
+    data_paths = {
+        'VGGish_audio'  :   VGGish_audio_feature_path,
+        'C3D_video'     :   C3D_video_feature_path,
+        'VGG19_frame'   :   VGG19_frame_feature_path,
+        'ASR_text'      :   asr_text_feature_path
+    }
+
+    # 创建Dataset和DataLoader
+    dataset = SVFENDDataset(data, data_paths)
+
+    dataloader=DataLoader(dataset, batch_size=1,
+            num_workers=0,
+            pin_memory=True,
+            shuffle=False,
+            worker_init_fn=_init_fn,
+            collate_fn=SVFEND_collate_fn)
+
+    # 进行预测
+    predictions = test(model_main, dataloader)
+    annotation = '真' if predictions[1][0]==0 else '假'
+    prob_softmax = predictions[2]
+    # annotation_prob = max(prob_softmax[0])
+    annotation_prob = prob_softmax[0][0]#真的概率
+    annotation_prob1 = prob_softmax[0][1]#假的概率
+    # 打印预测结果
+    print(annotation, annotation_prob, annotation_prob1)
+    
+    return annotation_prob1
+
+
+if __name__ == "__main__":
+    # 视频是否已经过预处理
+    preprocessed_flag = False
+    video_file_path = "./FakeVD/dataset/videos_1/douyin_6700861687563570439.mp4"
+    models = get_model()
+    main(models, video_file_path, preprocessed_flag)

FakeVD/code_test/run.py

@@ -0,0 +1,500 @@
+
+import collections
+import json
+import os
+import time
+
+import torch
+from torch.utils.data import DataLoader
+from torch.utils.tensorboard import SummaryWriter
+# from gensim.models import KeyedVectors 
+
+from FakeVD.code_test.models.Baselines import *
+from FakeVD.code_test.models.FANVM import FANVMModel
+from FakeVD.code_test.models.SVFEND import SVFENDModel
+from FakeVD.code_test.models.TikTec import TikTecModel
+
+from FakeVD.code_test.utils.dataloader import *
+from FakeVD.code_test.models.Trainer import Trainer
+from FakeVD.code_test.models.Trainer_3set import Trainer3
+
+
+def pad_sequence(seq_len,lst, emb):
+    result=[]
+    for video in lst:
+        if isinstance(video, list):
+            video = torch.stack(video)
+        ori_len=video.shape[0]
+        if ori_len == 0:
+            video = torch.zeros([seq_len,emb],dtype=torch.long)
+        elif ori_len>=seq_len:
+            if emb == 200:
+                video=torch.FloatTensor(video[:seq_len])
+            else:
+                video=torch.LongTensor(video[:seq_len])
+        else:
+            video=torch.cat([video,torch.zeros([seq_len-ori_len,video.shape[1]],dtype=torch.long)],dim=0)
+            if emb == 200:
+                video=torch.FloatTensor(video)
+            else:
+                video=torch.LongTensor(video)
+        result.append(video)
+    return torch.stack(result)
+
+def pad_sequence_bbox(seq_len,lst):
+    result=[]
+    for video in lst: 
+        if isinstance(video, list):
+            video = torch.stack(video)
+        ori_len=video.shape[0]
+        if ori_len == 0:
+            video = torch.zeros([seq_len,45,4096],dtype=torch.float)
+        elif ori_len>=seq_len:
+            video=torch.FloatTensor(video[:seq_len])
+        else:
+            video=torch.cat([video,torch.zeros([seq_len-ori_len,45,4096],dtype=torch.float)],dim=0)
+        result.append(video)
+    return torch.stack(result)
+
+def pad_frame_sequence(seq_len,lst):
+    attention_masks = []
+    result=[]
+    for video in lst:
+        video=torch.FloatTensor(video)
+        ori_len=video.shape[0]
+        if ori_len>=seq_len:
+            gap=ori_len//seq_len
+            video=video[::gap][:seq_len]
+            mask = np.ones((seq_len))
+        else:
+            video=torch.cat((video,torch.zeros([seq_len-ori_len,video.shape[1]],dtype=torch.float)),dim=0)
+            mask = np.append(np.ones(ori_len), np.zeros(seq_len-ori_len))
+        result.append(video)
+        mask = torch.IntTensor(mask)
+        attention_masks.append(mask)
+    return torch.stack(result), torch.stack(attention_masks)
+
+
+def _init_fn(worker_id):
+    np.random.seed(2022)
+
+def SVFEND_collate_fn(batch): 
+    num_frames = 83
+    num_audioframes = 50 
+
+    title_inputid = [item['title_inputid'] for item in batch]
+    title_mask = [item['title_mask'] for item in batch]
+
+    frames = [item['frames'] for item in batch]
+    frames, frames_masks = pad_frame_sequence(num_frames, frames)
+
+    audioframes  = [item['audioframes'] for item in batch]
+    audioframes, audioframes_masks = pad_frame_sequence(num_audioframes, audioframes)
+
+    c3d  = [item['c3d'] for item in batch]
+    c3d, c3d_masks = pad_frame_sequence(num_frames, c3d)
+
+    label = [item['label'] for item in batch]
+
+    return {
+        'label': torch.stack(label),
+        'title_inputid': torch.stack(title_inputid),
+        'title_mask': torch.stack(title_mask),
+        'audioframes': audioframes,
+        'audioframes_masks': audioframes_masks,
+        'frames':frames,
+        'frames_masks': frames_masks,
+        'c3d': c3d,
+        'c3d_masks': c3d_masks,
+    }
+
+def FANVM_collate_fn(batch): 
+    num_comments = 23 
+    num_frames = 83
+
+    title_inputid = [item['title_inputid'] for item in batch]
+    title_mask = [item['title_mask'] for item in batch]
+
+    comments_like = [item['comments_like'] for item in batch]
+    comments_inputid = [item['comments_inputid'] for item in batch]
+    comments_mask = [item['comments_mask'] for item in batch]
+
+    comments_inputid_resorted = [] 
+    comments_mask_resorted = []
+    comments_like_resorted = []
+
+    for idx in range(len(comments_like)):
+        comments_like_one = comments_like[idx]
+        comments_inputid_one = comments_inputid[idx]
+        comments_mask_one = comments_mask[idx]
+        if comments_like_one.shape != torch.Size([0]):
+            comments_inputid_one, comments_mask_one, comments_like_one = (list(t) for t in zip(*sorted(zip(comments_inputid_one, comments_mask_one, comments_like_one), key=lambda s: s[2], reverse=True)))
+        comments_inputid_resorted.append(comments_inputid_one)
+        comments_mask_resorted.append(comments_mask_one)
+        comments_like_resorted.append(comments_like_one)
+    
+    comments_inputid = pad_sequence(num_comments,comments_inputid_resorted,250)
+    comments_mask = pad_sequence(num_comments,comments_mask_resorted,250)
+    comments_like=[]
+    for idx in range(len(comments_like_resorted)):
+        comments_like_resorted_one = comments_like_resorted[idx]
+        if len(comments_like_resorted_one)>=num_comments:
+            comments_like.append(torch.tensor(comments_like_resorted_one[:num_comments]))
+        else:
+            if isinstance(comments_like_resorted_one, list):
+                comments_like.append(torch.tensor(comments_like_resorted_one+[0]*(num_comments-len(comments_like_resorted_one))))
+            else:
+                comments_like.append(torch.tensor(comments_like_resorted_one.tolist()+[0]*(num_comments-len(comments_like_resorted_one))))
+
+    frames = [item['frames'] for item in batch]
+    frames, frames_masks = pad_frame_sequence(num_frames, frames)
+    frame_thmub = [item['frame_thmub'] for item in batch]
+
+    label = [item['label'] for item in batch]
+    label_event = [item['label_event'] for item in batch]
+    s = [item['s'] for item in batch]
+
+    return {
+        'label': torch.stack(label),
+        'title_inputid': torch.stack(title_inputid),
+        'title_mask': torch.stack(title_mask),
+        'comments_inputid': comments_inputid,
+        'comments_mask': comments_mask,
+        'comments_like': torch.stack(comments_like),
+        'frames':frames,
+        'frames_masks': frames_masks,
+        'frame_thmub': torch.stack(frame_thmub),
+        's': torch.stack(s),
+        'label_event':torch.stack(label_event),
+    }
+
+def bbox_collate_fn(batch): 
+    num_frames = 83
+
+    bbox_vgg = [item['bbox_vgg'] for item in batch] 
+    bbox_vgg = pad_sequence_bbox(num_frames,bbox_vgg) 
+
+    label = [item['label'] for item in batch]
+
+    return {
+        'label': torch.stack(label),
+        'bbox_vgg': bbox_vgg,
+    }
+
+def c3d_collate_fn(batch):
+    num_frames = 83
+
+    c3d  = [item['c3d'] for item in batch]
+    c3d, c3d_masks = pad_frame_sequence(num_frames, c3d)
+
+    label = [item['label'] for item in batch]
+
+    return {
+        'label': torch.stack(label),
+        'c3d': c3d,
+        'c3d_masks': c3d_masks,
+    }
+
+def vgg_collate_fn(batch):
+    num_frames = 83
+
+    frames = [item['frames'] for item in batch]
+    frames, frames_masks = pad_frame_sequence(num_frames, frames)
+
+    label = [item['label'] for item in batch]
+
+    return {
+        'label': torch.stack(label),
+        'frames':frames,
+        'frames_masks': frames_masks,
+    }
+
+def comments_collate_fn(batch): 
+    num_comments = 23 
+
+    comments_like = [item['comments_like'] for item in batch]
+    comments_inputid = [item['comments_inputid'] for item in batch]
+    comments_mask = [item['comments_mask'] for item in batch]
+
+    comments_inputid_resorted = [] 
+    comments_mask_resorted = []
+    comments_like_resorted = []
+
+    for idx in range(len(comments_like)):
+        comments_like_one = comments_like[idx]
+        comments_inputid_one = comments_inputid[idx]
+        comments_mask_one = comments_mask[idx]
+        if comments_like_one.shape != torch.Size([0]):
+            comments_inputid_one, comments_mask_one, comments_like_one = (list(t) for t in zip(*sorted(zip(comments_inputid_one, comments_mask_one, comments_like_one), key=lambda s: s[2], reverse=True)))
+        comments_inputid_resorted.append(comments_inputid_one)
+        comments_mask_resorted.append(comments_mask_one)
+        comments_like_resorted.append(comments_like_one)
+    
+    comments_inputid = pad_sequence(num_comments,comments_inputid_resorted,250)
+    comments_mask = pad_sequence(num_comments,comments_mask_resorted,250)
+    comments_like=[]
+    for idx in range(len(comments_like_resorted)):
+        comments_like_resorted_one = comments_like_resorted[idx]
+        if len(comments_like_resorted_one)>=num_comments:
+            comments_like.append(torch.tensor(comments_like_resorted_one[:num_comments]))
+        else:
+            if isinstance(comments_like_resorted_one, list):
+                comments_like.append(torch.tensor(comments_like_resorted_one+[0]*(num_comments-len(comments_like_resorted_one))))
+            else:
+                comments_like.append(torch.tensor(comments_like_resorted_one.tolist()+[0]*(num_comments-len(comments_like_resorted_one))))
+
+    label = [item['label'] for item in batch]
+
+    return {
+        'label': torch.stack(label),
+        'comments_inputid': comments_inputid,
+        'comments_mask': comments_mask,
+        'comments_like': torch.stack(comments_like),
+    }
+
+def title_w2v_collate_fn(batch):
+    length_title = 128
+    title_w2v = [item['title_w2v'] for item in batch]
+    title_w2v = pad_sequence(length_title, title_w2v, 100)
+
+    label = [item['label'] for item in batch]
+
+    return {
+        'label': torch.stack(label),
+        'title_w2v': title_w2v,
+    }
+
+def tictec_collate_fn(batch):
+    """
+    将一批样本组合成一个批次。
+
+    Args:
+    batch (list of dict): 包含单个样本的列表，每个样本是一个字典，包含 'label'、'caption_feature'、'visual_feature'、'asr_feature'、'mask_K' 和 'mask_N'。
+
+    Returns:
+    dict: 包含批次数据的字典，'labels' 是一个张量，其他特征和掩码也是张量。
+    """
+    num_frames = 83
+
+
+    labels = torch.stack([item['label'] for item in batch])
+    caption_features = torch.stack([item['caption_feature'] for item in batch])
+    visual_features = torch.stack([item['visual_feature'] for item in batch])
+    asr_features = torch.stack([item['asr_feature'] for item in batch])
+    mask_Ks = torch.stack([item['mask_K'] for item in batch])
+    mask_Ns = torch.stack([item['mask_N'] for item in batch])
+
+    return {
+        'label': labels,
+        'caption_feature': caption_features,
+        'visual_feature': visual_features,
+        'asr_feature': asr_features,
+        'mask_K': mask_Ks,
+        'mask_N': mask_Ns,
+    }
+
+
+class Run():
+    def __init__(self,
+                 config
+                 ):
+
+        self.model_name = config['model_name']
+        self.mode_eval = config['mode_eval']
+        self.fold = config['fold']
+        self.data_type = 'SVFEND'
+
+        self.epoches = config['epoches']
+        self.batch_size = config['batch_size']
+        self.num_workers = config['num_workers']
+        self.epoch_stop = config['epoch_stop']
+        self.seed = config['seed']
+        self.device = config['device']
+        self.lr = config['lr']
+        self.lambd=config['lambd']
+        self.save_param_dir = config['path_param']
+        self.path_tensorboard = config['path_tensorboard']
+        self.dropout = config['dropout']
+        self.weight_decay = config['weight_decay']
+        self.event_num = 616 
+        self.mode ='normal'
+    
+
+    def get_dataloader(self,data_type,data_fold):
+        collate_fn=None
+
+        if data_type=='SVFEND':
+            dataset_train = SVFENDDataset(f'vid_fold_{1}.txt')
+            dataset_test = SVFENDDataset(f'vid_fold_{2}.txt')
+            collate_fn=SVFEND_collate_fn
+        elif data_type=='FANVM':
+            dataset_train = FANVMDataset_train(f'vid_fold_no_{data_fold}.txt')
+            dataset_test = FANVMDataset_test(path_vid_train=f'vid_fold_no_{data_fold}.txt', path_vid_test=f'vid_fold_{data_fold}.txt')
+            collate_fn = FANVM_collate_fn
+        elif data_type=='c3d':
+            dataset_train = C3DDataset(f'vid_fold_no_{data_fold}.txt')
+            dataset_test = C3DDataset(f'vid_fold_{data_fold}.txt')
+            collate_fn = c3d_collate_fn
+        elif data_type=='vgg':
+            dataset_train = VGGDataset(f'vid_fold_no_{data_fold}.txt')
+            dataset_test = VGGDataset(f'vid_fold_{data_fold}.txt')
+            collate_fn = vgg_collate_fn
+        elif data_type=='bbox':
+            dataset_train = BboxDataset('vid_fold_no1.txt')
+            dataset_test = BboxDataset('vid_fold_1.txt')
+            collate_fn = bbox_collate_fn
+        elif data_type=='comments':
+            dataset_train = CommentsDataset(f'vid_fold_no_{data_fold}.txt')
+            dataset_test = CommentsDataset(f'vid_fold_{data_fold}.txt')
+            collate_fn = comments_collate_fn
+        elif data_type=='TikTec':
+            dataset_train = TikTecDataset(f'vid_fold_no_{data_fold}.txt')
+            dataset_test = TikTecDataset(f'vid_fold_{data_fold}.txt')
+            collate_fn = tictec_collate_fn    
+        # elif data_type=='w2v':
+        #     wv_from_text = KeyedVectors.load_word2vec_format("./stores/tencent-ailab-embedding-zh-d100-v0.2.0-s/tencent-ailab-embedding-zh-d100-v0.2.0-s.txt", binary=False)
+        #     dataset_train = Title_W2V_Dataset(f'vid_fold_no{data_fold}.txt', wv_from_text)
+        #     dataset_test = Title_W2V_Dataset(f'vid_fold_{data_fold}.txt', wv_from_text)
+        #     collate_fn = title_w2v_collate_fn
+
+        train_dataloader = DataLoader(dataset_train, batch_size=self.batch_size,
+            num_workers=self.num_workers,
+            pin_memory=True,
+            shuffle=True,
+            worker_init_fn=_init_fn,
+            collate_fn=collate_fn)
+
+        test_dataloader=DataLoader(dataset_test, batch_size=self.batch_size,
+            num_workers=self.num_workers,
+            pin_memory=True,
+            shuffle=False,
+            worker_init_fn=_init_fn,
+            collate_fn=collate_fn)
+
+        dataloaders =  dict(zip(['train', 'test'],[train_dataloader, test_dataloader]))
+
+        return dataloaders
+
+
+    def get_dataloader_temporal(self, data_type):
+        collate_fn=None
+        if data_type=='SVFEND':
+            dataset_train = SVFENDDataset('vid_time3_train.txt')
+            dataset_val = SVFENDDataset('vid_time3_val.txt')
+            dataset_test = SVFENDDataset('vid_time3_test.txt')
+            collate_fn=SVFEND_collate_fn
+        elif data_type=='FANVM':
+            dataset_train = FANVMDataset_train('vid_time3_train.txt')
+            dataset_val = FANVMDataset_test(path_vid_train='vid_time3_train.txt', path_vid_test='vid_time3_valid.txt')
+            dataset_test = FANVMDataset_test(path_vid_train='vid_time3_train.txt', path_vid_test='vid_time3_test.txt')
+            collate_fn = FANVM_collate_fn
+        else:
+            # can be added
+            print ("Not available")
+
+        train_dataloader = DataLoader(dataset_train, batch_size=self.batch_size,
+            num_workers=self.num_workers,
+            pin_memory=True,
+            shuffle=True,
+            worker_init_fn=_init_fn,
+            collate_fn=collate_fn)
+        val_dataloader = DataLoader(dataset_val, batch_size=self.batch_size,
+            num_workers=self.num_workers,
+            pin_memory=True,
+            shuffle=False,
+            worker_init_fn=_init_fn,
+            collate_fn=collate_fn)
+        test_dataloader=DataLoader(dataset_test, batch_size=self.batch_size,
+            num_workers=self.num_workers,
+            pin_memory=True,
+            shuffle=False,
+            worker_init_fn=_init_fn,
+            collate_fn=collate_fn)
+ 
+        dataloaders =  dict(zip(['train', 'val', 'test'],[train_dataloader, val_dataloader, test_dataloader]))
+
+        return dataloaders
+
+
+    def get_model(self):
+        if self.model_name == 'SVFEND':
+            self.model = SVFENDModel(bert_model='bert-base-chinese', fea_dim=128,dropout=self.dropout)
+        elif self.model_name == 'FANVM':
+            self.model = FANVMModel(bert_model='bert-base-chinese', fea_dim=128)
+            self.data_type = "FANVM"
+            self.mode = 'eann'
+        elif self.model_name == 'C3D':
+            self.model = bC3D(fea_dim=128)
+            self.data_type = "c3d"
+        elif self.model_name == 'VGG':
+            self.model = bVGG(fea_dim=128)
+            self.data_type = "vgg"
+        elif self.model_name == 'Bbox':
+            self.model = bBbox(fea_dim=128)
+            self.data_type = "bbox"
+        elif self.model_name == 'Vggish':
+            self.model = bVggish(fea_dim=128)
+        elif self.model_name == 'Bert':
+            self.model = bBert(bert_model='bert-base-chinese', fea_dim=128,dropout=self.dropout)
+        elif self.model_name == 'TextCNN':
+            self.model = bTextCNN(fea_dim=128, vocab_size=100)
+            self.data_type = "w2v"
+        elif self.model_name == 'Comments':
+            self.model = bComments(bert_model='bert-base-chinese', fea_dim=128)
+            self.data_type = "comments"
+        elif self.model_name == 'TikTec':
+            self.model = TikTecModel(VCIF_dropout=self.dropout, MLP_dropout=self.dropout)
+            self.data_type = 'TikTec'
+
+        return self.model
+
+
+    def main(self):
+        if self.mode_eval == "nocv":
+            self.model = self.get_model()
+            dataloaders = self.get_dataloader(data_type=self.data_type, data_fold=self.fold)
+            trainer = Trainer(model=self.model, device = self.device, lr = self.lr, dataloaders = dataloaders, epoches = self.epoches, dropout = self.dropout, weight_decay = self.weight_decay, mode = self.mode, model_name = self.model_name, event_num = self.event_num, 
+                    epoch_stop = self.epoch_stop, save_param_path = self.save_param_dir+self.data_type+"/"+self.model_name+"/", writer = SummaryWriter(self.path_tensorboard))
+            result=trainer.train()
+            for metric in ['acc', 'f1', 'precision', 'recall', 'auc']:
+                print ('%s : %.4f' % (metric, result[metric]))
+
+        elif self.mode_eval == "temporal":
+            self.model = self.get_model()
+            dataloaders = self.get_dataloader_temporal(data_type=self.data_type)
+            trainer = Trainer3(model=self.model, device = self.device, lr = self.lr, dataloaders = dataloaders, epoches = self.epoches, dropout = self.dropout, weight_decay = self.weight_decay, mode = self.mode, model_name = self.model_name, event_num = self.event_num, 
+                    epoch_stop = self.epoch_stop, save_param_path = self.save_param_dir+self.data_type+"/"+self.model_name+"/", writer = SummaryWriter(self.path_tensorboard))
+            result=trainer.train()
+            for metric in ['acc', 'f1', 'precision', 'recall', 'auc']:
+                print ('%s : %.4f' % (metric, result[metric]))
+            return result
+        
+        elif self.mode_eval == "cv":
+            collate_fn=None
+            # if self.model_name == 'TextCNN':
+            #     wv_from_text = KeyedVectors.load_word2vec_format("./stores/tencent-ailab-embedding-zh-d100-v0.2.0-s/tencent-ailab-embedding-zh-d100-v0.2.0-s.txt", binary=False)
+
+            history = collections.defaultdict(list) 
+            for fold in range(1, 6): 
+                print('-' * 50)
+                print ('fold %d:' % fold)
+                print('-' * 50)
+                self.model = self.get_model()
+                dataloaders = self.get_dataloader(data_type=self.data_type, data_fold=fold)
+                trainer = Trainer(model = self.model, device = self.device, lr = self.lr, dataloaders = dataloaders, epoches = self.epoches, dropout = self.dropout, weight_decay = self.weight_decay, mode = self.mode, model_name = self.model_name, event_num = self.event_num, 
+                    epoch_stop = self.epoch_stop, save_param_path = self.save_param_dir+self.data_type+"/"+self.model_name+"/", writer = SummaryWriter(self.path_tensorboard+"fold_"+str(fold)+"/"))
+       
+                result = trainer.train()
+
+                history['auc'].append(result['auc'])
+                history['f1'].append(result['f1'])
+                history['recall'].append(result['recall'])
+                history['precision'].append(result['precision'])
+                history['acc'].append(result['acc'])
+                
+            print ('results on 5-fold cross-validation: ')
+            for metric in ['acc', 'f1', 'precision', 'recall', 'auc']:
+                print ('%s : %.4f +/- %.4f' % (metric, np.mean(history[metric]), np.std(history[metric])))
+        
+        else:
+            print ("Not Available")

FakeVD/code_test/test.py

@@ -0,0 +1,13 @@
+# cPickle是python2系列用的，3系列已经不用了，直接用pickle就好了
+import pickle
+
+# 重点是rb和r的区别，rb是打开2进制文件，文本文件用r
+f = open('/mnt/data10t/dazuoye/GROUP2024-GEN6/FakeSV/code_test/preprocessed_feature/douyin_6559701594739313923.pkl','rb')
+data = pickle.load(f)
+
+# 打印前五个键值对
+for i, (key, value) in enumerate(data.items()):
+    if i >= 2:
+        break
+    print(f"Key: {key}")
+    print(f"Value: {value}\n")