Kaggle competition: Deepfake Detection Challenge
<SOS> metric: LogLoss Kaggle data source: deepfake-detection-challenge<train_model>
%matplotlib inline
warnings.filterwarnings("ignore" )
history = model.fit(X_train, y_train,
validation_split=0.2,
verbose=1,
epochs=10 )<compute_test_metric>
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 frame_h = 5
frame_l = 5
len(test_videos )
final_predictions = model.predict(X_test)
 final_mse = mean_squared_error(y_test, final_predictions)
 final_rmse = np.sqrt(final_mse)
 print("RMSE on X_test ", round(final_rmse, 4))<predict_on_test>
print("PyTorch version:", torch.__version__)
 print("CUDA version:", torch.version.cuda)
 print("cuDNN version:", torch.backends.cudnn.version() )
final_predictions = model.predict(testNorm)
 final_predictions.shape<load_from_csv>
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 gpu
subm = pd.read_csv(input_path / 'sample_submission.csv')
 print(subm.shape)
 subm.head()<prepare_output>
facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False )
id_col=subm.id
subm=id_col.to_frame()
subm['target'] = final_predictions
subm.set_index('id',inplace=True)
 subm.head()<save_to_csv>
frames_per_video = 90
video_reader = VideoReader()
video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
 face_extractor = FaceExtractor(video_read_fn, facedet )
subm.to_csv('NNSubmission.csv' )<load_pretrained>
input_size = 250
weight_base64 = base64.b64encode(bz2.compress(pickle.dumps(torch.load('.. /input/alphageese-training/alphageese_epoch30.pth', map_location=torch.device('cpu')))))
 w = "weight= %s"%weight_base64
%store w >submission.py<set_options>
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize_transform = Normalize(mean, std )
%%writefile -a submission.py 
<set_options>
class MyResNeXt(models.resnet.ResNet):
def __init__(self, training=True):
super(MyResNeXt, self ).__init__(block=models.resnet.Bottleneck,
layers=[3, 4, 6, 3],
groups=32,
width_per_group=4)
 self.fc = nn.Linear(2048, 1 )
%%writefile -a submission.py
class MCTS() :
def __init__(self, game, nn_agent, eps=1e-8, cpuct=1.0):
self.game = game
self.nn_agent = nn_agent
self.eps = eps
self.cpuct = cpuct
self.Qsa = {}
self.Nsa = {}
self.Ns = {}
self.Ps = {}
self.Vs = {}
self.last_obs = None
def getActionProb(self, obs, timelimit=1.0):
start_time = time.time()
while time.time() - start_time < timelimit:
self.search(obs, self.last_obs)
 s = self.game.stringRepresentation(obs)
 i = obs.index
counts = [
self.Nsa[(s, i, a)] if(s, i, a)in self.Nsa else 0
for a in range(self.game.getActionSize())
 ]
prob = counts / np.sum(counts)
 self.last_obs = obs
return prob
def search(self, obs, last_obs):
s = self.game.stringRepresentation(obs)
 if s not in self.Ns:
values = [-10] * 4
for i in range(4):
if len(obs.geese[i])== 0:
continue
self.Ps[s, i], values[i] = self.nn_agent.predict(obs, last_obs, i)
 valids = self.game.getValidMoves(obs, last_obs, i)
 self.Ps[s, i] = self.Ps[s, i] * valids
sum_Ps_s = np.sum(self.Ps[s, i])
 if sum_Ps_s > 0:
self.Ps[s, i] /= sum_Ps_s
self.Vs[s, i] = valids
self.Ns[s] = 0
return values
best_acts = [None] * 4
for i in range(4):
if len(obs.geese[i])== 0:
continue
valids = self.Vs[s, i]
cur_best = -float('inf')
 best_act = self.game.actions[-1]
for a in range(self.game.getActionSize()):
if valids[a]:
if(s, i, a)in self.Qsa:
u = self.Qsa[(s, i, a)] + self.cpuct * self.Ps[s, i][a] * math.sqrt(
 self.Ns[s])/(1 + self.Nsa[(s, i, a)])
 else:
u = self.cpuct * self.Ps[s, i][a] * math.sqrt(
 self.Ns[s] + self.eps)
 if u > cur_best:
cur_best = u
best_act = self.game.actions[a]
best_acts[i] = best_act
next_obs = self.game.getNextState(obs, last_obs, best_acts)
 values = self.search(next_obs, obs)
 for i in range(4):
if len(obs.geese[i])== 0:
continue
a = self.game.actions.index(best_acts[i])
 v = values[i]
if(s, i, a)in self.Qsa:
self.Qsa[(s, i, a)] =(self.Nsa[(s, i, a)] * self.Qsa[ 
(s, i, a)] + v)/(self.Nsa[(s, i, a)] + 1)
 self.Nsa[(s, i, a)] += 1
else:
self.Qsa[(s, i, a)] = v
self.Nsa[(s, i, a)] = 1
self.Ns[s] += 1
return values<choose_model_class>
checkpoint = torch.load("/kaggle/input/deepfakes-inference-demo/resnext.pth", map_location=gpu)
 model = MyResNeXt().to(gpu)
 model.load_state_dict(checkpoint)
 _ = model.eval()
del checkpoint
%%writefile -a submission.py
class HungryGeese(object):
def __init__(self,
rows=7,
columns=11,
actions=[Action.NORTH, Action.SOUTH, Action.WEST, Action.EAST],
hunger_rate=40):
self.rows = rows
self.columns = columns
self.actions = actions
self.hunger_rate = hunger_rate
def getActionSize(self):
return len(self.actions)
 def getNextState(self, obs, last_obs, directions):
next_obs = pickle.loads(pickle.dumps(obs, -1))
next_obs.step += 1
geese = next_obs.geese
food = next_obs.food
for i in range(4):
goose = geese[i]
if len(goose)== 0:
continue
head = translate(goose[0], directions[i], self.columns, self.rows)
 if last_obs is not None and head == last_obs.geese[i][0]:
geese[i] = []
continue
if head in food:
food.remove(head)
 else:
goose.pop()
goose.insert(0, head)
 if next_obs.step % self.hunger_rate == 0:
if len(goose)> 0:
goose.pop()
goose_positions = histogram(
 position
for goose in geese
for position in goose 
)
 for i in range(4):
if len(geese[i])> 0:
head = geese[i][0]
if goose_positions[head] > 1:
geese[i] = []
return next_obs
def getValidMoves(self, obs, last_obs, index):
geese = obs.geese
pos = geese[index][0]
obstacles = {position for goose in geese for position in goose[:-1]}
if last_obs is not None: obstacles.add(last_obs.geese[index][0])
 valid_moves = [
translate(pos, action, self.columns, self.rows)not in obstacles
for action in self.actions
]
return valid_moves
def stringRepresentation(self, obs):
return str(obs.geese + obs.food )<train_on_grid>
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return np.percentile(y_pred[:n].to('cpu'), 52, interpolation="nearest")
 except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.481
%%writefile -a submission.py
class TorusConv2d(nn.Module):
def __init__(self, input_dim, output_dim, kernel_size, bn):
super().__init__()
self.edge_size =(kernel_size[0] // 2, kernel_size[1] // 2)
 self.conv = nn.Conv2d(input_dim, output_dim, kernel_size=kernel_size)
 self.bn = nn.BatchNorm2d(output_dim)if bn else None
def forward(self, x):
h = torch.cat([x[:,:,:,-self.edge_size[1]:], x, x[:,:,:,:self.edge_size[1]]], dim=3)
 h = torch.cat([h[:,:,-self.edge_size[0]:], h, h[:,:,:self.edge_size[0]]], dim=2)
 h = self.conv(h)
 h = self.bn(h)if self.bn is not None else h
return h
class GeeseNet(nn.Module):
def __init__(self):
super().__init__()
layers, filters = 12, 32
self.conv0 = TorusConv2d(17, filters,(3, 3), True)
 self.blocks = nn.ModuleList([TorusConv2d(filters, filters,(3, 3), True)for _ in range(layers)])
 self.head_p = nn.Linear(filters, 4, bias=False)
 self.head_v = nn.Linear(filters * 2, 1, bias=False)
 def forward(self, x):
h = F.relu_(self.conv0(x))
for block in self.blocks:
h = F.relu_(h + block(h))
h_head =(h * x[:,:1] ).view(h.size(0), h.size(1), -1 ).sum(-1)
 h_avg = h.view(h.size(0), h.size(1), -1 ).mean(-1)
 p = torch.softmax(self.head_p(h_head), 1)
 v = torch.tanh(self.head_v(torch.cat([h_head, h_avg], 1)))
 return p, v
class NNAgent() :
def __init__(self, state_dict):
self.model = GeeseNet()
self.model.load_state_dict(state_dict)
 self.model.eval()
def predict(self, obs, last_obs, index):
x = self._make_input(obs, last_obs, index)
 with torch.no_grad() :
xt = torch.from_numpy(x ).unsqueeze(0)
 p, v = self.model(xt)
 return p.squeeze(0 ).detach().numpy() , v.item()
def _make_input(self, obs, last_obs, index):
b = np.zeros(( 17, 7 * 11), dtype=np.float32)
 for p, pos_list in enumerate(obs.geese):
for pos in pos_list[:1]:
b[0 +(p - index)% 4, pos] = 1
for pos in pos_list[-1:]:
b[4 +(p - index)% 4, pos] = 1
for pos in pos_list:
b[8 +(p - index)% 4, pos] = 1
if last_obs is not None:
for p, pos_list in enumerate(last_obs.geese):
for pos in pos_list[:1]:
b[12 +(p - index)% 4, pos] = 1
for pos in obs.food:
b[16, pos] = 1
return b.reshape(-1, 7, 11 )<statistical_test>
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions )
%%writefile -a submission.py
game = HungryGeese()
state_dict = pickle.loads(bz2.decompress(base64.b64decode(weight)))
 agent = NNAgent(state_dict)
 mcts = MCTS(game, agent)
 def alphagoose_agent(obs, config):
timelimit = config.actTimeout+obs.remainingOverageTime/(config.episodeSteps-obs.step)if obs.step > 50 and obs.remainingOverageTime > 5 else 1
action = game.actions[np.argmax(
 mcts.getActionProb(obs, timelimit)) ]
return action.name<import_modules>
speed_test = True
<set_options>
if speed_test:
start_time = time.time()
speedtest_videos = test_videos[:5]
predictions = predict_on_video_set(speedtest_videos, num_workers=4)
 elapsed = time.time() - start_time
print("Elapsed %f sec.Average per video: %f sec." %(elapsed, elapsed / len(speedtest_videos)) )
seed = 50
os.environ['PYTHONHASHSEED'] = str(seed)
 random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.enabled = False<set_options>
predictions = predict_on_video_set(test_videos, num_workers=4 )
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )<load_from_csv>
submission_df_resnext = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df_resnext.to_csv("submission_resnext.csv", index=False )
data_path = '/kaggle/input/aerial-cactus-identification/'
labels = pd.read_csv(data_path + 'train.csv')
 submission = pd.read_csv(data_path + 'sample_submission.csv' )<load_pretrained>
!pip install.. /input/deepfake-xception-trained-model/pytorchcv-0.0.55-py2.py3-none-any.whl --quiet
with ZipFile(data_path + 'train.zip')as zipper:
zipper.extractall()
with ZipFile(data_path + 'test.zip')as zipper:
zipper.extractall()<split>
test_dir = "/kaggle/input/deepfake-detection-challenge/test_videos/"
test_videos = sorted([x for x in os.listdir(test_dir)if x[-4:] == ".mp4"])
 len(test_videos )
_, valid = train_test_split(labels,
test_size=0.1,
stratify=labels['has_cactus'],
random_state=50 )<normalization>
gpu = torch.device("cuda:0" if torch.cuda.is_available() else "cpu" )
class ImageDataset(Dataset):
def __init__(self, df, img_dir='./', transform=None):
super().__init__()
self.df = df
self.img_dir = img_dir
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
img_id = self.df.iloc[idx, 0]
img_path = self.img_dir + img_id
image = cv2.imread(img_path)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 label = self.df.iloc[idx, 1]
if self.transform is not None:
image = self.transform(image)
 return image, label<normalization>
facedet = BlazeFace().to(gpu)
 facedet.load_weights("/kaggle/input/blazeface-pytorch/blazeface.pth")
 facedet.load_anchors("/kaggle/input/blazeface-pytorch/anchors.npy")
 _ = facedet.train(False )
transform_train = transforms.Compose([transforms.ToPILImage() ,
transforms.Pad(32, padding_mode='symmetric'),
transforms.RandomHorizontalFlip() ,
transforms.RandomVerticalFlip() ,
transforms.RandomRotation(10),
transforms.ToTensor() ,
transforms.Normalize(( 0.485, 0.456, 0.406), 
(0.229, 0.224, 0.225)) ])
 transform_test= transforms.Compose([transforms.ToPILImage() ,
transforms.Pad(32, padding_mode='symmetric'),
transforms.ToTensor() ,
transforms.Normalize(( 0.485, 0.456, 0.406), 
(0.229, 0.224, 0.225)) ] )<create_dataframe>
frames_per_video = 83
video_reader = VideoReader()
video_read_fn = lambda x: video_reader.read_frames(x, num_frames=frames_per_video)
 face_extractor = FaceExtractor(video_read_fn, facedet )
dataset_train = ImageDataset(df=labels, img_dir='train/', transform=transform_train)
 dataset_valid = ImageDataset(df=valid, img_dir='train/', transform=transform_test )<load_pretrained>
input_size = 150
loader_train = DataLoader(dataset=dataset_train, batch_size=32, shuffle=True)
 loader_valid = DataLoader(dataset=dataset_valid, batch_size=32, shuffle=False )<import_modules>
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
normalize_transform = Normalize(mean, std )
class Model(nn.Module):
def __init__(self):
super().__init__()
self.layer1 = nn.Sequential(nn.Conv2d(in_channels=3, out_channels=32,
kernel_size=3, padding=2),
nn.BatchNorm2d(32),
nn.LeakyReLU() ,
nn.MaxPool2d(kernel_size=2))
self.layer2 = nn.Sequential(nn.Conv2d(in_channels=32, out_channels=64,
kernel_size=3, padding=2),
nn.BatchNorm2d(64),
nn.LeakyReLU() ,
nn.MaxPool2d(kernel_size=2))
self.layer3 = nn.Sequential(nn.Conv2d(in_channels=64, out_channels=128,
kernel_size=3, padding=2),
nn.BatchNorm2d(128),
nn.LeakyReLU() ,
nn.MaxPool2d(kernel_size=2))
self.layer4 = nn.Sequential(nn.Conv2d(in_channels=128, out_channels=256,
kernel_size=3, padding=2),
nn.BatchNorm2d(256),
nn.LeakyReLU() ,
nn.MaxPool2d(kernel_size=2))
self.layer5 = nn.Sequential(nn.Conv2d(in_channels=256, out_channels=512,
kernel_size=3, padding=2),
nn.BatchNorm2d(512),
nn.LeakyReLU() ,
nn.MaxPool2d(kernel_size=2))
self.avg_pool = nn.AvgPool2d(kernel_size=4)
 self.fc1 = nn.Linear(in_features=512 * 1 * 1, out_features=64)
 self.fc2 = nn.Linear(in_features=64, out_features=2)
 def forward(self, x):
x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 x = self.layer5(x)
 x = self.avg_pool(x)
 x = x.view(-1, 512 * 1 * 1)
 x = self.fc1(x)
 x = self.fc2(x)
 return x<train_model>
model = get_model("xception", pretrained=False)
 model = nn.Sequential(*list(model.children())[:-1])
 class Pooling(nn.Module):
def __init__(self):
super(Pooling, self ).__init__()
self.p1 = nn.AdaptiveAvgPool2d(( 1,1))
self.p2 = nn.AdaptiveMaxPool2d(( 1,1))
def forward(self, x):
x1 = self.p1(x)
 x2 = self.p2(x)
 return(x1+x2)* 0.5
model[0].final_block.pool = nn.Sequential(nn.AdaptiveAvgPool2d(( 1,1)))
 class Head(torch.nn.Module):
def __init__(self, in_f, out_f):
super(Head, self ).__init__()
self.f = nn.Flatten()
self.l = nn.Linear(in_f, 512)
 self.d = nn.Dropout(0.5)
 self.o = nn.Linear(512, out_f)
 self.b1 = nn.BatchNorm1d(in_f)
 self.b2 = nn.BatchNorm1d(512)
 self.r = nn.ReLU()
def forward(self, x):
x = self.f(x)
 x = self.b1(x)
 x = self.d(x)
 x = self.l(x)
 x = self.r(x)
 x = self.b2(x)
 x = self.d(x)
 out = self.o(x)
 return out
class FCN(torch.nn.Module):
def __init__(self, base, in_f):
super(FCN, self ).__init__()
self.base = base
self.h1 = Head(in_f, 1)
 def forward(self, x):
x = self.base(x)
 return self.h1(x)
 net = []
model = FCN(model, 2048)
 model = model.cuda()
model.load_state_dict(torch.load('.. /input/deepfake-xception-trained-model/model.pth'))
net.append(model )
model = Model().to(device )<choose_model_class>
def predict_on_video(video_path, batch_size):
try:
faces = face_extractor.process_video(video_path)
 face_extractor.keep_only_best_face(faces)
 if len(faces)> 0:
x = np.zeros(( batch_size, input_size, input_size, 3), dtype=np.uint8)
 n = 0
for frame_data in faces:
for face in frame_data["faces"]:
resized_face = isotropically_resize_image(face, input_size)
 resized_face = make_square_image(resized_face)
 if n < batch_size:
x[n] = resized_face
n += 1
else:
print("WARNING: have %d faces but batch size is %d" %(n, batch_size))
if n > 0:
x = torch.tensor(x, device=gpu ).float()
x = x.permute(( 0, 3, 1, 2))
for i in range(len(x)) :
x[i] = normalize_transform(x[i] / 255.)
with torch.no_grad() :
y_pred = model(x)
 y_pred = torch.sigmoid(y_pred.squeeze())
 return np.percentile(y_pred[:n].to('cpu'), 51, interpolation="nearest")
 except Exception as e:
print("Prediction error on video %s: %s" %(video_path, str(e)))
 return 0.481
criterion = nn.CrossEntropyLoss()<choose_model_class>
def predict_on_video_set(videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(os.path.join(test_dir, filename), batch_size=frames_per_video)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions )
optimizer = torch.optim.Adamax(model.parameters() , lr=0.00006 )<categorify>
speed_test = True
epochs = 70
for epoch in range(epochs):
epoch_loss = 0
for images, labels in loader_train:
images = images.to(device)
 labels = labels.to(device)
 optimizer.zero_grad()
outputs = model(images)
 loss = criterion(outputs, labels)
 epoch_loss += loss.item()
loss.backward()
optimizer.step()
print(f'에폭: [{epoch+1}/{epochs}], 손실값: {epoch_loss/len(loader_train):.4f}' )<load_pretrained>
if speed_test:
start_time = time.time()
speedtest_videos = test_videos[:5]
predictions = predict_on_video_set(speedtest_videos, num_workers=4)
 elapsed = time.time() - start_time
print("Elapsed %f sec.Average per video: %f sec." %(elapsed, elapsed / len(speedtest_videos)) )
dataset_test = ImageDataset(df=submission, img_dir='test/', transform=transform_test)
 loader_test = DataLoader(dataset=dataset_test, batch_size=32, shuffle=False)
 model.eval()
preds = []
with torch.no_grad() :
for images, _ in loader_test:
images = images.to(device)
 outputs = model(images)
 preds_part = torch.softmax(outputs.cpu() , dim=1)[:, 1].tolist()
preds.extend(preds_part )<save_to_csv>
%%time
model.eval()
predictions = predict_on_video_set(test_videos, num_workers=4 )
submission['has_cactus'] = preds
submission.to_csv('submission.csv', index=False )<set_options>
submission_df_xception = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df_xception.to_csv("submission_xception.csv", index=False )
shutil.rmtree('./train')
 shutil.rmtree('./test' )<set_options>
submission_df = pd.DataFrame({"filename": test_videos} )
warnings.filterwarnings('ignore' )<init_hyperparams>
r1 = 0.32
r2 = 0.68
total = r1 + r2
r11 = r1/total
r22 = r2/total
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df<predict_on_test>
submission_df["label"] = r22*submission_df_resnext["label"] + r11*submission_df_xception["label"]
<define_variables><EOS> 
submission_df.to_csv("submission.csv", index=False )