Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<define_search_model>
!pip install -q keras-bert keras-rectified-adam
!wget -q https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip
!unzip -o uncased_L-12_H-768_A-12.zip
class SeparableConv2d(nn.Module):
def __init__(self,in_channels,out_channels,kernel_size=1,stride=1,padding=0,dilation=1,bias=False):
super(SeparableConv2d,self ).__init__()
self.conv1 = nn.Conv2d(in_channels,in_channels,kernel_size,stride,padding,dilation,groups=in_channels,bias=bias)
 self.pointwise = nn.Conv2d(in_channels,out_channels,1,1,0,1,1,bias=bias)
 def forward(self,x):
x = self.conv1(x)
 x = self.pointwise(x)
 return x
class Block(nn.Module):
def __init__(self,in_filters,out_filters,reps,strides=1,start_with_relu=True,grow_first=True):
super(Block, self ).__init__()
if out_filters != in_filters or strides!=1:
self.skip = nn.Conv2d(in_filters,out_filters,1,stride=strides, bias=False)
 self.skipbn = nn.BatchNorm2d(out_filters)
 else:
self.skip=None
rep=[]
filters=in_filters
if grow_first:
rep.append(nn.ReLU(inplace=True))
rep.append(SeparableConv2d(in_filters,out_filters,3,stride=1,padding=1,bias=False))
rep.append(nn.BatchNorm2d(out_filters))
filters = out_filters
for i in range(reps-1):
rep.append(nn.ReLU(inplace=True))
rep.append(SeparableConv2d(filters,filters,3,stride=1,padding=1,bias=False))
rep.append(nn.BatchNorm2d(filters))
if not grow_first:
rep.append(nn.ReLU(inplace=True))
rep.append(SeparableConv2d(in_filters,out_filters,3,stride=1,padding=1,bias=False))
rep.append(nn.BatchNorm2d(out_filters))
if not start_with_relu:
rep = rep[1:]
else:
rep[0] = nn.ReLU(inplace=False)
 if strides != 1:
rep.append(nn.MaxPool2d(3,strides,1))
self.rep = nn.Sequential(*rep)
 def forward(self,inp):
x = self.rep(inp)
 if self.skip is not None:
skip = self.skip(inp)
 skip = self.skipbn(skip)
 else:
skip = inp
x+=skip
return x
class Xception(nn.Module):

 def __init__(self, num_classes=1000):

 super(Xception, self ).__init__()
self.num_classes = num_classes
self.conv1 = nn.Conv2d(3, 32, 3,2, 0, bias=False)
 self.bn1 = nn.BatchNorm2d(32)
 self.relu1 = nn.ReLU(inplace=True)
 self.conv2 = nn.Conv2d(32,64,3,bias=False)
 self.bn2 = nn.BatchNorm2d(64)
 self.relu2 = nn.ReLU(inplace=True)
 self.block1=Block(64,128,2,2,start_with_relu=False,grow_first=True)
 self.block2=Block(128,256,2,2,start_with_relu=True,grow_first=True)
 self.block3=Block(256,728,2,2,start_with_relu=True,grow_first=True)
 self.block4=Block(728,728,3,1,start_with_relu=True,grow_first=True)
 self.block5=Block(728,728,3,1,start_with_relu=True,grow_first=True)
 self.block6=Block(728,728,3,1,start_with_relu=True,grow_first=True)
 self.block7=Block(728,728,3,1,start_with_relu=True,grow_first=True)
 self.block8=Block(728,728,3,1,start_with_relu=True,grow_first=True)
 self.block9=Block(728,728,3,1,start_with_relu=True,grow_first=True)
 self.block10=Block(728,728,3,1,start_with_relu=True,grow_first=True)
 self.block11=Block(728,728,3,1,start_with_relu=True,grow_first=True)
 self.block12=Block(728,1024,2,2,start_with_relu=True,grow_first=False)
 self.conv3 = SeparableConv2d(1024,1536,3,1,1)
 self.bn3 = nn.BatchNorm2d(1536)
 self.relu3 = nn.ReLU(inplace=True)
 self.conv4 = SeparableConv2d(1536,2048,3,1,1)
 self.bn4 = nn.BatchNorm2d(2048)
 self.fc = nn.Linear(2048, num_classes)
 def features(self, input):
x = self.conv1(input)
 x = self.bn1(x)
 x = self.relu1(x)
 x = self.conv2(x)
 x = self.bn2(x)
 x = self.relu2(x)
 x = self.block1(x)
 x = self.block2(x)
 x = self.block3(x)
 x = self.block4(x)
 x = self.block5(x)
 x = self.block6(x)
 x = self.block7(x)
 x = self.block8(x)
 x = self.block9(x)
 x = self.block10(x)
 x = self.block11(x)
 x = self.block12(x)
 x = self.conv3(x)
 x = self.bn3(x)
 x = self.relu3(x)
 x = self.conv4(x)
 x = self.bn4(x)
 return x
def logits(self, features):
x = nn.ReLU(inplace=True )(features)
 x = F.adaptive_avg_pool2d(x,(1, 1))
x = x.view(x.size(0), -1)
 x = self.last_linear(x)
 return x
def forward(self, input):
x = self.features(input)
 x = self.logits(x)
 return x
def xception(num_classes=1000, pretrained='imagenet'):
model = Xception(num_classes=num_classes)
 if pretrained:
settings = pretrained_settings['xception'][pretrained]
assert num_classes == settings['num_classes'], \
"num_classes should be {}, but is {}".format(settings['num_classes'], num_classes)
 model = Xception(num_classes=num_classes)
 model.load_state_dict(model_zoo.load_url(settings['url']))
model.input_space = settings['input_space']
model.input_size = settings['input_size']
model.input_range = settings['input_range']
model.mean = settings['mean']
model.std = settings['std']
model.last_linear = model.fc
del model.fc
return model
net4 = Xception()
net4.last_linear = net4.fc
del net4.fc
net4.last_linear = nn.Linear(in_features=2048, out_features=1, bias=True)
 net4.load_state_dict(torch.load(CHECKPOINT4))
net4 = net4.to(device)
 net4.cuda()
net4.eval()
net10 = Xception()
net10.last_linear = net10.fc
del net10.fc
net10.last_linear = nn.Linear(in_features=2048, out_features=1, bias=True)
 net10.load_state_dict(torch.load(CHECKPOINT10))
net10 = net10.to(device)
 net10.cuda()
net10.eval()<normalization>
SEQ_LEN = 128
BATCH_SIZE = 1024
EPOCHS = 15
LR = 1e-4
class MaxPoolPad(nn.Module):
def __init__(self):
super(MaxPoolPad, self ).__init__()
self.pad = nn.ZeroPad2d(( 1, 0, 1, 0))
self.pool = nn.MaxPool2d(3, stride=2, padding=1)
 def forward(self, x):
x = self.pad(x)
 x = self.pool(x)
 x = x[:, :, 1:, 1:].contiguous()
return x
class AvgPoolPad(nn.Module):
def __init__(self, stride=2, padding=1):
super(AvgPoolPad, self ).__init__()
self.pad = nn.ZeroPad2d(( 1, 0, 1, 0))
self.pool = nn.AvgPool2d(3, stride=stride, padding=padding, count_include_pad=False)
 def forward(self, x):
x = self.pad(x)
 x = self.pool(x)
 x = x[:, :, 1:, 1:].contiguous()
return x
class SeparableConv2d(nn.Module):
def __init__(self, in_channels, out_channels, dw_kernel, dw_stride, dw_padding, bias=False):
super(SeparableConv2d, self ).__init__()
self.depthwise_conv2d = nn.Conv2d(in_channels, in_channels, dw_kernel,
stride=dw_stride,
padding=dw_padding,
bias=bias,
groups=in_channels)
 self.pointwise_conv2d = nn.Conv2d(in_channels, out_channels, 1, stride=1, bias=bias)
 def forward(self, x):
x = self.depthwise_conv2d(x)
 x = self.pointwise_conv2d(x)
 return x
class BranchSeparables(nn.Module):
def __init__(self, in_channels, out_channels, kernel_size, stride, padding, name=None, bias=False):
super(BranchSeparables, self ).__init__()
self.relu = nn.ReLU()
self.separable_1 = SeparableConv2d(in_channels, in_channels, kernel_size, stride, padding, bias=bias)
 self.bn_sep_1 = nn.BatchNorm2d(in_channels, eps=0.001, momentum=0.1, affine=True)
 self.relu1 = nn.ReLU()
self.separable_2 = SeparableConv2d(in_channels, out_channels, kernel_size, 1, padding, bias=bias)
 self.bn_sep_2 = nn.BatchNorm2d(out_channels, eps=0.001, momentum=0.1, affine=True)
 self.name = name
def forward(self, x):
x = self.relu(x)
 if self.name == 'specific':
x = nn.ZeroPad2d(( 1, 0, 1, 0))(x)
 x = self.separable_1(x)
 if self.name == 'specific':
x = x[:, :, 1:, 1:].contiguous()
x = self.bn_sep_1(x)
 x = self.relu1(x)
 x = self.separable_2(x)
 x = self.bn_sep_2(x)
 return x
class BranchSeparablesStem(nn.Module):
def __init__(self, in_channels, out_channels, kernel_size, stride, padding, bias=False):
super(BranchSeparablesStem, self ).__init__()
self.relu = nn.ReLU()
self.separable_1 = SeparableConv2d(in_channels, out_channels, kernel_size, stride, padding, bias=bias)
 self.bn_sep_1 = nn.BatchNorm2d(out_channels, eps=0.001, momentum=0.1, affine=True)
 self.relu1 = nn.ReLU()
self.separable_2 = SeparableConv2d(out_channels, out_channels, kernel_size, 1, padding, bias=bias)
 self.bn_sep_2 = nn.BatchNorm2d(out_channels, eps=0.001, momentum=0.1, affine=True)
 def forward(self, x):
x = self.relu(x)
 x = self.separable_1(x)
 x = self.bn_sep_1(x)
 x = self.relu1(x)
 x = self.separable_2(x)
 x = self.bn_sep_2(x)
 return x
class BranchSeparablesReduction(BranchSeparables):
def __init__(self, in_channels, out_channels, kernel_size, stride, padding, z_padding=1, bias=False):
BranchSeparables.__init__(self, in_channels, out_channels, kernel_size, stride, padding, bias)
 self.padding = nn.ZeroPad2d(( z_padding, 0, z_padding, 0))
def forward(self, x):
x = self.relu(x)
 x = self.padding(x)
 x = self.separable_1(x)
 x = x[:, :, 1:, 1:].contiguous()
x = self.bn_sep_1(x)
 x = self.relu1(x)
 x = self.separable_2(x)
 x = self.bn_sep_2(x)
 return x
class CellStem0(nn.Module):
def __init__(self, stem_filters, num_filters=42):
super(CellStem0, self ).__init__()
self.num_filters = num_filters
self.stem_filters = stem_filters
self.conv_1x1 = nn.Sequential()
self.conv_1x1.add_module('relu', nn.ReLU())
 self.conv_1x1.add_module('conv', nn.Conv2d(self.stem_filters, self.num_filters, 1, stride=1, bias=False))
self.conv_1x1.add_module('bn', nn.BatchNorm2d(self.num_filters, eps=0.001, momentum=0.1, affine=True))
self.comb_iter_0_left = BranchSeparables(self.num_filters, self.num_filters, 5, 2, 2)
 self.comb_iter_0_right = BranchSeparablesStem(self.stem_filters, self.num_filters, 7, 2, 3, bias=False)
 self.comb_iter_1_left = nn.MaxPool2d(3, stride=2, padding=1)
 self.comb_iter_1_right = BranchSeparablesStem(self.stem_filters, self.num_filters, 7, 2, 3, bias=False)
 self.comb_iter_2_left = nn.AvgPool2d(3, stride=2, padding=1, count_include_pad=False)
 self.comb_iter_2_right = BranchSeparablesStem(self.stem_filters, self.num_filters, 5, 2, 2, bias=False)
 self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
 self.comb_iter_4_left = BranchSeparables(self.num_filters, self.num_filters, 3, 1, 1, bias=False)
 self.comb_iter_4_right = nn.MaxPool2d(3, stride=2, padding=1)
 def forward(self, x):
x1 = self.conv_1x1(x)
 x_comb_iter_0_left = self.comb_iter_0_left(x1)
 x_comb_iter_0_right = self.comb_iter_0_right(x)
 x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right
x_comb_iter_1_left = self.comb_iter_1_left(x1)
 x_comb_iter_1_right = self.comb_iter_1_right(x)
 x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right
x_comb_iter_2_left = self.comb_iter_2_left(x1)
 x_comb_iter_2_right = self.comb_iter_2_right(x)
 x_comb_iter_2 = x_comb_iter_2_left + x_comb_iter_2_right
x_comb_iter_3_right = self.comb_iter_3_right(x_comb_iter_0)
 x_comb_iter_3 = x_comb_iter_3_right + x_comb_iter_1
x_comb_iter_4_left = self.comb_iter_4_left(x_comb_iter_0)
 x_comb_iter_4_right = self.comb_iter_4_right(x1)
 x_comb_iter_4 = x_comb_iter_4_left + x_comb_iter_4_right
x_out = torch.cat([x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)
 return x_out
class CellStem1(nn.Module):
def __init__(self, stem_filters, num_filters):
super(CellStem1, self ).__init__()
self.num_filters = num_filters
self.stem_filters = stem_filters
self.conv_1x1 = nn.Sequential()
self.conv_1x1.add_module('relu', nn.ReLU())
 self.conv_1x1.add_module('conv', nn.Conv2d(2*self.num_filters, self.num_filters, 1, stride=1, bias=False))
self.conv_1x1.add_module('bn', nn.BatchNorm2d(self.num_filters, eps=0.001, momentum=0.1, affine=True))
self.relu = nn.ReLU()
self.path_1 = nn.Sequential()
self.path_1.add_module('avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False))
self.path_1.add_module('conv', nn.Conv2d(self.stem_filters, self.num_filters//2, 1, stride=1, bias=False))
self.path_2 = nn.ModuleList()
self.path_2.add_module('pad', nn.ZeroPad2d(( 0, 1, 0, 1)))
 self.path_2.add_module('avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False))
self.path_2.add_module('conv', nn.Conv2d(self.stem_filters, self.num_filters//2, 1, stride=1, bias=False))
self.final_path_bn = nn.BatchNorm2d(self.num_filters, eps=0.001, momentum=0.1, affine=True)
 self.comb_iter_0_left = BranchSeparables(self.num_filters, self.num_filters, 5, 2, 2, name='specific', bias=False)
 self.comb_iter_0_right = BranchSeparables(self.num_filters, self.num_filters, 7, 2, 3, name='specific', bias=False)
 self.comb_iter_1_left = MaxPoolPad()
self.comb_iter_1_right = BranchSeparables(self.num_filters, self.num_filters, 7, 2, 3, name='specific', bias=False)
 self.comb_iter_2_left = AvgPoolPad()
self.comb_iter_2_right = BranchSeparables(self.num_filters, self.num_filters, 5, 2, 2, name='specific', bias=False)
 self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
 self.comb_iter_4_left = BranchSeparables(self.num_filters, self.num_filters, 3, 1, 1, name='specific', bias=False)
 self.comb_iter_4_right = MaxPoolPad()
def forward(self, x_conv0, x_stem_0):
x_left = self.conv_1x1(x_stem_0)
 x_relu = self.relu(x_conv0)
 x_path1 = self.path_1(x_relu)
 x_path2 = self.path_2.pad(x_relu)
 x_path2 = x_path2[:, :, 1:, 1:]
x_path2 = self.path_2.avgpool(x_path2)
 x_path2 = self.path_2.conv(x_path2)
 x_right = self.final_path_bn(torch.cat([x_path1, x_path2], 1))
x_comb_iter_0_left = self.comb_iter_0_left(x_left)
 x_comb_iter_0_right = self.comb_iter_0_right(x_right)
 x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right
x_comb_iter_1_left = self.comb_iter_1_left(x_left)
 x_comb_iter_1_right = self.comb_iter_1_right(x_right)
 x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right
x_comb_iter_2_left = self.comb_iter_2_left(x_left)
 x_comb_iter_2_right = self.comb_iter_2_right(x_right)
 x_comb_iter_2 = x_comb_iter_2_left + x_comb_iter_2_right
x_comb_iter_3_right = self.comb_iter_3_right(x_comb_iter_0)
 x_comb_iter_3 = x_comb_iter_3_right + x_comb_iter_1
x_comb_iter_4_left = self.comb_iter_4_left(x_comb_iter_0)
 x_comb_iter_4_right = self.comb_iter_4_right(x_left)
 x_comb_iter_4 = x_comb_iter_4_left + x_comb_iter_4_right
x_out = torch.cat([x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)
 return x_out
class FirstCell(nn.Module):
def __init__(self, in_channels_left, out_channels_left, in_channels_right, out_channels_right):
super(FirstCell, self ).__init__()
self.conv_1x1 = nn.Sequential()
self.conv_1x1.add_module('relu', nn.ReLU())
 self.conv_1x1.add_module('conv', nn.Conv2d(in_channels_right, out_channels_right, 1, stride=1, bias=False))
self.conv_1x1.add_module('bn', nn.BatchNorm2d(out_channels_right, eps=0.001, momentum=0.1, affine=True))
self.relu = nn.ReLU()
self.path_1 = nn.Sequential()
self.path_1.add_module('avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False))
self.path_1.add_module('conv', nn.Conv2d(in_channels_left, out_channels_left, 1, stride=1, bias=False))
self.path_2 = nn.ModuleList()
self.path_2.add_module('pad', nn.ZeroPad2d(( 0, 1, 0, 1)))
 self.path_2.add_module('avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False))
self.path_2.add_module('conv', nn.Conv2d(in_channels_left, out_channels_left, 1, stride=1, bias=False))
self.final_path_bn = nn.BatchNorm2d(out_channels_left * 2, eps=0.001, momentum=0.1, affine=True)
 self.comb_iter_0_left = BranchSeparables(out_channels_right, out_channels_right, 5, 1, 2, bias=False)
 self.comb_iter_0_right = BranchSeparables(out_channels_right, out_channels_right, 3, 1, 1, bias=False)
 self.comb_iter_1_left = BranchSeparables(out_channels_right, out_channels_right, 5, 1, 2, bias=False)
 self.comb_iter_1_right = BranchSeparables(out_channels_right, out_channels_right, 3, 1, 1, bias=False)
 self.comb_iter_2_left = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
 self.comb_iter_3_left = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
 self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
 self.comb_iter_4_left = BranchSeparables(out_channels_right, out_channels_right, 3, 1, 1, bias=False)
 def forward(self, x, x_prev):
x_relu = self.relu(x_prev)
 x_path1 = self.path_1(x_relu)
 x_path2 = self.path_2.pad(x_relu)
 x_path2 = x_path2[:, :, 1:, 1:]
x_path2 = self.path_2.avgpool(x_path2)
 x_path2 = self.path_2.conv(x_path2)
 x_left = self.final_path_bn(torch.cat([x_path1, x_path2], 1))
x_right = self.conv_1x1(x)
 x_comb_iter_0_left = self.comb_iter_0_left(x_right)
 x_comb_iter_0_right = self.comb_iter_0_right(x_left)
 x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right
x_comb_iter_1_left = self.comb_iter_1_left(x_left)
 x_comb_iter_1_right = self.comb_iter_1_right(x_left)
 x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right
x_comb_iter_2_left = self.comb_iter_2_left(x_right)
 x_comb_iter_2 = x_comb_iter_2_left + x_left
x_comb_iter_3_left = self.comb_iter_3_left(x_left)
 x_comb_iter_3_right = self.comb_iter_3_right(x_left)
 x_comb_iter_3 = x_comb_iter_3_left + x_comb_iter_3_right
x_comb_iter_4_left = self.comb_iter_4_left(x_right)
 x_comb_iter_4 = x_comb_iter_4_left + x_right
x_out = torch.cat([x_left, x_comb_iter_0, x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)
 return x_out
class NormalCell(nn.Module):
def __init__(self, in_channels_left, out_channels_left, in_channels_right, out_channels_right):
super(NormalCell, self ).__init__()
self.conv_prev_1x1 = nn.Sequential()
self.conv_prev_1x1.add_module('relu', nn.ReLU())
 self.conv_prev_1x1.add_module('conv', nn.Conv2d(in_channels_left, out_channels_left, 1, stride=1, bias=False))
self.conv_prev_1x1.add_module('bn', nn.BatchNorm2d(out_channels_left, eps=0.001, momentum=0.1, affine=True))
self.conv_1x1 = nn.Sequential()
self.conv_1x1.add_module('relu', nn.ReLU())
 self.conv_1x1.add_module('conv', nn.Conv2d(in_channels_right, out_channels_right, 1, stride=1, bias=False))
self.conv_1x1.add_module('bn', nn.BatchNorm2d(out_channels_right, eps=0.001, momentum=0.1, affine=True))
self.comb_iter_0_left = BranchSeparables(out_channels_right, out_channels_right, 5, 1, 2, bias=False)
 self.comb_iter_0_right = BranchSeparables(out_channels_left, out_channels_left, 3, 1, 1, bias=False)
 self.comb_iter_1_left = BranchSeparables(out_channels_left, out_channels_left, 5, 1, 2, bias=False)
 self.comb_iter_1_right = BranchSeparables(out_channels_left, out_channels_left, 3, 1, 1, bias=False)
 self.comb_iter_2_left = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
 self.comb_iter_3_left = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
 self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
 self.comb_iter_4_left = BranchSeparables(out_channels_right, out_channels_right, 3, 1, 1, bias=False)
 def forward(self, x, x_prev):
x_left = self.conv_prev_1x1(x_prev)
 x_right = self.conv_1x1(x)
 x_comb_iter_0_left = self.comb_iter_0_left(x_right)
 x_comb_iter_0_right = self.comb_iter_0_right(x_left)
 x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right
x_comb_iter_1_left = self.comb_iter_1_left(x_left)
 x_comb_iter_1_right = self.comb_iter_1_right(x_left)
 x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right
x_comb_iter_2_left = self.comb_iter_2_left(x_right)
 x_comb_iter_2 = x_comb_iter_2_left + x_left
x_comb_iter_3_left = self.comb_iter_3_left(x_left)
 x_comb_iter_3_right = self.comb_iter_3_right(x_left)
 x_comb_iter_3 = x_comb_iter_3_left + x_comb_iter_3_right
x_comb_iter_4_left = self.comb_iter_4_left(x_right)
 x_comb_iter_4 = x_comb_iter_4_left + x_right
x_out = torch.cat([x_left, x_comb_iter_0, x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)
 return x_out
class ReductionCell0(nn.Module):
def __init__(self, in_channels_left, out_channels_left, in_channels_right, out_channels_right):
super(ReductionCell0, self ).__init__()
self.conv_prev_1x1 = nn.Sequential()
self.conv_prev_1x1.add_module('relu', nn.ReLU())
 self.conv_prev_1x1.add_module('conv', nn.Conv2d(in_channels_left, out_channels_left, 1, stride=1, bias=False))
self.conv_prev_1x1.add_module('bn', nn.BatchNorm2d(out_channels_left, eps=0.001, momentum=0.1, affine=True))
self.conv_1x1 = nn.Sequential()
self.conv_1x1.add_module('relu', nn.ReLU())
 self.conv_1x1.add_module('conv', nn.Conv2d(in_channels_right, out_channels_right, 1, stride=1, bias=False))
self.conv_1x1.add_module('bn', nn.BatchNorm2d(out_channels_right, eps=0.001, momentum=0.1, affine=True))
self.comb_iter_0_left = BranchSeparablesReduction(out_channels_right, out_channels_right, 5, 2, 2, bias=False)
 self.comb_iter_0_right = BranchSeparablesReduction(out_channels_right, out_channels_right, 7, 2, 3, bias=False)
 self.comb_iter_1_left = MaxPoolPad()
self.comb_iter_1_right = BranchSeparablesReduction(out_channels_right, out_channels_right, 7, 2, 3, bias=False)
 self.comb_iter_2_left = AvgPoolPad()
self.comb_iter_2_right = BranchSeparablesReduction(out_channels_right, out_channels_right, 5, 2, 2, bias=False)
 self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
 self.comb_iter_4_left = BranchSeparablesReduction(out_channels_right, out_channels_right, 3, 1, 1, bias=False)
 self.comb_iter_4_right = MaxPoolPad()
def forward(self, x, x_prev):
x_left = self.conv_prev_1x1(x_prev)
 x_right = self.conv_1x1(x)
 x_comb_iter_0_left = self.comb_iter_0_left(x_right)
 x_comb_iter_0_right = self.comb_iter_0_right(x_left)
 x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right
x_comb_iter_1_left = self.comb_iter_1_left(x_right)
 x_comb_iter_1_right = self.comb_iter_1_right(x_left)
 x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right
x_comb_iter_2_left = self.comb_iter_2_left(x_right)
 x_comb_iter_2_right = self.comb_iter_2_right(x_left)
 x_comb_iter_2 = x_comb_iter_2_left + x_comb_iter_2_right
x_comb_iter_3_right = self.comb_iter_3_right(x_comb_iter_0)
 x_comb_iter_3 = x_comb_iter_3_right + x_comb_iter_1
x_comb_iter_4_left = self.comb_iter_4_left(x_comb_iter_0)
 x_comb_iter_4_right = self.comb_iter_4_right(x_right)
 x_comb_iter_4 = x_comb_iter_4_left + x_comb_iter_4_right
x_out = torch.cat([x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)
 return x_out
class ReductionCell1(nn.Module):
def __init__(self, in_channels_left, out_channels_left, in_channels_right, out_channels_right):
super(ReductionCell1, self ).__init__()
self.conv_prev_1x1 = nn.Sequential()
self.conv_prev_1x1.add_module('relu', nn.ReLU())
 self.conv_prev_1x1.add_module('conv', nn.Conv2d(in_channels_left, out_channels_left, 1, stride=1, bias=False))
self.conv_prev_1x1.add_module('bn', nn.BatchNorm2d(out_channels_left, eps=0.001, momentum=0.1, affine=True))
self.conv_1x1 = nn.Sequential()
self.conv_1x1.add_module('relu', nn.ReLU())
 self.conv_1x1.add_module('conv', nn.Conv2d(in_channels_right, out_channels_right, 1, stride=1, bias=False))
self.conv_1x1.add_module('bn', nn.BatchNorm2d(out_channels_right, eps=0.001, momentum=0.1, affine=True))
self.comb_iter_0_left = BranchSeparables(out_channels_right, out_channels_right, 5, 2, 2, name='specific', bias=False)
 self.comb_iter_0_right = BranchSeparables(out_channels_right, out_channels_right, 7, 2, 3, name='specific', bias=False)
 self.comb_iter_1_left = MaxPoolPad()
self.comb_iter_1_right = BranchSeparables(out_channels_right, out_channels_right, 7, 2, 3, name='specific', bias=False)
 self.comb_iter_2_left = AvgPoolPad()
self.comb_iter_2_right = BranchSeparables(out_channels_right, out_channels_right, 5, 2, 2, name='specific', bias=False)
 self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)
 self.comb_iter_4_left = BranchSeparables(out_channels_right, out_channels_right, 3, 1, 1, name='specific', bias=False)
 self.comb_iter_4_right =MaxPoolPad()
def forward(self, x, x_prev):
x_left = self.conv_prev_1x1(x_prev)
 x_right = self.conv_1x1(x)
 x_comb_iter_0_left = self.comb_iter_0_left(x_right)
 x_comb_iter_0_right = self.comb_iter_0_right(x_left)
 x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right
x_comb_iter_1_left = self.comb_iter_1_left(x_right)
 x_comb_iter_1_right = self.comb_iter_1_right(x_left)
 x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right
x_comb_iter_2_left = self.comb_iter_2_left(x_right)
 x_comb_iter_2_right = self.comb_iter_2_right(x_left)
 x_comb_iter_2 = x_comb_iter_2_left + x_comb_iter_2_right
x_comb_iter_3_right = self.comb_iter_3_right(x_comb_iter_0)
 x_comb_iter_3 = x_comb_iter_3_right + x_comb_iter_1
x_comb_iter_4_left = self.comb_iter_4_left(x_comb_iter_0)
 x_comb_iter_4_right = self.comb_iter_4_right(x_right)
 x_comb_iter_4 = x_comb_iter_4_left + x_comb_iter_4_right
x_out = torch.cat([x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)
 return x_out
class NASNetAMobile(nn.Module):

 def __init__(self, num_classes=1000, stem_filters=32, penultimate_filters=1056, filters_multiplier=2):
super(NASNetAMobile, self ).__init__()
self.num_classes = num_classes
self.stem_filters = stem_filters
self.penultimate_filters = penultimate_filters
self.filters_multiplier = filters_multiplier
filters = self.penultimate_filters // 24
self.conv0 = nn.Sequential()
self.conv0.add_module('conv', nn.Conv2d(in_channels=3, out_channels=self.stem_filters, kernel_size=3, padding=0, stride=2,
bias=False))
self.conv0.add_module('bn', nn.BatchNorm2d(self.stem_filters, eps=0.001, momentum=0.1, affine=True))
self.cell_stem_0 = CellStem0(self.stem_filters, num_filters=filters //(filters_multiplier ** 2))
self.cell_stem_1 = CellStem1(self.stem_filters, num_filters=filters // filters_multiplier)
 self.cell_0 = FirstCell(in_channels_left=filters, out_channels_left=filters//2,
in_channels_right=2*filters, out_channels_right=filters)
 self.cell_1 = NormalCell(in_channels_left=2*filters, out_channels_left=filters,
in_channels_right=6*filters, out_channels_right=filters)
 self.cell_2 = NormalCell(in_channels_left=6*filters, out_channels_left=filters,
in_channels_right=6*filters, out_channels_right=filters)
 self.cell_3 = NormalCell(in_channels_left=6*filters, out_channels_left=filters,
in_channels_right=6*filters, out_channels_right=filters)
 self.reduction_cell_0 = ReductionCell0(in_channels_left=6*filters, out_channels_left=2*filters,
in_channels_right=6*filters, out_channels_right=2*filters)
 self.cell_6 = FirstCell(in_channels_left=6*filters, out_channels_left=filters,
in_channels_right=8*filters, out_channels_right=2*filters)
 self.cell_7 = NormalCell(in_channels_left=8*filters, out_channels_left=2*filters,
in_channels_right=12*filters, out_channels_right=2*filters)
 self.cell_8 = NormalCell(in_channels_left=12*filters, out_channels_left=2*filters,
in_channels_right=12*filters, out_channels_right=2*filters)
 self.cell_9 = NormalCell(in_channels_left=12*filters, out_channels_left=2*filters,
in_channels_right=12*filters, out_channels_right=2*filters)
 self.reduction_cell_1 = ReductionCell1(in_channels_left=12*filters, out_channels_left=4*filters,
in_channels_right=12*filters, out_channels_right=4*filters)
 self.cell_12 = FirstCell(in_channels_left=12*filters, out_channels_left=2*filters,
in_channels_right=16*filters, out_channels_right=4*filters)
 self.cell_13 = NormalCell(in_channels_left=16*filters, out_channels_left=4*filters,
in_channels_right=24*filters, out_channels_right=4*filters)
 self.cell_14 = NormalCell(in_channels_left=24*filters, out_channels_left=4*filters,
in_channels_right=24*filters, out_channels_right=4*filters)
 self.cell_15 = NormalCell(in_channels_left=24*filters, out_channels_left=4*filters,
in_channels_right=24*filters, out_channels_right=4*filters)
 self.relu = nn.ReLU()
self.avg_pool = nn.AvgPool2d(7, stride=1, padding=0)
 self.dropout = nn.Dropout()
self.last_linear = nn.Linear(24*filters, self.num_classes)
 def features(self, input):
x_conv0 = self.conv0(input)
 x_stem_0 = self.cell_stem_0(x_conv0)
 x_stem_1 = self.cell_stem_1(x_conv0, x_stem_0)
 x_cell_0 = self.cell_0(x_stem_1, x_stem_0)
 x_cell_1 = self.cell_1(x_cell_0, x_stem_1)
 x_cell_2 = self.cell_2(x_cell_1, x_cell_0)
 x_cell_3 = self.cell_3(x_cell_2, x_cell_1)
 x_reduction_cell_0 = self.reduction_cell_0(x_cell_3, x_cell_2)
 x_cell_6 = self.cell_6(x_reduction_cell_0, x_cell_3)
 x_cell_7 = self.cell_7(x_cell_6, x_reduction_cell_0)
 x_cell_8 = self.cell_8(x_cell_7, x_cell_6)
 x_cell_9 = self.cell_9(x_cell_8, x_cell_7)
 x_reduction_cell_1 = self.reduction_cell_1(x_cell_9, x_cell_8)
 x_cell_12 = self.cell_12(x_reduction_cell_1, x_cell_9)
 x_cell_13 = self.cell_13(x_cell_12, x_reduction_cell_1)
 x_cell_14 = self.cell_14(x_cell_13, x_cell_12)
 x_cell_15 = self.cell_15(x_cell_14, x_cell_13)
 return x_cell_15
def logits(self, features):
x = self.relu(features)
 x = self.avg_pool(x)
 x = x.view(x.size(0), -1)
 x = self.dropout(x)
 x = self.last_linear(x)
 return x
def forward(self, input):
x = self.features(input)
 x = self.logits(x)
 return x
net5 = NASNetAMobile()
net5.last_linear = nn.Linear(in_features=1056, out_features=1, bias=True)
 net5.load_state_dict(torch.load(CHECKPOINT5))
net5 = net5.to(device)
 net5.cuda()
net5.eval()
net9 = NASNetAMobile()
net9.last_linear = nn.Linear(in_features=1056, out_features=1, bias=True)
 net9.load_state_dict(torch.load(CHECKPOINT9))
net9 = net9.to(device)
 net9.cuda()
net9.eval()<choose_model_class>
pretrained_path = 'uncased_L-12_H-768_A-12'
config_path = os.path.join(pretrained_path, 'bert_config.json')
 checkpoint_path = os.path.join(pretrained_path, 'bert_model.ckpt')
 vocab_path = os.path.join(pretrained_path, 'vocab.txt')
 os.environ['TF_KERAS'] = '1'
class CFG:
seq_len=10
lstm_in = 16
lstm_out = 16
class LSTM_Model(nn.Module):
def __init__(self):
super(LSTM_Model, self ).__init__()
self.cnn_net = mobilenetv3(mode='small', pretrained=False)
 self.cnn_net.classifier[1] = nn.Linear(in_features=1280, out_features=1)
 self.cnn_net.classifier[1] = nn.Linear(in_features=1280, out_features=CFG.lstm_in)
 self.lstm = nn.LSTM(CFG.lstm_in, CFG.lstm_out, bidirectional=False, batch_first=True)
 self.reg_layer = nn.Sequential(nn.Dropout(0.5),
nn.Linear(CFG.lstm_out, 1))
def forward(self, x):
n_samples = x.shape[0]
x = x.view(-1, 3, 224, 224)
 x = self.cnn_net(x)
 x = x.view(n_samples, CFG.seq_len, -1)
 x,(h_n, h_c)= self.lstm(x)
 x = x[:,-1]
y = self.reg_layer(x)
 return y
net7 = LSTM_Model()
net7.load_state_dict(torch.load(CHECKPOINT7))
net7 = net7.to(device)
 net7.cuda()
net7.eval()
net8 = LSTM_Model()
net8.load_state_dict(torch.load(CHECKPOINT8))
net8 = net8.to(device)
 net8.cuda()
net8.eval()<choose_model_class>
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu )
<find_best_params>
token_dict = {}
with codecs.open(vocab_path, 'r', 'utf8')as reader:
for line in reader:
token = line.strip()
token_dict[token] = len(token_dict)
 with tpu_strategy.scope() :
model = load_trained_model_from_checkpoint(
 config_path,
checkpoint_path,
training=True,
trainable=True,
seq_len=SEQ_LEN,
)
net11 = torchvision.models.resnet18(pretrained=False)
 net11.fc = nn.Linear(in_features=512, out_features=1, bias=True)
 net11.load_state_dict(torch.load(CHECKPOINT11))
net11 = net11.to(device)
 net11.cuda()
net11.eval()<import_modules>
train_df = pd.read_csv('.. /input/nlp-getting-started/train.csv', index_col='id')
 test_df = pd.read_csv('.. /input/nlp-getting-started/test.csv', index_col='id' )
import albumentations
from albumentations.augmentations.transforms import ShiftScaleRotate, HorizontalFlip, RandomBrightnessContrast, MotionBlur, Blur, GaussNoise, JpegCompression 
<define_variables>
def strip_html(text):
soup = BeautifulSoup(text, "html.parser")
 return soup.get_text()
def remove_between_square_brackets(text):
return re.sub('\[[^]]*\]', '', text)
 def remove_url(text):
return re.sub(r'http\S+', '', text)
 def add_space(text):
return re.sub('%20', ' ', text)
 def remove_hashtags(text):
return re.sub('
def remove_at_signs(text):
return re.sub('@', '', text)
 def remove_punctuation(text):
return text.translate(str.maketrans('', '', string.punctuation))
def denoise_text(text):
text = strip_html(text)
 text = remove_between_square_brackets(text)
 text = add_space(text)
 text = remove_url(text)
 text = remove_hashtags(text)
 text = remove_at_signs(text)
 text = remove_punctuation(text)
 return text
def predict_on_video(model, model2, model3, model4, model5, model6, model7, model8, model9, model10, model11, video_path):
try:
x, x_sqr, x_299_sqr, x_lstm, frame_skip = extract_frames(video_path)
 if x is None or x_sqr is None: return 0.5
else:
with torch.no_grad() :
y_pred = model(x.to(device))
y_pred = torch.sigmoid(y_pred.squeeze().mean())
 y_pred2 = model2(x_sqr.to(device))
y_pred2 = torch.sigmoid(y_pred2.squeeze().mean())
 y_pred3 = model3(x_sqr.to(device))
y_pred3 = torch.sigmoid(y_pred3.squeeze().mean())
 y_pred4 = model4(x_299_sqr.to(device))
y_pred4 = torch.sigmoid(y_pred4.squeeze().mean())
 y_pred5 = model5(x.to(device))
y_pred5 = torch.sigmoid(y_pred5.squeeze().mean())
 y_pred6 = model6(x_sqr.to(device))
y_pred6 = torch.sigmoid(y_pred6.squeeze().mean())
 seq_len = len(x_lstm)//2
x_sqr_1, x_sqr_2 = x_lstm[:seq_len], x_lstm[seq_len:]
skip = max(1, seq_len//15)
 start_indices = list(range(0, seq_len, skip))
gap = max(1, int(np.round(20/frame_skip)))
 for ss, start in enumerate(start_indices):
indices = list(range(start, 500, gap))
indices = [i%seq_len for i in indices]
indices = indices[:CFG.seq_len]
seqs1, seqs2 = x_sqr_1[indices].unsqueeze(0), x_sqr_2[indices].unsqueeze(0)
 if ss == 0:
batch_seqs_1 = seqs1
batch_seqs_2 = seqs2
else:
batch_seqs_1 = torch.cat(( batch_seqs_1, seqs1), 0)
 batch_seqs_2 = torch.cat(( batch_seqs_2, seqs2), 0)
 batch_seqs = torch.cat(( batch_seqs_1, batch_seqs_2), 0)
 y_pred7 = model7(batch_seqs.to(device))
y_pred7 = torch.sigmoid(y_pred7.squeeze().mean())
 y_pred8 = model8(batch_seqs.to(device))
y_pred8 = torch.sigmoid(y_pred8.squeeze().mean())
 y_pred9 = model9(x_sqr.to(device))
y_pred9 = torch.sigmoid(y_pred9.squeeze().mean())
 y_pred10 = model10(x_299_sqr.to(device))
y_pred10 = torch.sigmoid(y_pred10.squeeze().mean())
 y_pred11 = model11(x.to(device))
y_pred11 = torch.sigmoid(y_pred11.squeeze().mean())
 w = [3, 2, 8, 2, 2, 2, 4, 4, 2, 2, 3]
return(w[0]*y_pred.item() + w[1]*y_pred2.item() + w[2]*y_pred3.item() + w[3]*y_pred4.item()
+ w[4]*y_pred5.item() + w[5]*y_pred6.item() +
w[6]*y_pred7.item() + w[7]*y_pred8.item() +
w[8]*y_pred9.item() + w[9]*y_pred10.item() + w[10]*y_pred11.item())/sum(w)
 except Exception as e:
print("Prediction error on video", e)
 return 0.5
return 0.5<categorify>
def preprocess_df(df):
df = df.fillna("")
 df['text'] = df['keyword'] + " " + df['text']
del df['keyword']
df['location'] = df['location'].astype('category')
 df['location'] = df['location'].cat.codes
df['text'] = df['text'].apply(denoise_text)
 return df
train_df = preprocess_df(train_df)
 test_df = preprocess_df(test_df )
class FastMTCNN(object):
def __init__(self, resize=1, *args, **kwargs):
self.resize = resize
self.mtcnn = MTCNN(*args, **kwargs)
 def __call__(self, frames):
if self.resize != 1:
frames = [f.resize([int(d * self.resize)for d in f.size])for f in frames]
boxes, probs = self.mtcnn.detect(frames)
 boxes = [b.astype(int ).tolist() if b is not None and type(b)==np.ndarray else [] for b in boxes]
probs = [b.tolist() if b is not None and type(b)==np.ndarray else [] for b in probs]
return boxes, probs
fast_mtcnn = FastMTCNN(
 resize=SCALE,
margin=14,
keep_all=True,
device=device
)<define_variables>
np.random.seed(1)
 msk = np.random.rand(len(train_df)) < 0.8
train_df, dev_df = train_df[msk], train_df[~msk]
test_videos = sorted([x for x in os.listdir(TEST_DIR)if x[-4:] == ".mp4"])
 len(test_videos )<predict_on_test>
tokenizer = Tokenizer(token_dict)
 def tokenize(df):
X, y = [], []
for i, index in enumerate(tqdm(df.index.values)) :
ids, segments = tokenizer.encode(df.text.values[i], max_len=SEQ_LEN)
 X.append(ids)
 try:
label = df.target.values[i]
y.append(label)
 except:
y.append(0)
 items = list(zip(X, y))
np.random.shuffle(items)
 indices, sentiments = zip(*items)
 indices = np.array(indices)
 mod = indices.shape[0] % BATCH_SIZE
if mod > 0:
indices, sentiments = indices[:-mod], sentiments[:-mod]
return [indices, np.zeros_like(indices)], np.array(sentiments)
 X_train, y_train = tokenize(train_df)
 X_dev, y_dev = tokenize(dev_df )
def predict_on_video_set(model, model2, model3, model4, model5, model6, model7, model8, model9, model10, model11, videos, num_workers):
def process_file(i):
filename = videos[i]
y_pred = predict_on_video(model, model2, model3, model4, model5, model6, model7, model8, model9, model10, model11, os.path.join(TEST_DIR, filename))
print(i, y_pred)
 return y_pred
with ThreadPoolExecutor(max_workers=num_workers)as ex:
predictions = ex.map(process_file, range(len(videos)))
 return list(predictions)
 predictions = predict_on_video_set(net, net2, net3, net4, net5, net6, net7, net8, net9, net10, net11, test_videos, num_workers=NUM_WORKERS )<save_to_csv>
X_test = []
for i, index in enumerate(tqdm(test_df.index.values)) :
ids, segments = tokenizer.encode(test_df.text.values[i], max_len=SEQ_LEN)
 X_test.append(ids)
 X_test = [np.array(X_test), np.zeros_like(X_test)]
predictions = np.clip(predictions, 0.005, 0.995)
 submission_df = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df.to_csv("submission.csv", index=False )<import_modules>
with tpu_strategy.scope() :
inputs = model.inputs[:2]
dense = model.get_layer('NSP-Dense' ).output
outputs = keras.layers.Dense(units=2, activation='softmax' )(dense)
 model = keras.models.Model(inputs, outputs)
 model.compile(
 RAdam(lr=LR),
loss='sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'],
)
import random
import re
from copy import deepcopy
from typing import Union, List, Tuple, Optional, Callable
from collections import OrderedDict, defaultdict
import math
import cv2
import torch
import torch.nn as nn
from torch.utils.data import Dataset,DataLoader
from torch.utils.data.sampler import SequentialSampler, RandomSampler
from torchvision import transforms, models
from torchvision.transforms import Normalize
from tqdm import tqdm
from sklearn.cluster import DBSCAN<define_variables>
learning_rate_reduction = keras.callbacks.ReduceLROnPlateau(monitor='val_sparse_categorical_accuracy', patience=2, verbose=1,factor=0.5, min_lr=1e-5 )
TARGET_H, TARGET_W = 224, 224
FRAMES_PER_VIDEO = 30
TEST_VIDEOS_PATH = '.. /input/deepfake-detection-challenge/test_videos'
NN_MODEL_PATHS = [
'.. /input/kdold-deepfake-effb2/fold0-effb2-000epoch.pt',
'.. /input/kdold-deepfake-effb2/fold0-effb2-001epoch.pt',
'.. /input/kdold-deepfake-effb2/fold0-effb2-002epoch.pt',
'.. /input/kfolddeepfakeeffb2-flip/fold0-flip-effb2-000epoch.pt',
'.. /input/kfolddeepfakeeffb2-flip/fold0-flip-effb2-001epoch.pt',
'.. /input/kfolddeepfakeeffb2-flip/fold0-flip-effb2-002epoch.pt',
'.. /input/kdold-deepfake-effb2/fold1-effb2-000epoch.pt',
'.. /input/kdold-deepfake-effb2/fold1-effb2-001epoch.pt',
'.. /input/kdold-deepfake-effb2/fold1-effb2-002epoch.pt',
'.. /input/kfolddeepfakeeffb2-flip/fold1-flip-effb2-000epoch.pt',
'.. /input/kfolddeepfakeeffb2-flip/fold1-flip-effb2-001epoch.pt',
'.. /input/kfolddeepfakeeffb2-flip/fold1-flip-effb2-002epoch.pt',
'.. /input/kdold-deepfake-effb2/fold2-effb2-000epoch.pt',
'.. /input/kdold-deepfake-effb2/fold2-effb2-001epoch.pt',
'.. /input/kdold-deepfake-effb2/fold2-effb2-002epoch.pt',
'.. /input/kfolddeepfakeeffb2-flip/fold2-flip-effb2-000epoch.pt',
'.. /input/kfolddeepfakeeffb2-flip/fold2-flip-effb2-001epoch.pt',
'.. /input/kfolddeepfakeeffb2-flip/fold2-flip-effb2-002epoch.pt',
'.. /input/kdold-deepfake-effb2/fold3-effb2-000epoch.pt',
'.. /input/kdold-deepfake-effb2/fold3-effb2-001epoch.pt',
'.. /input/kdold-deepfake-effb2/fold3-effb2-002epoch.pt',
'.. /input/kfolddeepfakeeffb2-flip/fold3-flip-effb2-000epoch.pt',
'.. /input/kfolddeepfakeeffb2-flip/fold3-flip-effb2-001epoch.pt',
'.. /input/kfolddeepfakeeffb2-flip/fold3-flip-effb2-002epoch.pt',
'.. /input/kdold-deepfake-effb2/fold4-effb2-000epoch.pt',
'.. /input/kdold-deepfake-effb2/fold4-effb2-001epoch.pt',
'.. /input/kdold-deepfake-effb2/fold4-effb2-002epoch.pt',
'.. /input/kfolddeepfakeeffb2-flip/fold4-flip-effb2-000epoch.pt',
'.. /input/kfolddeepfakeeffb2-flip/fold4-flip-effb2-001epoch.pt',
'.. /input/kfolddeepfakeeffb2-flip/fold4-flip-effb2-002epoch.pt',
] 
<set_options>
hist = model.fit(X_train, y_train, validation_data=(X_dev, y_dev), epochs=EPOCHS, batch_size=BATCH_SIZE, callbacks=[learning_rate_reduction] )
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = False
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<choose_model_class>
print("Accuracy of the model on Training Data is - {} %".format(model.evaluate(X_train,y_train)[1]*100))
print("Accuracy of the model on Dev Data is - {} %".format(model.evaluate(X_dev,y_dev)[1]*100))
!pip install.. /input/pytorchefficientnet/EfficientNet-PyTorch-master > /dev/null
def get_net() :
net = EfficientNet.from_name('efficientnet-b2')
 net._fc = nn.Linear(in_features=net._fc.in_features, out_features=2, bias=True)
 return net<feature_engineering>
classes = model.predict(X_test)[:, 0]
class DatasetRetriever(Dataset):
def __init__(self, df):
self.video_paths = df['video_path']
self.filenames = df.index
self.face_dr = FaceDetector(frames_per_video=FRAMES_PER_VIDEO)
 mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
self.normalize_transform = Normalize(mean, std)
 self.video_reader = VideoReader()
self.video_read_fn = lambda x: self.video_reader.read_frames(x, num_frames=FRAMES_PER_VIDEO)
 def __len__(self):
return self.filenames.shape[0]
def __getitem__(self, idx):
video_path = self.video_paths[idx]
filename = self.filenames[idx]
my_frames, my_idxs = self.video_read_fn(video_path)
 faces = self.face_dr.get_faces(
 my_frames, my_idxs,
0.7, 0.7, 0.7, 0.6 
)
 n = len(faces)
 video = torch.zeros(( n, 3, TARGET_H, TARGET_W))
for i, face in enumerate(faces[:n]):
face = 255 - face
face = face.astype(np.float32)/255.
 face = torch.tensor(face)
 face = face.permute(2,0,1)
 face = self.normalize_transform(face)
 video[i] = face
return filename, video<find_best_model_class>
submission = pd.DataFrame(
 {'id': list(test_df.index.values),
'target': list(( classes < 0.5 ).astype(int)) ,
} ).set_index('id' )
class DeepFakePredictor:
def __init__(self):
self.models = [self.prepare_model(get_net() , path)for path in NN_MODEL_PATHS]
self.models_count = len(self.models)
 def predict(self, dataset):
result = []
with torch.no_grad() :
for filename, video in dataset:
video = video.to(self.device, dtype=torch.float32)
 try:
label = self.predict_ensemble(video)
 except Exception as e:
print(f'Warning! {e}, {type(e)}')
 label = 0.5
result.append({
'filename': filename,
'label': label,
})
 return pd.DataFrame(result ).set_index('filename')
 def prepare_model(self, model, path):
self.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
 model.to(self.device);
if torch.cuda.is_available() :
model = model.cuda()
if torch.cuda.is_available() :
checkpoint = torch.load(path)
 else:
checkpoint = torch.load(path, map_location=torch.device('cpu'))
model.load_state_dict(checkpoint['model_state_dict'])
 model.eval()
print(f'Model prepared.Device is {self.device}')
 return model
@staticmethod
def net_forward(net, inputs):
bs = inputs.size(0)
 x = net.extract_features(inputs)
 x = net._avg_pooling(x)
 emb = x.view(bs, -1)
 x = net._dropout(emb)
 x = net._fc(x)
 return emb, x
def postprocess(self, embs, predictions):
clusters = defaultdict(list)
 for prediction, cluster_id in zip(predictions, DBSCAN(eps=1.2, min_samples=1 ).fit_predict(embs)) :
clusters[cluster_id].append(prediction)
 sorted_clusters = sorted(clusters.items() , key=lambda x: -len(x[1]))
if len(sorted_clusters)< 2:
return sorted_clusters[0][1]
if len(sorted_clusters[1][1])/ len(predictions)> 0.25:
return sorted_clusters[0][1] + sorted_clusters[1][1]
return sorted_clusters[0][1]
def predict_ensemble(self, video):
embs, predictions = 0, 0
for model in self.models:
emb, prediction = self.net_forward(model, video)
 predictions += prediction / self.models_count
embs += emb / self.models_count
predictions = nn.functional.softmax(predictions, dim=1 ).data.cpu().numpy() [:,1]
embs = embs.cpu().numpy()
predictions = self.postprocess(embs, predictions)
 return np.mean(predictions )<predict_on_test>
submission.to_csv('submission.csv' )