Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<define_variables>
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
all_list = [G_pred, V_pred, C_pred]<save_to_csv>
print(device_lib.list_local_devices() )
labels = ['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']
sub = pd.DataFrame({'row_id': [f'Test_{i}_{j}' for i in range(len(G_pred)) for j in labels ],'target':[data[i].item() for i in range(len(G_pred)) for data in all_list]})
 sub.to_csv('submission.csv', index=False)
 sub<import_modules>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
import numpy as np
import pandas as pd
import os 
<import_modules>
train_df, val_df = train_test_split(train_df, test_size=0.04, random_state=2018)
 embed_size = 300
max_features = 95000
maxlen = 65
train_X = train_df["question_text"].fillna("_
val_X = val_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
class DenseNetHead(nn.Module):
def __init__(self, in_features: int, n_classes: int):
super().__init__()
self.classifier = nn.Linear(in_features, n_classes)
 def forward(self, x):
return self.classifier(x)
 class DenseNetBase(nn.Module):
def __init__(self, name: str, pretrained: bool = True):
super().__init__()
self.base = getattr(models, name )(pretrained=pretrained)
 if name.endswith('121'):
self.out_features = 1024
elif name.endswith('161'):
self.out_features = 2208
elif name.endswith('169'):
self.out_features = 1664
else:
self.out_features = 1920
def forward(self, x):
base = self.base
features = base.features(x)
 out = F.relu(features, inplace=True)
 out = F.adaptive_avg_pool2d(out,(1, 1))
out = torch.flatten(out, 1)
 return out 
<feature_engineering>
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]
class ResNetHead(nn.Module):
def __init__(self, in_features: int, n_classes: int):
super().__init__()
self.pooling = nn.AdaptiveAvgPool2d(( 1, 1))
self.fc1 = nn.Linear(in_features, n_classes)
 def forward(self, x):
x = self.pooling(x)
 x = torch.flatten(x, start_dim=1)
 x = self.apply_fc_out(x)
 return x
def apply_fc_out(self, x):
return self.fc1(x)
 class ResNetBase(nn.Module):
def __init__(self, name: str, pretrained: bool, frozen_start: bool = False):
super().__init__()
self.base = getattr(models, name )(pretrained=pretrained)
 self.frozen_start = frozen_start
if name == 'resnet34' or name == 'resnet18':
self.out_features = 512
else:
self.out_features = 2048
self.frozen = []
if self.frozen_start:
self.frozen = [self.base.layer1, self.base.conv1, self.base.bn1]
for m in self.frozen:
self._freeze(m)
 def forward(self, x):
base = self.base
x = base.conv1(x)
 x = base.bn1(x)
 x = base.relu(x)
 x = base.maxpool(x)
 x = base.layer1(x)
 x = base.layer2(x)
 x = base.layer3(x)
 x = base.layer4(x)
 return x
def train(self, mode=True):
super().train(mode=mode)
 for m in self.frozen:
self._bn_to_eval(m)
 def _freeze(self, module):
for p in module.parameters() :
p.requires_grad = False
def _bn_to_eval(self, module):
for m in module.modules() :
if isinstance(m, nn.BatchNorm2d):
m.eval()<import_modules>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8"))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 class TPS_SpatialTransformerNetwork(nn.Module):

 def __init__(self, F, I_size, I_r_size, I_channel_num=1):

 super(TPS_SpatialTransformerNetwork, self ).__init__()
self.F = F
self.I_size = I_size
self.I_r_size = I_r_size
self.I_channel_num = I_channel_num
self.LocalizationNetwork = LocalizationNetwork(self.F, self.I_channel_num)
 self.GridGenerator = GridGenerator(self.F, self.I_r_size)
 def forward(self, batch_I):
batch_C_prime = self.LocalizationNetwork(batch_I)
 build_P_prime = self.GridGenerator.build_P_prime(
 batch_C_prime)
 build_P_prime_reshape = build_P_prime.reshape(
 [build_P_prime.size(0), self.I_r_size[0], self.I_r_size[1], 2])
 if torch.__version__ > "1.2.0":
batch_I_r = F.grid_sample(batch_I, build_P_prime_reshape,
padding_mode='border', align_corners=True)
 else:
batch_I_r = F.grid_sample(batch_I, build_P_prime_reshape,
padding_mode='border')
 return batch_I_r
class LocalizationNetwork(nn.Module):

 def __init__(self, F, I_channel_num):
super(LocalizationNetwork, self ).__init__()
self.F = F
self.I_channel_num = I_channel_num
self.conv = nn.Sequential(
 nn.Conv2d(in_channels=self.I_channel_num, out_channels=64, kernel_size=3,
stride=1, padding=1,
bias=False), nn.BatchNorm2d(64), nn.ReLU(True),
nn.MaxPool2d(2, 2),
nn.Conv2d(64, 128, 3, 1, 1, bias=False), nn.BatchNorm2d(128), nn.ReLU(True),
nn.MaxPool2d(2, 2),
nn.Conv2d(128, 256, 3, 1, 1, bias=False), nn.BatchNorm2d(256), nn.ReLU(True),
nn.MaxPool2d(2, 2),
nn.Conv2d(256, 512, 3, 1, 1, bias=False), nn.BatchNorm2d(512), nn.ReLU(True),
nn.AdaptiveAvgPool2d(1)
)
 self.localization_fc1 = nn.Sequential(nn.Linear(512, 256), nn.ReLU(True))
self.localization_fc2 = nn.Linear(256, self.F * 2)
 self.localization_fc2.weight.data.fill_(0)

ctrl_pts_x = np.linspace(-1.0, 1.0, int(F / 2))
ctrl_pts_y_top = np.linspace(0.0, -1.0, num=int(F / 2))
ctrl_pts_y_bottom = np.linspace(1.0, 0.0, num=int(F / 2))
ctrl_pts_top = np.stack([ctrl_pts_x, ctrl_pts_y_top], axis=1)
 ctrl_pts_bottom = np.stack([ctrl_pts_x, ctrl_pts_y_bottom], axis=1)
 initial_bias = np.concatenate([ctrl_pts_top, ctrl_pts_bottom], axis=0)
 self.localization_fc2.bias.data = torch.from_numpy(initial_bias ).float().view(-1)
 def forward(self, batch_I):

 batch_size = batch_I.size(0)
 features = self.conv(batch_I ).view(batch_size, -1)
 batch_C_prime = self.localization_fc2(self.localization_fc1(features)).view(
 batch_size, self.F, 2)
 return batch_C_prime
class GridGenerator(nn.Module):

 def __init__(self, F, I_r_size):

 super(GridGenerator, self ).__init__()
self.eps = 1e-6
self.I_r_height, self.I_r_width = I_r_size
self.F = F
self.C = self._build_C(self.F)
 self.P = self._build_P(self.I_r_width, self.I_r_height)
 self.register_buffer("inv_delta_C", torch.tensor(
 self._build_inv_delta_C(self.F, self.C)).float())
 self.register_buffer("P_hat", torch.tensor(
 self._build_P_hat(self.F, self.C, self.P)).float())
 def _build_C(self, F):

 ctrl_pts_x = np.linspace(-1.0, 1.0, int(F / 2))
ctrl_pts_y_top = -1 * np.ones(int(F / 2))
ctrl_pts_y_bottom = np.ones(int(F / 2))
ctrl_pts_top = np.stack([ctrl_pts_x, ctrl_pts_y_top], axis=1)
 ctrl_pts_bottom = np.stack([ctrl_pts_x, ctrl_pts_y_bottom], axis=1)
 C = np.concatenate([ctrl_pts_top, ctrl_pts_bottom], axis=0)
 return C
def _build_inv_delta_C(self, F, C):

 hat_C = np.zeros(( F, F), dtype=float)
 for i in range(0, F):
for j in range(i, F):
r = np.linalg.norm(C[i] - C[j])
 hat_C[i, j] = r
hat_C[j, i] = r
np.fill_diagonal(hat_C, 1)
 hat_C =(hat_C ** 2)* np.log(hat_C)
 delta_C = np.concatenate(
 [
np.concatenate([np.ones(( F, 1)) , C, hat_C], axis=1),
np.concatenate([np.zeros(( 2, 3)) , np.transpose(C)], axis=1),
np.concatenate([np.zeros(( 1, 3)) , np.ones(( 1, F)) ], axis=1)
 ],
axis=0 
)
 inv_delta_C = np.linalg.inv(delta_C)
 return inv_delta_C
def _build_P(self, I_r_width, I_r_height):
I_r_grid_x =(np.arange(-I_r_width, I_r_width,
2)+ 1.0)/ I_r_width
I_r_grid_y =(np.arange(-I_r_height, I_r_height,
2)+ 1.0)/ I_r_height
P = np.stack(
 np.meshgrid(I_r_grid_x, I_r_grid_y),
axis=2 
)
 return P.reshape([-1, 2])
 def _build_P_hat(self, F, C, P):
n = P.shape[0]
P_tile = np.tile(np.expand_dims(P, axis=1), 
(1, F, 1))
C_tile = np.expand_dims(C, axis=0)
 P_diff = P_tile - C_tile
rbf_norm = np.linalg.norm(P_diff, ord=2, axis=2, keepdims=False)
 rbf = np.multiply(np.square(rbf_norm), np.log(rbf_norm + self.eps))
P_hat = np.concatenate([np.ones(( n, 1)) , P, rbf], axis=1)
 return P_hat
def build_P_prime(self, batch_C_prime):

 batch_size = batch_C_prime.size(0)
 batch_inv_delta_C = self.inv_delta_C.repeat(batch_size, 1, 1)
 batch_P_hat = self.P_hat.repeat(batch_size, 1, 1)
 batch_C_prime_with_zeros = torch.cat(( batch_C_prime, torch.zeros(
 batch_size, 3, 2 ).float().to(device)) , dim=1)
 batch_T = torch.bmm(batch_inv_delta_C,
batch_C_prime_with_zeros)
 batch_P_prime = torch.bmm(batch_P_hat, batch_T)
 return batch_P_prime<choose_model_class>
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer='he_normal', activation='relu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )
def build_grapheme_model(base: str, n_classes: int, **kwargs)-> nn.Module:
return GraphemeModel(base=base, n_classes=n_classes, **kwargs)
 class GraphemeModel(nn.Module):
def __init__(self, *, base: str, n_classes: int, **base_kwargs,):
super().__init__()
if base.startswith('resne'):
self.base = ResNetBase(base, **base_kwargs)
 self.in_features = self.base.out_features
self.head = ResNetHead(
 in_features=self.in_features,
n_classes=n_classes, 
)
 elif base.startswith('vgg'):
self.base = VGGBase(base, **base_kwargs)
 self.head = VGGHead(
 n_classes=n_classes, 
)
 else:
self.base = DenseNetBase(base, **base_kwargs)
 self.in_features = self.base.out_features
self.head = DenseNetHead(
 in_features=self.in_features,
n_classes=n_classes, 
)
 def forward(self, x):
x = self.base(x)
 x = self.head(x)
 return x<choose_model_class>
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))
def build_vc_model(base: str, n_classes: List[int], **kwargs)-> nn.Module:
return VCModel(base=base, n_classes=n_classes, **kwargs)
 class VCModel(nn.Module):
def __init__(self, *, base: str, n_classes: List[int], **base_kwargs,):
super().__init__()
self.transformation = TPS_SpatialTransformerNetwork(
 F=20,
I_size=(224, 224),
I_r_size=(224, 224),
I_channel_num=3, 
)
 if base.startswith('resne'):
self.base = ResNetBase(base, **base_kwargs)
 self.in_features = self.base.out_features
self.vowel_head = ResNetHead(
 in_features=self.in_features,
n_classes=n_classes[0], 
)
 self.consonant_head = ResNetHead(
 in_features=self.in_features,
n_classes=n_classes[1], 
)
 elif base.startswith('vgg'):
self.base = VGGBase(base, **base_kwargs)
 self.vowel_head = VGGHead(
 n_classes=n_classes[0], 
)
 self.consonant_head = VGGHead(
 n_classes=n_classes[1], 
)
 else:
self.base = DenseNetBase(base, **base_kwargs)
 self.in_features = self.base.out_features
self.vowel_head = DenseNetHead(
 in_features=self.in_features,
n_classes=n_classes[0], 
)
 self.consonant_head = DenseNetHead(
 in_features=self.in_features,
n_classes=n_classes[1], 
)
 def forward(self, x):
x = self.transformation(x)
 x = self.base(x)
 vowel = self.vowel_head(x)
 consonant = self.consonant_head(x)
 return vowel, consonant 
<import_modules>
pred_cnn_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_cnn_val_y>thresh ).astype(int))))
from pathlib import Path
import pandas as pd
import os
import torchvision
import torch
from torch.nn import Module
from PIL import Image
import cv2
from torch.utils.data import Dataset, DataLoader
import numpy as np
from tqdm import tqdm
from albumentations import Compose
from albumentations.pytorch import ToTensorV2
import albumentations as A<import_modules>
pred_cnn_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
import torch.nn.functional as F<define_variables>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
BASE = 'resnext50_32x4d'
HEAD_DROPOUT = 0.5
FROZEN_START = 0
CLASSES = [168, 11, 7]
FP16 = True
HEAD = 'SimpleHead'<define_variables>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
GRAPHEME_BASE = 'resnext50_32x4d'
VC_BASE = 'resnext50_32x4d'
GRAPHEME_CLASSES = 168
VC_CLASSES = [11, 7]
SIZE=224
HEIGHT = 137
WIDTH = 236
GRAPHEME_WEIGHTS_FILE = os.path.abspath('/kaggle/input/vcmodels/model_best_g_0.pth')
 VC_WEIGHTS_FILE = os.path.abspath('/kaggle/input/vcmodels/stn_resnext50/model_best_1.pth' )<data_type_conversions>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8"))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )
class GraphemeDataset(Dataset):
def __init__(self, fname, transform=None):
self.df = pd.read_parquet(fname)
 self.data = self.df.iloc[:, 1:].values
self.transform = transform
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
name = self.df.iloc[idx,0]
img = 255 - self.data[idx, :].reshape(HEIGHT, WIDTH ).astype(np.uint8)
 img =(img*(255.0/img.max())).astype(np.uint8)
 img = crop_resize(img)
 if self.transform:
transformed = self.transform(image=img)
 img = transformed['image']
return img, name<define_variables>
%%time
model.fit(train_X, train_y, batch_size=512, epochs=5, validation_data=(val_X, val_y))
TEST = ['/kaggle/input/bengaliai-cv19/test_image_data_0.parquet',
'/kaggle/input/bengaliai-cv19/test_image_data_1.parquet',
'/kaggle/input/bengaliai-cv19/test_image_data_2.parquet',
'/kaggle/input/bengaliai-cv19/test_image_data_3.parquet']<categorify>
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))
transform_test = Compose([
A.Normalize(mean=(0.0692, 0.0692, 0.0692),
std=(0.2052, 0.2052, 0.2052)) ,
ToTensorV2()
] )<load_pretrained>
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
 grapheme_model: Module = build_grapheme_model(
 base=GRAPHEME_BASE,
pretrained=False,
n_classes=GRAPHEME_CLASSES, 
)
 print('Creating model...')
 grapheme_model.load_state_dict(torch.load(GRAPHEME_WEIGHTS_FILE, map_location=device))
grapheme_model = grapheme_model.to(device)
 grapheme_model.eval()<load_pretrained>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
vc_model: Module = build_vc_model(
 base=VC_BASE,
pretrained=False,
n_classes=VC_CLASSES, 
)
 print('Creating model...')
 vc_model.load_state_dict(torch.load(VC_WEIGHTS_FILE, map_location=device))
vc_model = vc_model.to(device)
 vc_model.eval()<create_dataframe>
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8")if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )
row_id,target = [],[]
for fname in TEST:
ds = GraphemeDataset(fname, transform=transform_test)
 dl = DataLoader(ds, batch_size=256, shuffle=False)
 with torch.no_grad() :
for x, y in tqdm(dl):
x = x.to(device, dtype=torch.float)
 pred_g = grapheme_model(x)
 pred_v, pred_c = vc_model(x)
 pred_g = F.softmax(pred_g, dim=1 ).data.cpu().numpy().argmax(axis=1)
 pred_v = F.softmax(pred_v, dim=1 ).data.cpu().numpy().argmax(axis=1)
 pred_c = F.softmax(pred_c, dim=1 ).data.cpu().numpy().argmax(axis=1)
 for idx,name in enumerate(y):
row_id += [f'{name}_grapheme_root',f'{name}_vowel_diacritic',
f'{name}_consonant_diacritic']
target += [pred_g[idx].item() ,pred_v[idx].item() ,pred_c[idx].item() ]<save_to_csv>
model.fit(train_X, train_y, batch_size=512, epochs=5, validation_data=(val_X, val_y))
sub_df = pd.DataFrame({'row_id': row_id, 'target': target})
 sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<define_variables>
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))
DEBUG = False<import_modules>
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
import os
import gc
import cv2
import math
import glob
import numpy as np
import pandas as pd
from tqdm import tqdm_notebook as tqdm
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, Dataset
from efficientnet_pytorch import model as enet<load_from_csv>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
data_dir = '.. /input/bengaliai-cv19'
device = torch.device('cuda')
 HEIGHT = 137
WIDTH = 236
img_size = 128
seen_th = 0.825
c0_dim = 1295
c1_dim = 168
c2_dim = 11
c3_dim = 7
out_dim = c0_dim + c1_dim + c2_dim + c3_dim
num_workers = 4
batch_size = 32
files_test = [f'test_image_data_{fid}.parquet' for fid in range(4)]
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df_test = pd.read_csv(os.path.join(data_dir, 'test.csv'))
df_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
id2grapheme = {i: grapheme for i, grapheme in enumerate(df_train.grapheme.unique())}
grapheme2id = {grapheme: i for i, grapheme in enumerate(df_train.grapheme.unique())}
df_train['grapheme_id'] = df_train['grapheme'].map(grapheme2id)
 df_label_map = []
for i, df in tqdm(df_train.groupby('grapheme_id')) :
df_label_map.append(df.iloc[:, 1:6].drop_duplicates())
 df_label_map = pd.concat(df_label_map ).reset_index(drop=True)
 if DEBUG:
files_test = [f'train_image_data_{fid}.parquet' for fid in range(4)]
df_test = pd.read_csv(os.path.join(data_dir, 'train.csv'))
seen_th = 0.94 
<load_from_csv>
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )
def read_data(f):
f = os.path.join(data_dir, f)
 data = pd.read_parquet(f)
 data = data.iloc[:, 1:].values
return data<data_type_conversions>
model.fit(train_X, train_y, batch_size=256, epochs=5, validation_data=(val_X, val_y))
class BengaliDataset(Dataset):
def __init__(self, data, image_size=128):
self.data = data
self.image_size = image_size
def __len__(self):
return self.data.shape[0]
def __getitem__(self, index):
image = 255 - self.data[index].reshape(HEIGHT, WIDTH)
 image = cv2.resize(image,(self.image_size, self.image_size))
image = image.astype(np.float32)/ 255
image = image[np.newaxis, :, :]
image = np.repeat(image, 3, 0)
 return torch.tensor(image )<feature_engineering>
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))
sigmoid = torch.nn.Sigmoid()
class Swish(torch.autograd.Function):
@staticmethod
def forward(ctx, i):
result = i * sigmoid(i)
 ctx.save_for_backward(i)
 return result
@staticmethod
def backward(ctx, grad_output):
i = ctx.saved_variables[0]
sigmoid_i = sigmoid(i)
 return grad_output *(sigmoid_i *(1 + i *(1 - sigmoid_i)))
 swish = Swish.apply
class Swish_module(nn.Module):
def forward(self, x):
return swish(x)
 swish_layer = Swish_module()
def relu_fn(x):

 return swish_layer(x)
 class DenseCrossEntropy(nn.Module):
def forward(self, x, target, reduction='mean'):
x = x.float()
target = target.float()
logprobs = torch.nn.functional.log_softmax(x, dim=-1)
 loss = -logprobs * target
loss = loss.sum(-1)
 if reduction == 'mean':
return loss.mean()
elif reduction == 'sum':
return loss.sum()
elif reduction == 'none':
return loss
class ArcFaceLoss(nn.modules.Module):
def __init__(self, s=30.0, m=0.5, reduction='mean'):
super().__init__()
self.reduction = reduction
self.s = s
self.cos_m = math.cos(m)
 self.sin_m = math.sin(m)
 self.th = math.cos(math.pi - m)
 self.mm = math.sin(math.pi - m)* m
def forward(self, logits, labels):
logits = logits.float()
cosine = logits
sine = torch.sqrt(1.0 - torch.pow(cosine, 2))
phi = cosine * self.cos_m - sine * self.sin_m
phi = torch.where(cosine > self.th, phi, cosine - self.mm)
 output =(labels * phi)+(( 1.0 - labels)* cosine)
 output *= self.s
loss = DenseCrossEntropy()(output, labels, self.reduction)
 return loss / 2
class ArcMarginProduct(nn.Module):
def __init__(self, in_features, out_features):
super().__init__()
self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))
self.reset_parameters()
def reset_parameters(self):
stdv = 1./ math.sqrt(self.weight.size(1))
self.weight.data.uniform_(-stdv, stdv)
 def forward(self, features):
cosine = F.linear(F.normalize(features), F.normalize(self.weight))
return cosine
class enet_3cg(nn.Module):
def __init__(self, backbone, out_dim_1, out_dim_2):
super(enet_3cg, self ).__init__()
self.enet = enet.EfficientNet.from_name(backbone)
 self.dropouts = nn.ModuleList([
nn.Dropout(0.5)for _ in range(5)
 ])
 self.myfc_1 = nn.Linear(self.enet._fc.in_features, out_dim_2)
 self.activate = Swish_module()
self.myfc_2 = nn.Linear(out_dim_2, out_dim_1)
 self.enet._fc = nn.Identity()
def extract(self, x):
return self.enet(x)
 def forward(self, x):
x = self.extract(x)
 for i, dropout in enumerate(self.dropouts):
if i == 0:
out_2 = self.myfc_1(dropout(x))
else:
out_2 += self.myfc_1(dropout(x))
out_2 /= len(self.dropouts)
 out_1 = self.myfc_2(self.activate(out_2))
return out_1, out_2
class enet_arcface_v2(nn.Module):
def __init__(self, backbone, out_dim_1, out_dim_2):
super(enet_arcface_v2, self ).__init__()
self.enet = enet.EfficientNet.from_name(backbone)
 self.dropouts = nn.ModuleList([
nn.Dropout(0.5)for _ in range(5)
 ])
 self.gfc = nn.Linear(self.enet._fc.in_features, 4096)
 self.metric_classify = ArcMarginProduct(4096, out_dim_1)
 self.myfc_1 = nn.Linear(4096, out_dim_1)
 self.myfc_2_1 = nn.Linear(4096, 512)
 self.myfc_2_2 = nn.Linear(512, out_dim_2)
 self.enet._fc = nn.Identity()
def extract(self, x):
return self.enet(x)
 def forward(self, x):
x = self.extract(x)
 x = Swish_module()(self.gfc(x))
for i, dropout in enumerate(self.dropouts):
if i == 0:
out_1 = self.myfc_1(dropout(x))
out_2 = self.myfc_2_1(dropout(x))
else:
out_1 += self.myfc_1(dropout(x))
out_2 += self.myfc_2_1(dropout(x))
out_1 /= len(self.dropouts)
 out_2 /= len(self.dropouts)
 out_2 = self.myfc_2_2(Swish_module()(out_2))
metric_output = self.metric_classify(x)
 return out_1, out_2, metric_output<load_pretrained>
pred_paragram_test_y = model.predict([test_X], batch_size=512, verbose=1 )
def get_models(model_files, model_class):
enet_type = 'efficientnet-b1'
models = []
for model_f in model_files:
model = model_class(enet_type, out_dim_1=c0_dim, out_dim_2=c1_dim+c2_dim+c3_dim)
 model = model.to(device)
 model.load_state_dict(torch.load(model_f, map_location=lambda storage, loc: storage), strict=True)
 model.eval()
models.append(model)
 print(model_f, 'loaded!')
 return models<define_variables>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
model_files_unseen = [
'.. /input/bengali-train-unseen-model/effnet-b1-unseen_model_fold0.pth',
]
model_files_arcface = [
'.. /input/bengali-train-seen-model/effnet-b1-seen_model_fold0.pth',
]<load_pretrained>
pred_val_y =(3 * pred_glove_val_y + 2 * pred_fasttext_val_y + 3 * pred_paragram_val_y + 2 * pred_cnn_val_y)/ 10.0
thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )
print('loading unseen models...')
 models_unseen = get_models(model_files_unseen, enet_3cg)
 print('loading arcface models...')
 models_arcface = get_models(model_files_arcface, enet_arcface_v2)
 print(len(models_unseen), len(models_arcface))<categorify>
pred_test_y =(3 * pred_glove_test_y + 2 * pred_fasttext_test_y + 3 * pred_paragram_test_y + 2 * pred_cnn_test_y)/ 10.0
pred_test_y =(pred_test_y > best_thresh ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )