Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<prepare_x_and_y>
pyLDAvis.enable_notebook()
np.random.seed(2018)
 warnings.filterwarnings('ignore')
 np.set_printoptions(suppress=True )
class Whaledataset(Dataset):
def __init__(self, datafolder, datatype, transform, y, df):
self.datafolder = datafolder
self.datatype = datatype
self.transform = transform
self.y = y
if datatype =="train":
self.df = df.values
if datatype =="val":
self.df = df.values
self.image_list = [s for s in os.listdir(datafolder)]
def __len__(self):
if self.datatype == "train":
return 20286
elif self.datatype == "val":
return 5075
elif self.datatype == "test":
return len(os.listdir(self.datafolder))
def __getitem__(self, idx):
if self.datatype =='train' or self.datatype =="val":
img_name = os.path.join(self.datafolder, self.df[idx][0])
 label = self.y[idx]
elif self.datatype == 'test':
img_name = os.path.join(self.datafolder, self.image_list[idx])
 label = np.zeros(( 5005,))
image = Image.open(img_name ).convert('RGB')
 image = self.transform(image)
 if self.datatype == 'train' or self.datatype == "val":
return image, label
elif self.datatype == 'test':
return image, label, self.image_list[idx] 
<choose_model_class>
input_columns = ['question_title', 'question_body', 'answer']
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]
def set_parameter_requires_grad(model, feature_extracting):
if feature_extracting:
for param in model.parameters() :
param.requires_grad = False
def initialize_model(model_name, num_classes, feature_extract, use_pretrained=True):
model_ft = None
input_size = 0
if model_name == "resnet":

 model_ft = models.resnet34(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, num_classes)
 input_size = 224
elif model_name == "alexnet":

 model_ft = models.alexnet(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 num_ftrs = model_ft.classifier[6].in_features
model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)
 input_size = 224
elif model_name == "vgg":

 model_ft = models.vgg11_bn(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 num_ftrs = model_ft.classifier[6].in_features
model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)
 input_size = 224
elif model_name == "squeezenet":

 model_ft = models.squeezenet1_0(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 model_ft.classifier[1] = nn.Conv2d(512, num_classes, kernel_size=(1,1), stride=(1,1))
model_ft.num_classes = num_classes
input_size = 224
elif model_name == "densenet":

 model_ft = models.densenet121(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 num_ftrs = model_ft.classifier.in_features
model_ft.classifier = nn.Linear(num_ftrs, num_classes)
 input_size = 224
elif model_name == "inception":

 model_ft = models.inception_v3(pretrained=use_pretrained)
 set_parameter_requires_grad(model_ft, feature_extract)
 num_ftrs = model_ft.AuxLogits.fc.in_features
model_ft.AuxLogits.fc = nn.Linear(num_ftrs, num_classes)
 num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs,num_classes)
 input_size = 299
else:
print("Invalid model name, exiting...")
 exit()
return model_ft, input_size
model_ft, input_size = initialize_model(model_name, num_classes, feature_extract, use_pretrained=True)
 print(model_ft)
<categorify>
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv',index_col='qa_id')
 test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv',index_col='qa_id')
 submission = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv')
 train.shape,test.shape,submission.shape
train_transforms = transforms.Compose([
transforms.Resize(( 224,224)) ,
transforms.RandomHorizontalFlip(0.5),
transforms.RandomAffine(30),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
 ])
 val_transforms = transforms.Compose([
transforms.Resize(( 224,224)) ,
transforms.RandomHorizontalFlip(0.5),
transforms.RandomAffine(30),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
 ])
 test_transforms = transforms.Compose([
transforms.Resize(( 224,224)) ,
transforms.ToTensor() ,
transforms.Normalize(mean = [0.485, 0.456, 0.406],
std = [0.229, 0.224, 0.225])
 ] )<train_model>
def accuracy(outputs,labels):
preds = outputs
preds_idx = torch.argmax(preds, dim=1)
 labels_idx = torch.argmax(labels, dim=1)
 return(preds_idx==labels_idx ).float().sum()
train_datafolder = ".. /input/humpback-whale-identification/train"
train_datatype = "train"
train_df = df_train
train_transform = train_transforms
train_y = y_train
batch_size = 32
num_workers = 0
val_datafolder = ".. /input/humpback-whale-identification/train"
val_datatype = "val"
val_df = df_val
val_transform = val_transforms
val_y = y_val
batch_size = 32
num_workers = 0
def train(batch_size = 32, epochs = 10, transform = None):
tfmd_dataset = Whaledataset(datafolder = train_datafolder, datatype = train_datatype, df = train_df ,
transform = train_transform, y = train_y)
 dataloader = torch.utils.data.DataLoader(tfmd_dataset, batch_size = batch_size, shuffle=True,
num_workers=num_workers)
 val_dataset = Whaledataset(datafolder = val_datafolder, datatype = val_datatype, df = val_df ,
transform = val_transform, y = val_y)
 val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size = batch_size, shuffle=True,
num_workers=num_workers)
 criterion = nn.BCEWithLogitsLoss()
optimizer = optim.Adam(model_ft.parameters() , lr=0.0001)
 if torch.cuda.is_available() :
model_ft.cuda()
nn.DataParallel(model_ft)
 print("GPU")
 epoch_loss_data = []
epoch_accuracy_data = []
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print(device)
 for epoch in range(1,epochs+1):
i = 0.0
running_loss, running_loss_total, epoch_accuracy = 0.0, 0.0, 0.0
val_running_loss, val_running_loss_total, val_epoch_accuracy = 0.0, 0.0, 0.0
for i, data in enumerate(dataloader, 0):
inputs, labels = data
labels = labels.double()
inputs, labels = inputs.to(device), labels.to(device)
 optimizer.zero_grad()
outputs = model_ft(inputs ).double()
loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
epoch_accuracy += accuracy(outputs,labels)
 running_loss += loss.item()


epoch_accuracy = float(epoch_accuracy)/float(32 *(i+1))
print('[%d, %5d] loss: %.3f, epoch_accuracy: %f' %(epoch, i + 1, running_loss, epoch_accuracy))
running_loss = 0.0
epoch_accuracy = 0.0
for i, data in enumerate(val_dataloader,0):
inputs, labels = data
labels = labels.double()
inputs, labels = inputs.to(device), labels.to(device)
 outputs = model_ft(inputs ).double()
loss = criterion(outputs, labels)
 val_epoch_accuracy += accuracy(outputs,labels)
 val_running_epoch_acc = float(val_epoch_accuracy)/float(( labels.size() [0] *(i+1)))
 val_running_loss += loss.item()

 val_epoch_accuracy = float(val_epoch_accuracy)/float(32 *(i+1))
print('[%d, %5d] val loss: %.3f, val_epoch_accuracy: %f' %(epoch, i + 1, val_running_loss, val_epoch_accuracy))
val_running_loss = 0.0
val_epoch_accuracy = 0.0 
<train_model>
train(transform=train_transforms,epochs=25,batch_size=32 )<load_from_csv>
%%time
def fetch_vectors(string_list, batch_size=64):
DEVICE = torch.device("cuda")
 tokenizer = transformers.BertTokenizer.from_pretrained(".. /input/bertbaseuncased/bert-base-uncased/")
 model = transformers.BertModel.from_pretrained(".. /input/bertbaseuncased/bert-base-uncased/")
 model.to(DEVICE)
 fin_features = []
for data in chunks(string_list, batch_size):
tokenized = []
for x in data:
x = " ".join(x.strip().split() [:300])
 tok = tokenizer.encode(x, add_special_tokens=True)
 tokenized.append(tok[:512])
 max_len = 512
padded = np.array([i + [0] *(max_len - len(i)) for i in tokenized])
 attention_mask = np.where(padded != 0, 1, 0)
 input_ids = torch.tensor(padded ).to(DEVICE)
 attention_mask = torch.tensor(attention_mask ).to(DEVICE)
 with torch.no_grad() :
last_hidden_states = model(input_ids, attention_mask=attention_mask)
 features = last_hidden_states[0][:, 0, :].cpu().numpy()
fin_features.append(features)
 fin_features = np.vstack(fin_features)
 return fin_features
def chunks(l, n):

 for i in range(0, len(l), n):
yield l[i:i + n]
sys.path.insert(0, ".. /input/transformers/transformers-master/")
 train_question_body_dense = fetch_vectors(train.question_body.values)
 train_answer_dense = fetch_vectors(train.answer.values)
 test_question_body_dense = fetch_vectors(test.question_body.values)
 test_answer_dense = fetch_vectors(test.answer.values )
test_datafolder = ".. /input/humpback-whale-identification/test"
test_datatype = "test"
test_transform = test_transforms
batch_size = 32
num_workers = 0
sub = pd.read_csv(".. /input/humpback-whale-identification/sample_submission.csv")
 def test(batch_size, transform, model):
test_set = Whaledataset(datafolder = test_datafolder, datatype = test_datatype, transform = test_transform, y = y_train, df = df_train)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size=32, num_workers=num_workers)
 model.cuda()
model.eval()
for(data, target, name)in test_loader:
data = data.cuda()
output = model(data)
 print(output)
 output = output.cpu().detach().numpy()
for i,(e, n)in enumerate(list(zip(output, name))):
sub.loc[sub['Image'] == n, 'Id'] = ' '.join(le_train.inverse_transform(e.argsort() [-5:][::-1]))
sub.to_csv('basic_model.csv', index=False )<train_model>
%%time
find = re.compile(r"^[^.]*")
 train['netloc'] = train['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 test['netloc'] = test['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 features = ['netloc', 'category']
merged = pd.concat([train[features], test[features]])
 ohe = OneHotEncoder()
ohe.fit(merged)
 features_train = ohe.transform(train[features] ).toarray()
features_test = ohe.transform(test[features] ).toarray()
test(batch_size, test_transform, model_ft )<import_modules>
Conv2D, SeparableConv1D, Add, BatchNormalization, Activation, GlobalAveragePooling2D, LeakyReLU, Flatten
Lambda, Multiply, LSTM, Bidirectional, PReLU, MaxPooling1D
%matplotlib inline<define_variables>
%%time
sys.path.insert(0, "/kaggle/input/tftext/tensorflow_text/")
 embed = hub.load("/kaggle/input/useqa3/USEQA3/" )
input_directory = r".. /input/"
training_dir = input_directory + r"train"
testing_dir = input_directory + r"test"<prepare_output>
%%time
embeddings_train = {}
embeddings_test = {}
print("preparing embeddings for train data.... ")
 train['question_title']=train['question_title'].apply(lambda x:x.strip('
'))
train['question_body']=train['question_body'].apply(lambda x:x.strip('
'))
train['answer']=train['answer'].apply(lambda x:x.strip('
'))
train_ans_emb = []
train_questitle_emb = []
train_quesbody_emb = []
for q_t,q_b,a in tqdm(zip(list(train['question_title']),list(train['question_body']),list(train['answer']))):
question_title=[q_t]
question_body=[q_b]
responses=[a]
response_contexts = responses
question_title_embeddings = embed.signatures['question_encoder'](tf.constant(question_title)) ['outputs']
question_body_embeddings = embed.signatures['question_encoder'](tf.constant(question_body)) ['outputs']
response_embeddings = embed.signatures['response_encoder'](input=tf.constant(responses),context=tf.constant(response_contexts)) ['outputs']
train_ans_emb.append(response_embeddings.numpy())
 train_questitle_emb.append(question_title_embeddings.numpy())
 train_quesbody_emb.append(question_body_embeddings.numpy())
 embeddings_train['answer_embedding'] = np.vstack(train_ans_emb)
 embeddings_train['question_body_embedding'] = np.vstack(train_quesbody_emb)
 embeddings_train['question_title_embedding'] = np.vstack(train_questitle_emb)
 print("preparing embeddings for test data.... ")
 test['question_title']=test['question_title'].apply(lambda x:x.strip('
'))
test['question_body']=test['question_body'].apply(lambda x:x.strip('
'))
test['answer']=test['answer'].apply(lambda x:x.strip('
'))
test_ans_emb = []
test_questitle_emb = []
test_quesbody_emb = []
for q_t,q_b,a in tqdm(zip(list(test['question_title']),list(test['question_body']),list(test['answer']))):
question_title=[q_t]
question_body=[q_b]
responses=[a]
response_contexts = responses
question_title_embeddings = embed.signatures['question_encoder'](tf.constant(question_title)) ['outputs']
question_body_embeddings = embed.signatures['question_encoder'](tf.constant(question_body)) ['outputs']
response_embeddings = embed.signatures['response_encoder'](input=tf.constant(responses),context=tf.constant(response_contexts)) ['outputs']
test_ans_emb.append(response_embeddings.numpy())
 test_questitle_emb.append(question_title_embeddings.numpy())
 test_quesbody_emb.append(question_body_embeddings.numpy())
 embeddings_test['answer_embedding'] = np.vstack(test_ans_emb)
 embeddings_test['question_body_embedding'] = np.vstack(test_quesbody_emb)
 embeddings_test['question_title_embedding'] = np.vstack(test_questitle_emb)
 del embed
K.clear_session()
gc.collect()
train_files = os.listdir(training_dir)
 test_files = os.listdir(testing_dir)
 train_labels = []
for file in train_files:
train_labels.append(file.split(".")[0])
 df_train = pd.DataFrame({"id": train_files, "label": train_labels})
 df_train.head()<prepare_output>
l2_dist = lambda x, y: np.power(x - y, 2 ).sum(axis=1)
 cos_dist = lambda x, y:(x*y ).sum(axis=1)
 dist_features_train = np.array([
l2_dist(embeddings_train['question_title_embedding'], embeddings_train['answer_embedding']),
l2_dist(embeddings_train['question_body_embedding'], embeddings_train['answer_embedding']),
l2_dist(embeddings_train['question_body_embedding'], embeddings_train['question_title_embedding']),
cos_dist(embeddings_train['question_title_embedding'], embeddings_train['answer_embedding']),
cos_dist(embeddings_train['question_body_embedding'], embeddings_train['answer_embedding']),
cos_dist(embeddings_train['question_body_embedding'], embeddings_train['question_title_embedding'])
 ] ).T
dist_features_test = np.array([
l2_dist(embeddings_test['question_title_embedding'], embeddings_test['answer_embedding']),
l2_dist(embeddings_test['question_body_embedding'], embeddings_test['answer_embedding']),
l2_dist(embeddings_test['question_body_embedding'], embeddings_test['question_title_embedding']),
cos_dist(embeddings_test['question_title_embedding'], embeddings_test['answer_embedding']),
cos_dist(embeddings_test['question_body_embedding'], embeddings_test['answer_embedding']),
cos_dist(embeddings_test['question_body_embedding'], embeddings_test['question_title_embedding'])
 ] ).T
X_train = np.hstack([item for k, item in embeddings_train.items() ] + [features_train,dist_features_train])
 X_test = np.hstack([item for k, item in embeddings_test.items() ] + [features_test,dist_features_test])
 y_train = train[targets].values
df_test = pd.DataFrame({"id": test_files})
 df_test["label"] = ["cat"]*(len(test_files))
df_test.head()<feature_engineering>
X_train = np.hstack(( X_train, train_question_body_dense, train_answer_dense))
X_test = np.hstack(( X_test, test_question_body_dense, test_answer_dense))
X_train.shape,X_test.shape,y_train.shape
def get_class_files(df, file="id", label = "label", count=5):
label_map = {}
for l in set(df[label]):
label_map[l] = []
for i in range(len(df)) :
label_map[df[label][i]].append(df[file][i])
 return label_map
def select_image_by_label(label_map, image_count_per_label):
label_map_select = {}
for l in label_map:
label_map_select[l] = []
for i in label_map:
num_image = len(label_map[i])
 image_mem = {}
for j in range(image_count_per_label):
image_index = random.randint(0, num_image - 1)
 while image_index in image_mem:
image_index = random.randint(0, num_image - 1)
 label_map_select[i].append(label_map[i][image_index])
 return label_map_select
def plot_sample_image(label_map_select, directory=training_dir, nrows=2, ncols=5, figsize=(16, 4), aspect=None):
fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(16, 4))
i=0
for label in label_map_select:
print(label)
 j=0
for file in label_map_select[label]:
plt.subplot(i+1, ncols, j+1)
 plot_image(file, directory, aspect=aspect)
 j=j+1
plt.tight_layout()
plt.show()
i+=1
def plot_sample_test_image(directory=testing_dir, count=5, nrows=1, ncols=5, figsize=(16, 4), aspect=None):
selected_files = random.sample(os.listdir(testing_dir), count)
 fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(16, 4))
i=0
for file in selected_files:
plt.subplot(1, ncols, i+1)
 path = directory + file
plot_image(file, directory, aspect=aspect)
 i=i+1
plt.tight_layout()
plt.show()
def plot_image(file, directory=None, sub=False, aspect=None):
path = directory + file
img = plt.imread(path)
 plt.imshow(img, aspect=aspect)
 plt.xticks([])
 plt.yticks([])
 plt.title(path.split("/")[-1])
 if sub:
plt.show()<set_options>
pd.DataFrame(X_train ).to_csv('X_train_USEQA_BERTuncased.csv')
 pd.DataFrame(y_train ).to_csv('y_train_USEQA_BERTuncased.csv')
 pd.DataFrame(X_test ).to_csv('X_test_USEQA_BERTuncased.csv' )
def reset_graph(model=None):
if model:
try:
del model
except:
return False
tf.reset_default_graph()
K.clear_session()
gc.collect()
return True
def reset_callbacks(checkpoint=None, reduce_lr=None, early_stopping=None, tensorboard=None):
checkpoint = None
reduce_lr = None
early_stopping = None
tensorboard = None 
<define_variables>
class SpearmanRhoCallback(Callback):
def __init__(self, training_data, validation_data, patience, model_name):
self.x = training_data[0]
self.y = training_data[1]
self.x_val = validation_data[0]
self.y_val = validation_data[1]
self.patience = patience
self.value = -1
self.bad_epochs = 0
self.model_name = model_name
def on_train_begin(self, logs={}):
return
def on_train_end(self, logs={}):
return
def on_epoch_begin(self, epoch, logs={}):
return
def on_epoch_end(self, epoch, logs={}):
y_pred_val = self.model.predict(self.x_val)
 rho_val = np.mean([spearmanr(self.y_val[:, ind], y_pred_val[:, ind] + np.random.normal(0, 1e-7, y_pred_val.shape[0])).correlation for ind in range(y_pred_val.shape[1])])
 if rho_val >= self.value:
self.value = rho_val
else:
self.bad_epochs += 1
if self.bad_epochs >= self.patience:
print("Epoch %05d: early stopping Threshold" % epoch)
 self.model.stop_training = True
print('\rval_spearman-rho: %s' %(str(round(rho_val, 4))), end=100*' '+'
')
 return rho_val
def on_batch_begin(self, batch, logs={}):
return
def on_batch_end(self, batch, logs={}):
return
classes = ['cat', 'dog']<define_variables>
def create_model(n_dense1=256,dropout1=0.30,lr_rate=0.00003):
model = Sequential()
model.add(Dense(n_dense1, input_dim=X_train.shape[1], activation='elu'))
model.add(Dropout(dropout1))
model.add(Dense(y_train.shape[1], activation='sigmoid'))
model.compile(optimizer=tf.keras.optimizers.Adam(lr=lr_rate),loss=tf.keras.losses.binary_crossentropy,metrics=['accuracy'])
 model.summary()
return model
def get_data(batch_size=32, target_size=(96,96), class_mode="categorical", training_dir=training_dir, testing_dir=testing_dir, classes=classes, df_train=df_train, df_test=df_test):
print("Generating data following preprocessing...
 ")
 rescale = 1.0/255
train_batch_size = batch_size
test_batch_size = batch_size
train_shuffle = True
val_shuffle = True
test_shuffle = False
train_datagen = ImageDataGenerator(
 horizontal_flip=True,
shear_range=0.2,
zoom_range=0.2,
rescale=rescale,
validation_split=0.25 
)
 train_generator = train_datagen.flow_from_dataframe(
 df_train,
training_dir,
x_col='id',
y_col='label',
has_ext=True,
target_size=target_size,
classes = classes,
class_mode=class_mode,
batch_size=batch_size,
shuffle=True,
seed=42,
subset='training')
 validation_generator = train_datagen.flow_from_dataframe(
 df_train,
training_dir,
x_col='id',
y_col='label',
has_ext=True,
target_size=target_size,
classes = classes,
class_mode=class_mode,
batch_size=batch_size,
shuffle=True,
seed=42,
subset='validation')
 test_datagen = ImageDataGenerator(rescale=rescale)
 test_generator = test_datagen.flow_from_dataframe(
 df_test,
testing_dir,
x_col='id',
y_col='label',
has_ext=True,
target_size=target_size,
classes = classes,
class_mode=class_mode,
batch_size=batch_size,
shuffle=False 
)
 class_weights = get_weight(train_generator.classes)
 steps_per_epoch = len(train_generator)
 validation_steps = len(validation_generator)
 print("
Data batches generated.")
 return train_generator, validation_generator, test_generator, class_weights, steps_per_epoch, validation_steps
def get_weight(y):
class_weight_current = cw.compute_class_weight('balanced', np.unique(y), y)
 return class_weight_current<choose_model_class>
%%time
all_predictions = []
kf = KFold(n_splits=5, random_state=42, shuffle=True)
 for ind,(tr, val)in enumerate(kf.split(X_train)) :
X_tr = X_train[tr]
y_tr = y_train[tr]
X_vl = X_train[val]
y_vl = y_train[val]
model = create_model()
print(X_tr.shape,y_tr.shape,X_vl.shape,y_vl.shape)
 model.fit(
 X_tr, y_tr, epochs=100, batch_size=32, validation_data=(X_vl, y_vl), verbose=False,
callbacks=[SpearmanRhoCallback(training_data=(X_tr, y_tr), validation_data=(X_vl, y_vl),
patience=5, model_name=f'best_model_batch{ind}.h5')] 
)
 all_predictions.append(model.predict(X_test))
def get_conv_model() :
model = Sequential()
model.add(Conv2D(16,(3, 3), activation='relu', padding="same", input_shape=(3,96,96)))
 model.add(Conv2D(16,(3, 3), padding="same", activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(32,(3, 3), activation='relu', padding="same"))
model.add(Conv2D(32,(3, 3), padding="same", activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(64,(3, 3), activation='relu', padding="same"))
model.add(Conv2D(64,(3, 3), padding="same", activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(96,(3, 3), dilation_rate=(2, 2), activation='relu', padding="same"))
model.add(Conv2D(96,(3, 3), padding="valid", activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(128,(3, 3), dilation_rate=(2, 2), activation='relu', padding="same"))
model.add(Conv2D(128,(3, 3), padding="valid", activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Flatten())
 model.add(Dense(64, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(2 , activation='softmax'))
print(model.summary())
 return model
def get_model(model_name, input_shape=(96, 96, 3)) :
inputs = Input(input_shape)
 if model_name == "Xception":
base_model = Xception(include_top=False, input_shape=input_shape)
 elif model_name == "ResNet50":
base_model = ResNet50(include_top=False, input_shape=input_shape)
 elif model_name == "InceptionV3":
base_model = InceptionV3(weights='imagenet', include_top=False, input_shape=input_shape)
 elif model_name == "InceptionResNetV2":
base_model = InceptionResNetV2(include_top=False, input_shape=input_shape)
 if model_name == "DenseNet201":
base_model = DenseNet201(include_top=False, input_shape=input_shape)
 if model_name == "NASNetMobile":
base_model = NASNetMobile(include_top=False, input_shape=input_shape)
 if model_name == "NASNetLarge":
base_model = NASNetLarge(include_top=False, input_shape=input_shape)
 for layer in base_model.layers:
layer.trainable = False
model = Model(inputs=base_model.input, outputs=predictions)
 model.summary()
return model<compute_test_metric>
model = create_model()
model.fit(X_train, y_train, epochs=33, batch_size=32, verbose=False)
 all_predictions.append(model.predict(X_test))
def auroc(y_true, y_pred):
return tf.py_func(roc_auc_score,(y_true, y_pred), tf.double )<define_variables>
%%time
kf = KFold(n_splits=5, random_state=2019, shuffle=True)
 for ind,(tr, val)in enumerate(kf.split(X_train)) :
X_tr = X_train[tr]
y_tr = y_train[tr]
X_vl = X_train[val]
y_vl = y_train[val]
model = MultiTaskElasticNet(alpha=0.001, random_state=42, l1_ratio=0.5)
 model.fit(X_tr, y_tr)
 all_predictions.append(model.predict(X_test))
model_dir = main_model_dir + time.strftime('%Y-%m-%d %H-%M-%S')+ "/"
log_dir = main_log_dir + time.strftime('%Y-%m-%d %H-%M-%S')
 os.mkdir(model_dir)
 os.mkdir(log_dir)
 model_file = model_dir + "{epoch:02d}-val_acc-{val_acc:.2f}-val_loss-{val_loss:.2f}.hdf5"
model_dir, os.listdir(model_dir), log_dir, os.listdir(log_dir )<choose_model_class>
%%time
model = MultiTaskElasticNet(alpha=0.001, random_state=42, l1_ratio=0.5)
 model.fit(X_train, y_train)
 all_predictions.append(model.predict(X_test))
len(all_predictions )
print("Settting Callbacks")
 checkpoint = ModelCheckpoint(
 model_file,
monitor='val_acc',
save_best_only=True)
 tensorboard = TensorBoard(
 log_dir=log_dir,
update_freq='batch')
 early_stopping = EarlyStopping(
 monitor='val_loss',
patience=2,
verbose=1,
restore_best_weights=True)
 reduce_lr = ReduceLROnPlateau(
 monitor='val_loss',
factor=0.5,
patience=1,
verbose=1)
 callbacks = [reduce_lr, early_stopping, checkpoint]
print("Completed")
 model_dir, os.listdir(model_dir), log_dir, os.listdir(log_dir )<train_model>
%%time
test_preds = np.array([np.array([rankdata(c)for c in p.T] ).T for p in all_predictions] ).mean(axis=0)
 max_val = test_preds.max() + 1
test_preds = test_preds/max_val + 1e-12
print("Starting...
 ")
 start_time = time.time()
batch_size = 32
target_size =(299, 299)
 train_generator, validation_generator, test_generator, class_weights, steps_per_epoch, validation_steps = get_data(batch_size=batch_size, target_size=target_size, classes=classes, df_test=df_test)
 print("
Completed...
 " )<define_variables>
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length, t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32), np.asarray(input_masks, dtype=np.int32), np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
loss = 'categorical_crossentropy'
metrics = ['accuracy']<choose_model_class>
class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(self.model.predict(self.test_inputs, batch_size=self.batch_size))
print("Starting...
 ")
 start_time = time.time()
base_model = InceptionV3(weights='imagenet', include_top=False)
 x = base_model.output
x = GlobalAveragePooling2D()(x)
 x = Dense(1024, activation='relu' )(x)
 predictions = Dense(2, activation='softmax' )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 for layer in base_model.layers:
layer.trainable = False
print("
Compliling Model...
 ")
 learning_rate = 0.0001
optimizer = Adam(learning_rate)
 model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
 verbose = 1
epochs = 30
print("Trainning Model...
 ")
 history = model.fit_generator(
 train_generator,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
verbose=verbose,
callbacks=callbacks,
validation_data=validation_generator,
validation_steps=validation_steps,
class_weight=class_weights)
 elapsed_time = time.time() - start_time
elapsed_time = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
print("
Elapsed Time: " + elapsed_time)
 ", date_time(1))
print("
" )<predict_on_test>
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
def generate_result(model, test_generator, nsteps=len(test_generator)) :
y_preds = model.predict_generator(test_generator, steps=nsteps, verbose=1)
 return y_preds, y_preds[:,1]<prepare_x_and_y>
def train_bert_model(input_units,output_units):
input_word_ids = tf.keras.layers.Input(( input_units,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(( input_units,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(( input_units,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(output_units, activation="sigmoid",name="dense_output" )(x)
 model = tf.keras.models.Model(inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data, learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(( valid_data[0], valid_data[1]),test_data,batch_size,fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate)
 model.compile(loss_function,optimizer)
 model.fit(train_data[0], train_data[1],epochs,batch_size, callbacks=[custom_callback])
 return custom_callback
y_preds_all, y_preds = generate_result(model, test_generator )<save_to_csv>
df_test = pd.DataFrame({"id": test_generator.filenames, "label": y_preds})
 df_test['id'] = df_test['id'].map(lambda x: x.split('.')[0])
 df_test['id'] = df_test['id'].astype(int)
 df_test = df_test.sort_values('id')
 df_test.to_csv('submission.csv', index=False)
 df_test.head()<load_from_csv>
test_predictions = []
test_predictions.append(test_preds)
 final_predictions = np.mean(test_predictions, axis=0)
 final_predictions.shape
df_test2 = pd.read_csv("submission.csv")
 df_test2.head()<import_modules>
submission.iloc[:,1:] = final_predictions
submission.to_csv('submission.csv', index=False)
 submission.head()
print('fast.ai version:{}'.format(fastai.__version__))<define_variables>