Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<save_to_csv>
warnings.filterwarnings('ignore')
submission['target'] = predictions.mean(axis=1)
 submission.to_csv('stack-mean.csv' )<define_variables>
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 print('train_features: {}'.format(train_features.shape))
print('train_targets_scored: {}'.format(train_targets_scored.shape))
print('train_targets_nonscored: {}'.format(train_targets_nonscored.shape))
print('train_drug: {}'.format(train_drug.shape))
print('test_features: {}'.format(test_features.shape))
print('sample_submission: {}'.format(sample_submission.shape))
scores = kernels['score']
sum_scores = sum(scores)
 weights = [x / sum_scores for x in scores]<concatenate>
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
weighted_sum_prediction = predictions.dot(pd.Series(weights, index=predictions.columns))<save_to_csv>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
submission['target'] = weighted_sum_prediction
submission.to_csv('stack-weighted-sum.csv' )<define_variables>
SEED_VALUE = 42
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=SEED_VALUE )
scores = kernels['score']
sum_scores = sum(scores)
 weights = [x / sum_scores for x in scores]<concatenate>
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=SEED_VALUE ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 print('train_features: {}'.format(train_features.shape))
print('test_features: {}'.format(test_features.shape))
def blend_by_ranking(data, weights):
out = np.zeros(data.shape[0])
 for idx,column in enumerate(data.columns):
out += weights[idx] * rankdata(data[column].values)
 out /= np.max(out)
 return out<merge>
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=SEED_VALUE ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 print('train_features: {}'.format(train_features.shape))
print('test_features: {}'.format(test_features.shape))
blend_by_ranking_prediction = blend_by_ranking(predictions, weights )<save_to_csv>
var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 print('train_features: {}'.format(train_features.shape))
print('test_features: {}'.format(test_features.shape))
submission['target'] = blend_by_ranking_prediction
submission.to_csv('stack-blend-by-ranking.csv' )<load_from_csv>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 train = train[train['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True )
sub1 = pd.read_csv('.. /input/faceinthewild/vgg_face_88.csv')
 sub2 = pd.read_csv('.. /input/faceinthewild/vgg_face_90.csv')
 sub3 = pd.read_csv('.. /input/faceinthewild/vgg_face_891.csv')
 sub4 = pd.read_csv('.. /input/facesinrheworld/vgg_face_85.csv')
 sub5 = pd.read_csv('.. /input/facesinrheworld/vgg_face_v122.csv')
 sub6 = pd.read_csv('.. /input/facesinrheworld/vgg_face_v14.csv')
 sub7 = pd.read_csv('.. /input/facesinrheworld/vgg_face_v19.csv')
 temp= pd.read_csv('.. /input/faceinthewild/vgg_face_891.csv' )<save_to_csv>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
temp['is_related'] = 0.5*sub1['is_related'] + 0.125*sub2['is_related'] + 0.125*sub3['is_related'] + 0.15*sub4['is_related'] + 0.1*sub5['is_related'] + 0.1*sub6['is_related'] + 0.1*sub7['is_related']
temp.to_csv('submission4.csv', index=False )<import_modules>
target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 print('num_targets: {}'.format(num_targets))
print('num_aux_targets: {}'.format(num_aux_targets))
print('num_all_targets: {}'.format(num_all_targets))
import numpy as np
import os
import matplotlib.pyplot as plt
import cv2
from imageio import imread
from skimage.transform import resize
from keras.models import load_model
import pandas as pd
from tqdm import tqdm<install_modules>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
!pip install git+https://github.com/rcmalli/keras-vggface.git<import_modules>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
import h5py
from collections import defaultdict
from glob import glob
from random import choice, sample
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau
from keras import backend as K
from keras.layers import Input, Dense, Flatten, GlobalMaxPool2D, GlobalAvgPool2D, Concatenate, Multiply, Dropout, Subtract, Add, Conv2D, Lambda, Reshape
from keras.models import Model
from keras.optimizers import Adam
from keras_vggface.utils import preprocess_input
from keras_vggface.vggface import VGGFace<define_variables>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
train_file_path = ".. /input/recognizing-faces-in-the-wild/train_relationships.csv"
train_folders_path = ".. /input/recognizing-faces-in-the-wild/train/"
val_families = "F09"<define_variables>
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
all_images = glob(train_folders_path + "*/*/*.jpg")
 print(all_images[0] )<define_variables>
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]
train_images = [x for x in all_images if val_families not in x]
val_images = [x for x in all_images if val_families in x]<define_variables>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
ppl = [x.split("/")[-3] + "/" + x.split("/")[-2] for x in all_images]
print(ppl[0] )<load_from_csv>
feature_cols = [c for c in process_data(train ).columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 num_features
train_person_to_images_map = defaultdict(list)
 for x in train_images:
train_person_to_images_map[x.split("/")[-3] + "/" + x.split("/")[-2]].append(x)
 val_person_to_images_map = defaultdict(list)
 for x in val_images:
val_person_to_images_map[x.split("/")[-3] + "/" + x.split("/")[-2]].append(x)
 relationships = pd.read_csv(train_file_path)
 relationships = list(zip(relationships.p1.values, relationships.p2.values))
relationships = [x for x in relationships if x[0] in ppl and x[1] in ppl]
train = [x for x in relationships if val_families not in x[0]]
val = [x for x in relationships if val_families in x[0]]<categorify>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1
def prewhiten(x):
if x.ndim == 4:
axis =(1, 2, 3)
 size = x[0].size
elif x.ndim == 3:
axis =(0, 1, 2)
 size = x.size
else:
raise ValueError('Dimension should be 3 or 4')
 mean = np.mean(x, axis=axis, keepdims=True)
 std = np.std(x, axis=axis, keepdims=True)
 std_adj = np.maximum(std, 1.0/np.sqrt(size))
y =(x - mean)/ std_adj
return y<load_pretrained>
model = Model(num_features, num_all_targets)
 model
model_path = '.. /input/facenet-keras/facenet_keras.h5'
model_fn = load_model(model_path )<set_options>
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = 7
NFOLDS = 7
DRUG_THRESH = 18
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 train.head()
for layer in model_fn.layers[:-3]:
layer.trainable = True<choose_model_class>
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 train_ = process_data(train)
 test_ = process_data(test)
 kfold_col = f'kfold_{seed_id}'
trn_idx = train_[train_[kfold_col] != fold_id].index
val_idx = train_[train_[kfold_col] == fold_id].index
train_df = train_[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 def train_model(model, tag_name, target_cols_now, fine_tune_scheduler=None):
x_train, y_train = train_df[feature_cols].values, train_df[target_cols_now].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols_now].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3, weight_decay=WEIGHT_DECAY[tag_name])
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
steps_per_epoch=len(trainloader),
pct_start=PCT_START,
div_factor=DIV_FACTOR[tag_name],
max_lr=MAX_LR[tag_name],
epochs=EPOCHS)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing=0.001)
 oof = np.zeros(( len(train), len(target_cols_now)))
 best_loss = np.inf
for epoch in range(EPOCHS):
if fine_tune_scheduler is not None:
fine_tune_scheduler.step(epoch, model)
 train_loss = train_fn(model, optimizer, scheduler, loss_tr, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed_id}, FOLD: {fold_id}, {tag_name}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}")
 if np.isnan(valid_loss):
break
if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"{tag_name}_FOLD{fold_id}_.pth")
 return oof
fine_tune_scheduler = FineTuneScheduler(EPOCHS)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.to(DEVICE)
 train_model(pretrained_model, 'ALL_TARGETS', all_target_cols)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.load_state_dict(torch.load(f"ALL_TARGETS_FOLD{fold_id}_.pth"))
pretrained_model.to(DEVICE)
 final_model = fine_tune_scheduler.copy_without_top(pretrained_model, num_features, num_all_targets, num_targets)
 oof = train_model(final_model, 'SCORED_ONLY', target_cols, fine_tune_scheduler)
 model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f"SCORED_ONLY_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
model_vgg = VGGFace(model='resnet50', include_top=False)
 for layer in model_vgg.layers[:-3]:
layer.trainable = True<define_variables>
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
IMG_SIZE_FN = 160
IMG_SIZE_VGG = 224<categorify>
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_begin = time()
for seed_id in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed_id)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 time_diff = time() - time_begin
train[target_cols] = oof
test[target_cols] = predictions
def read_img_fn(path):
img = cv2.imread(path)
 img = cv2.resize(img,(IMG_SIZE_FN,IMG_SIZE_FN))
img = np.array(img ).astype(np.float)
 return prewhiten(img)
 def read_img_vgg(path):
img = cv2.imread(path)
 img = cv2.resize(img,(IMG_SIZE_VGG,IMG_SIZE_VGG))
img = np.array(img ).astype(np.float)
 return preprocess_input(img, version=2)
 def gen(list_tuples, person_to_images_map, batch_size=16):
ppl = list(person_to_images_map.keys())
 while True:
batch_tuples = sample(list_tuples, batch_size // 2)
 labels = [1] * len(batch_tuples)
 while len(batch_tuples)< batch_size:
p1 = choice(ppl)
 p2 = choice(ppl)
 if p1 != p2 and(p1, p2)not in list_tuples and(p2, p1)not in list_tuples:
batch_tuples.append(( p1, p2))
labels.append(0)
 for x in batch_tuples:
if not len(person_to_images_map[x[0]]):
print(x[0])
 X1 = [choice(person_to_images_map[x[0]])for x in batch_tuples]
X1_FN = np.array([read_img_fn(x)for x in X1])
 X1_VGG = np.array([read_img_vgg(x)for x in X1])
 X2 = [choice(person_to_images_map[x[1]])for x in batch_tuples]
X2_FN = np.array([read_img_fn(x)for x in X2])
 X2_VGG = np.array([read_img_vgg(x)for x in X2])
 yield [X1_FN, X2_FN, X1_VGG, X2_VGG], labels 
<compute_test_metric>
str(timedelta(seconds=time_diff))
def signed_sqrt(x):
return K.sign(x)*K.sqrt(K.abs(x)+1e-9 )<define_search_space>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1] )
def baseline_model() :
input_1 = Input(shape=(IMG_SIZE_FN, IMG_SIZE_FN, 3))
input_2 = Input(shape=(IMG_SIZE_FN, IMG_SIZE_FN, 3))
input_3 = Input(shape=(IMG_SIZE_VGG, IMG_SIZE_VGG, 3))
input_4 = Input(shape=(IMG_SIZE_VGG, IMG_SIZE_VGG, 3))
x1 = model_fn(input_1)
 x2 = model_fn(input_2)
 x3 = model_vgg(input_3)
 x4 = model_vgg(input_4)
 x1 = Reshape(( 1, 1 ,128))(x1)
 x2 = Reshape(( 1, 1 ,128))(x2)
 x1 = Concatenate(axis=-1 )([GlobalMaxPool2D()(x1), GlobalAvgPool2D()(x1)])
 x2 = Concatenate(axis=-1 )([GlobalMaxPool2D()(x2), GlobalAvgPool2D()(x2)])
 x1t = Lambda(lambda tensor : K.square(tensor))(x1)
 x2t = Lambda(lambda tensor : K.square(tensor))(x2)
 x3t = Lambda(lambda tensor : K.square(tensor))(x3)
 x4t = Lambda(lambda tensor : K.square(tensor))(x4)
 merged_add_fn = Add()([x1, x2])
 merged_add_vgg = Add()([x3, x4])
 merged_sub1_fn = Subtract()([x1,x2])
 merged_sub1_vgg = Subtract()([x3,x4])
 merged_sub2_fn = Subtract()([x2,x1])
 merged_sub2_vgg = Subtract()([x4,x3])
 merged_mul1_fn = Multiply()([x1,x2])
 merged_mul1_vgg = Multiply()([x3,x4])
 merged_sq1_fn = Add()([x1t,x2t])
 merged_sq1_vgg = Add()([x3t,x4t])
 merged_sqrt_fn = Lambda(lambda tensor : signed_sqrt(tensor))(merged_mul1_fn)
 merged_sqrt_vgg = Lambda(lambda tensor : signed_sqrt(tensor))(merged_mul1_vgg)
 merged_add_vgg = Conv2D(128 , [1,1] )(merged_add_vgg)
 merged_sub1_vgg = Conv2D(128 , [1,1] )(merged_sub1_vgg)
 merged_sub2_vgg = Conv2D(128 , [1,1] )(merged_sub2_vgg)
 merged_mul1_vgg = Conv2D(128 , [1,1] )(merged_mul1_vgg)
 merged_sq1_vgg = Conv2D(128 , [1,1] )(merged_sq1_vgg)
 merged_sqrt_vgg = Conv2D(128 , [1,1] )(merged_sqrt_vgg)
 merged = Concatenate(axis=-1 )([Flatten()(merged_add_vgg),(merged_add_fn), Flatten()(merged_sub1_vgg),(merged_sub1_fn),
Flatten()(merged_sub2_vgg),(merged_sub2_fn), Flatten()(merged_mul1_vgg),(merged_mul1_fn),
Flatten()(merged_sq1_vgg),(merged_sq1_fn), Flatten()(merged_sqrt_vgg),(merged_sqrt_fn)])
 merged = Dense(100, activation="relu" )(merged)
 merged = Dropout(0.1 )(merged)
 merged = Dense(25, activation="relu" )(merged)
 merged = Dropout(0.1 )(merged)
 out = Dense(1, activation="sigmoid" )(merged)
 model = Model([input_1, input_2, input_3, input_4], out)
 model.compile(loss="binary_crossentropy", metrics=['acc'], optimizer=Adam(0.00001))
model.summary()
return model
file_path = "facenet_vgg.h5"
checkpoint = ModelCheckpoint(file_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
 reduce_on_plateau = ReduceLROnPlateau(monitor="val_acc", mode="max", factor=0.1, patience=20, verbose=1)
 callbacks_list = [checkpoint, reduce_on_plateau]
model = baseline_model()<train_model>
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )
model.fit_generator(gen(train, train_person_to_images_map, batch_size=16), use_multiprocessing=True,
validation_data=gen(val, val_person_to_images_map, batch_size=16), epochs=150, verbose=1,
workers = 4, callbacks=callbacks_list, steps_per_epoch=200, validation_steps=100 )<save_to_csv>
freq_drugs_MoA = [[],
['egfr_inhibitor'],
['nfkb_inhibitor', 'proteasome_inhibitor'],
['cdk_inhibitor'],
['hmgcr_inhibitor'],
['tubulin_inhibitor'],
['raf_inhibitor'],
['flt3_inhibitor', 'kit_inhibitor', 'pdgfr_inhibitor']]
plot_model(model, to_file='model.png',show_shapes = True )<load_from_csv>
pred_cols=[f'pred{i}' for i in range(9)]
eps=1e-5
class_df=pd.read_csv('.. /input/moa-drug-classification-fastai/drug_predictions.csv')
 class_df[pred_cols]=class_df[pred_cols].clip(upper=1-eps )
test_path = ".. /input/recognizing-faces-in-the-wild/test/"
def chunker(seq, size=32):
return(seq[pos:pos + size] for pos in range(0, len(seq), size))
submission = pd.read_csv('.. /input/recognizing-faces-in-the-wild/sample_submission.csv' )<predict_on_test>
train_copy=train.copy()
train_copy=train_copy.merge(class_df[['sig_id']+pred_cols],on='sig_id')
 for i in range(len(freq_drugs_MoA)) :
clipping_1=train_copy[f'pred{i}']
train_copy[freq_drugs_MoA[i]]=train_copy[freq_drugs_MoA[i]].clip(lower=clipping_1,axis=0)
 zero_cols=[col for col in target_cols if col not in freq_drugs_MoA[i]]
clipping_0=(1-train_copy[f'pred{i}'] ).pow(2)
 train_copy[zero_cols]=train_copy[zero_cols].clip(upper=clipping_0,axis=0 )
predictions = []
for batch in tqdm(chunker(submission.img_pair.values)) :
X1 = [x.split("-")[0] for x in batch]
X1_FN = np.array([read_img_fn(test_path + x)for x in X1])
 X1_VGG = np.array([read_img_vgg(test_path + x)for x in X1])
 X2 = [x.split("-")[1] for x in batch]
X2_FN = np.array([read_img_fn(test_path + x)for x in X2])
 X2_VGG = np.array([read_img_vgg(test_path + x)for x in X2])
 pred = model.predict([X1_FN, X2_FN, X1_VGG, X2_VGG] ).ravel().tolist()
predictions += pred
submission['is_related'] = predictions
submission.to_csv("facenetvgg.csv", index=False )<import_modules>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train_copy[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1] )
from collections import defaultdict
from glob import glob
from random import choice, sample
import cv2
import numpy as np
import pandas as pd
from keras.applications.resnet50 import ResNet50, preprocess_input
from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau
from keras.layers import Input, Dense, GlobalMaxPool2D, GlobalAvgPool2D, Concatenate, Multiply, Dropout, Subtract, BatchNormalization
from keras.models import Model
from keras.optimizers import adam<define_variables>
train_features=pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets=pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test_df=pd.read_csv('.. /input/lish-moa/test_features.csv')
 drug_df=pd.read_csv('.. /input/lish-moa/train_drug.csv' )
train_file_path = ".. /input/train_relationships.csv"
train_folders_path = ".. /input/train/"
val_famillies = "F09"<define_variables>
train_features['cp_dose_cont']=train_features.cp_dose.apply(lambda x:int(x[1]))
train_features['cp_time_cont']=train_features.cp_time
test_df['cp_dose_cont']=test_df.cp_dose.apply(lambda x:int(x[1]))
test_df['cp_time_cont']=test_df.cp_time
all_images = glob(train_folders_path + "*/*/*.jpg")
 train_images = [x for x in all_images if val_famillies not in x]
val_images = [x for x in all_images if val_famillies in x] 
<define_variables>
y_names=train_targets.columns.tolist() [1:]
train_person_to_images_map = defaultdict(list)
 ppl = [x.split("/")[-3] + "/" + x.split("/")[-2] for x in all_images]
for x in train_images:
train_person_to_images_map[x.split("/")[-3] + "/" + x.split("/")[-2]].append(x)
<define_variables>
train_df=train_features.merge(train_targets,how='left',on='sig_id')
 train_df.head()
val_person_to_images_map = defaultdict(list)
 for x in val_images:
val_person_to_images_map[x.split("/")[-3] + "/" + x.split("/")[-2]].append(x)
<load_from_csv>
train_df=train_df[train_df.cp_type!='ctl_vehicle']
train_df.reset_index(drop=True,inplace=True)
 test_df=test_df[test_df.cp_type!='ctl_vehicle']
test_df.reset_index(drop=True,inplace=True )
relationships = pd.read_csv(train_file_path)
 relationships = list(zip(relationships.p1.values, relationships.p2.values))
relationships = [x for x in relationships if x[0] in ppl and x[1] in ppl] 
<define_variables>
train_df=train_df.merge(drug_df)
 train_df['vc']=train_df.groupby('drug_id')['sig_id'].transform('count')
 train_df.loc[train_df.vc>19,'drug']=train_df[train_df.vc>19].drug_id
train_df.loc[train_df.vc<=19,'drug']='misc'
train = [x for x in relationships if val_famillies not in x[0]]
val = [x for x in relationships if val_famillies in x[0]] 
<define_variables>
train_df.drug.value_counts().sort_index()
def gen(list_tuples, person_to_images_map, batch_size=16):
ppl = list(person_to_images_map.keys())
 while True:
batch_tuples = sample(list_tuples, batch_size // 2)
 labels = [1] * len(batch_tuples)
 while len(batch_tuples)< batch_size:
p1 = choice(ppl)
 p2 = choice(ppl)
 if p1 != p2 and(p1, p2)not in list_tuples and(p2, p1)not in list_tuples:
batch_tuples.append(( p1, p2))
labels.append(0)
 for x in batch_tuples:
if not len(person_to_images_map[x[0]]):
print(x[0])
 X1 = [choice(person_to_images_map[x[0]])for x in batch_tuples]
X1 = np.array([read_img(x)for x in X1])
 X2 = [choice(person_to_images_map[x[1]])for x in batch_tuples]
X2 = np.array([read_img(x)for x in X2])
 yield [X1, X2], labels 
<install_modules>
class RankGauss(TabularProc):
def setups(self, to:Tabular,n_quantiles=200):
self.cont_names=to.cont_names
self.transformer = QuantileTransformer(n_quantiles=n_quantiles, output_distribution="normal")
 self.transformer.fit(to.train[self.cont_names])
 return self(to)
 def encodes(self, to:Tabular):
to[self.cont_names]=self.transformer.transform(to[self.cont_names])
 return to
!pip install git+https://github.com/rcmalli/keras-vggface.git
!pip install keras_vggface<import_modules>
procs = [Categorify,RankGauss]
from keras.engine import Model
from keras.layers import Input
from keras_vggface.vggface import VGGFace<choose_model_class>
splits=[L(range(len(train_df))),[]]
to=TabularPandas(train_df,procs=procs, cat_names=cat_names, cont_names=cont_names, y_names='drug',
y_block=CategoryBlock() ,splits=splits )
def baseline_model() :
input_1 = Input(shape=(197, 197, 3))
input_2 = Input(shape=(197, 197, 3))
base_model = VGGFace(model='resnet50',include_top=False)
 for x in base_model.layers[:-3]:
x.trainable = True
x1 = base_model(input_1)
 x2 = base_model(input_2)
 x1 = Concatenate(axis=-1 )([GlobalMaxPool2D()(x1), GlobalAvgPool2D()(x1)])
 x2 = Concatenate(axis=-1 )([GlobalMaxPool2D()(x2), GlobalAvgPool2D()(x2)])
 x3 = Subtract()([x1, x2])
 x3 = Dropout(0.25 )(x3)
 x3 = Multiply()([x3, x3])
 x = Multiply()([x1, x2])
 x = Dropout(0.5 )(x)
 x = Concatenate(axis=-1 )([x, x3])
 x = Dense(128, activation="relu" )(x)
 x = Dropout(0.2 )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dropout(0.01 )(x)
 out = Dense(1, activation="sigmoid" )(x)
 model = Model([input_1, input_2], out)
 model.compile(loss="binary_crossentropy", metrics=['acc'], optimizer=adam(lr=0.00001))
model.summary()
return model<train_model>
bs=50
num_layers=2
hidden_units=525
layers=[hidden_units]*num_layers
dropout_rate=0.3
config=tabular_config(ps=[dropout_rate]*num_layers,embed_p=dropout_rate,
act_cls=nn.LeakyReLU(negative_slope=0.1,inplace=True))
dls = to.dataloaders(bs=bs,val_bs=10000)
 learn = tabular_learner(dls, layers=layers,
config=config,
cbs=[],
opt_func = Lamb,
metrics = [] )
model.fit_generator(gen(train, train_person_to_images_map, batch_size=16), use_multiprocessing=True,
validation_data=gen(val, val_person_to_images_map, batch_size=16), epochs=100, verbose=1,
workers=4, callbacks=callbacks_list, steps_per_epoch=200, validation_steps=100 )<string_transform>
path_models=Path('.. /input/moa-drug-classification-fastai')
 fnames=[fname for fname in(path_models/'models' ).ls() if not str(fname ).endswith('model.pth')]
n_runs=len(fnames)
 preds=None
for fname in fnames:
learn.path=path_models
learn.load(fname.stem);
learn.path=Path('.')
 dl=learn.dls.test_dl(test_df)
 with learn.no_bar() :
pred=learn.get_preds(dl=dl)[0]
if preds is None: preds = pred/n_runs
else: preds += pred/n_runs
test_path = ".. /input/test/"
def chunker(seq, size=32):
return(seq[pos:pos + size] for pos in range(0, len(seq), size))<load_from_csv>
test[pred_cols]=preds.numpy()
test[pred_cols]=test[pred_cols].clip(upper=1-eps)
 for i in range(len(freq_drugs_MoA)) :
clipping_1=test[f'pred{i}']
test[freq_drugs_MoA[i]]=test[freq_drugs_MoA[i]].clip(lower=clipping_1,axis=0)
 zero_cols=[col for col in y_names if col not in freq_drugs_MoA[i]]
clipping_0=(1-test[f'pred{i}'] ).pow(2.)
test[zero_cols]=test[zero_cols].clip(upper=clipping_0,axis=0 )
<predict_on_test><EOS> 
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )