Kaggle competition: TGS Salt Identification Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: tgs-salt-identification-challenge<load_from_csv>
plt.style.use('seaborn-white')
 sns.set_style("white")
def notebook_7_inference() :
EXP_NAME = 'first_of_threemodels_net_tuning'
OMIT_VALIDATION = True
data_dir = '.. /input/lish-moa/'
os.listdir(data_dir)
 train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 print('train_features: {}'.format(train_features.shape))
print('train_targets_scored: {}'.format(train_targets_scored.shape))
print('train_targets_nonscored: {}'.format(train_targets_nonscored.shape))
print('train_drug: {}'.format(train_drug.shape))
print('test_features: {}'.format(test_features.shape))
print('sample_submission: {}'.format(sample_submission.shape))
train_features2=train_features.copy()
test_features2=test_features.copy()
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
qt = QuantileTransformer(n_quantiles=100,random_state=42,output_distribution='normal')
 qt.fit(pd.DataFrame(train_features[GENES+CELLS]))
train_features[GENES+CELLS] = qt.transform(train_features[GENES+CELLS])
 test_features[GENES+CELLS] = qt.transform(test_features[GENES+CELLS])
 def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 seed_everything(seed=42)
 n_comp = 600
gen_pca = PCA(n_components=n_comp, random_state=42 ).fit(train_features[GENES])
 train2 = gen_pca.transform(train_features[GENES]); test2 = gen_pca.transform(test_features[GENES])
 train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1)
 n_comp = 50
cell_pca = PCA(n_components=n_comp, random_state=42 ).fit(train_features[CELLS])
 train2 = cell_pca.transform(train_features[CELLS]); test2 = cell_pca.transform(test_features[CELLS])
 train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1)
 var_thresh = VarianceThreshold(0.85 ).fit(train_features.iloc[:,4:])
 train_features_transformed = var_thresh.transform(train_features.iloc[:,4:])
 test_features_transformed = var_thresh.transform(test_features.iloc[:,4:])
 train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 def fe_cluster(train, test, n_clusters_g = 22, n_clusters_c = 4, SEED = 42):
features_g = GENES
features_c = CELLS
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(train_)
 train[f'clusters_{kind}'] = kmeans.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster(train_features2,test_features2)
 train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1)
 def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(train)
 train[f'clusters_pca'] = kmeans.predict(train.values)
 test[f'clusters_pca'] = kmeans.predict(test.values)
 train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca)
 train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2)
 train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:]
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1)
 train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 train = train[train['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
 target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 print('num_targets: {}'.format(num_targets))
print('num_aux_targets: {}'.format(num_aux_targets))
print('num_all_targets: {}'.format(num_all_targets))
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 seed_everything(seed=42)
 class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1024, 1024, 1024, 512]
self.dropout_value = [0.4, 0.25, 0.25, 0.2]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
feature_cols = [c for c in process_data(train ).columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1
model = Model(num_features, num_all_targets)
 model
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = 7
NFOLDS = 7
DRUG_THRESH = 18
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 train.head()
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 save_model_path = f".. /input/lish-moa-final-models/{EXP_NAME}/{EXP_NAME}/seed_{seed_id}"
if not OMIT_VALIDATION:
train_ = process_data(train)
 test_ = process_data(test)
 kfold_col = f'kfold_{seed_id}'
if not OMIT_VALIDATION:
val_idx = train_[train_[kfold_col] == fold_id].index
if not OMIT_VALIDATION:
valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f"{save_model_path}/SCORED_ONLY_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 oof = np.zeros(( len(train), num_targets))
if not OMIT_VALIDATION:
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
valid_dataset = MoADataset(x_valid, y_valid)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 loss_fn = nn.BCEWithLogitsLoss()
valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 oof[val_idx] = valid_preds
print(f"loss : {valid_loss}")
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
SEED = [0,1,2,3,4,5,6]
result_oof = np.zeros(( len(train), len(target_cols)))
 result_predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
fold_oof, fold_predictions = run_k_fold(NFOLDS, seed)
 result_oof += fold_oof
result_predictions += fold_predictions
train[target_cols] =(result_oof / len(SEED))
test[target_cols] =(result_predictions / len(SEED))
if not OMIT_VALIDATION:
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 score = score / y_pred.shape[1]
print("CV log_loss: ", score)
 sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 return sub<load_from_csv>
img_size_ori = 101
img_size_target = 128
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True)
 def iouMetric(true, pred):
true = K.batch_flatten(true)
 pred = K.batch_flatten(pred)
 pred = K.cast(K.greater(pred, 0.5), K.floatx())
 intersec = true * pred
iou = K.sum(intersec)/(K.sum(true + pred - intersec)+ K.epsilon())
 return iou
def notebook_8_inference() :
EXP_NAME = 'first_of_threemodels_longlr_tail'
OMIT_VALIDATION = True
data_dir = '.. /input/lish-moa/'
os.listdir(data_dir)
 train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 print('train_features: {}'.format(train_features.shape))
print('train_targets_scored: {}'.format(train_targets_scored.shape))
print('train_targets_nonscored: {}'.format(train_targets_nonscored.shape))
print('train_drug: {}'.format(train_drug.shape))
print('test_features: {}'.format(test_features.shape))
print('sample_submission: {}'.format(sample_submission.shape))
train_features2=train_features.copy()
test_features2=test_features.copy()
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
qt = QuantileTransformer(n_quantiles=100,random_state=42,output_distribution='normal')
 qt.fit(pd.DataFrame(train_features[GENES+CELLS]))
train_features[GENES+CELLS] = qt.transform(train_features[GENES+CELLS])
 test_features[GENES+CELLS] = qt.transform(test_features[GENES+CELLS])
 def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 seed_everything(seed=42)
 n_comp = 600
gen_pca = PCA(n_components=n_comp, random_state=42 ).fit(train_features[GENES])
 train2 = gen_pca.transform(train_features[GENES]); test2 = gen_pca.transform(test_features[GENES])
 train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1)
 n_comp = 50
cell_pca = PCA(n_components=n_comp, random_state=42 ).fit(train_features[CELLS])
 train2 = cell_pca.transform(train_features[CELLS]); test2 = cell_pca.transform(test_features[CELLS])
 train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1)
 var_thresh = VarianceThreshold(0.85 ).fit(train_features.iloc[:,4:])
 train_features_transformed = var_thresh.transform(train_features.iloc[:,4:])
 test_features_transformed = var_thresh.transform(test_features.iloc[:,4:])
 train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 def fe_cluster(train, test, n_clusters_g = 22, n_clusters_c = 4, SEED = 42):
features_g = GENES
features_c = CELLS
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(train_)
 train[f'clusters_{kind}'] = kmeans.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster(train_features2,test_features2)
 train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1)
 def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(train)
 train[f'clusters_pca'] = kmeans.predict(train.values)
 test[f'clusters_pca'] = kmeans.predict(test.values)
 train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca)
 train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2)
 train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:]
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1)
 train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 train = train[train['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
 target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 print('num_targets: {}'.format(num_targets))
print('num_aux_targets: {}'.format(num_aux_targets))
print('num_all_targets: {}'.format(num_all_targets))
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 seed_everything(seed=42)
 class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
feature_cols = [c for c in process_data(train ).columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1
model = Model(num_features, num_all_targets)
 model
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = 7
NFOLDS = 7
DRUG_THRESH = 18
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 train.head()
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 save_model_path = f".. /input/lish-moa-final-models/{EXP_NAME}/{EXP_NAME}/seed_{seed_id}"
if not OMIT_VALIDATION:
train_ = process_data(train)
 test_ = process_data(test)
 kfold_col = f'kfold_{seed_id}'
if not OMIT_VALIDATION:
val_idx = train_[train_[kfold_col] == fold_id].index
if not OMIT_VALIDATION:
valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f"{save_model_path}/SCORED_ONLY_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 oof = np.zeros(( len(train), num_targets))
if not OMIT_VALIDATION:
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
valid_dataset = MoADataset(x_valid, y_valid)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 loss_fn = nn.BCEWithLogitsLoss()
valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 oof[val_idx] = valid_preds
print(f"loss : {valid_loss}")
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
SEED = [0,1,2,3,4,5,6]
result_oof = np.zeros(( len(train), len(target_cols)))
 result_predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
fold_oof, fold_predictions = run_k_fold(NFOLDS, seed)
 result_oof += fold_oof
result_predictions += fold_predictions
train[target_cols] =(result_oof / len(SEED))
test[target_cols] =(result_predictions / len(SEED))
if not OMIT_VALIDATION:
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 score = score / y_pred.shape[1]
print("CV log_loss: ", score)
 sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 return sub<define_variables>
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
 y_pred = K.cast(y_pred, 'float32')
 y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')
 intersection = y_true_f * y_pred_f
score = 2.* K.sum(intersection)/(K.sum(y_true_f)+ K.sum(y_pred_f))
return score
def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred)
 def bce_logdice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)- K.log(1.- dice_loss(y_true, y_pred))
def weighted_bce_loss(y_true, y_pred, weight):
epsilon = 1e-7
y_pred = K.clip(y_pred, epsilon, 1.- epsilon)
 logit_y_pred = K.log(y_pred /(1.- y_pred))
loss = weight *(logit_y_pred *(1.- y_true)+
K.log(1.+ K.exp(-K.abs(logit_y_pred)))+ K.maximum(-logit_y_pred, 0.))
 return K.sum(loss)/ K.sum(weight)
 def weighted_dice_loss(y_true, y_pred, weight):
smooth = 1.
 w, m1, m2 = weight, y_true, y_pred
intersection =(m1 * m2)
 score =(2.* K.sum(w * intersection)+ smooth)/(K.sum(w * m1)+ K.sum(w * m2)+ smooth)
 loss = 1.- K.sum(score)
 return loss
def weighted_bce_dice_loss(y_true, y_pred):
y_true = K.cast(y_true, 'float32')
 y_pred = K.cast(y_pred, 'float32')
 averaged_mask = K.pool2d(
 y_true, pool_size=(50, 50), strides=(1, 1), padding='same', pool_mode='avg')
 weight = K.ones_like(averaged_mask)
 w0 = K.sum(weight)
 weight = 5.* K.exp(-5.* K.abs(averaged_mask - 0.5))
w1 = K.sum(weight)
 weight *=(w0 / w1)
 loss = weighted_bce_loss(y_true, y_pred, weight)+ dice_loss(y_true, y_pred)
 return loss
start_time = time()
sub_1 = notebook_1_inference()
taken_time = time() - start_time
print(f"Notebook 1 took : {round(taken_time / 60, 3)} minutes")
 start_time = time()
sub_2 = notebook_2_inference()
taken_time = time() - start_time
print(f"Notebook 2 took : {round(taken_time / 60, 3)} minutes")
 start_time = time()
sub_3 = notebook_3_inference()
taken_time = time() - start_time
print(f"Notebook 3 took : {round(taken_time / 60, 3)} minutes")
 start_time = time()
sub_4 = notebook_4_inference()
taken_time = time() - start_time
print(f"Notebook 4 took : {round(taken_time / 60, 3)} minutes")
 start_time = time()
sub_5 = notebook_5_inference()
taken_time = time() - start_time
print(f"Notebook 5 took : {round(taken_time / 60, 3)} minutes")
 start_time = time()
sub_6 = notebook_6_inference()
taken_time = time() - start_time
print(f"Notebook 6 took : {round(taken_time / 60, 3)} minutes")
 start_time = time()
sub_7 = notebook_7_inference()
taken_time = time() - start_time
print(f"Notebook 7 took : {round(taken_time / 60, 3)} minutes")
 start_time = time()
sub_8 = notebook_8_inference()
taken_time = time() - start_time
print(f"Notebook 8 took : {round(taken_time / 60, 3)} minutes" )<sort_values>
train_df = pd.read_csv(".. /input/tgs-salt-identification-challenge/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/tgs-salt-identification-challenge/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
def order_sub(sub):
return sub.sort_values('sig_id' ).reset_index(drop=True)
 sub_1 = order_sub(sub_1)
 sub_2 = order_sub(sub_2)
 sub_3 = order_sub(sub_3)
 sub_4 = order_sub(sub_4)
 sub_5 = order_sub(sub_5)
 sub_6 = order_sub(sub_6)
 sub_7 = order_sub(sub_7)
 sub_8 = order_sub(sub_8 )<prepare_output>
train_df["images"] = [np.array(load_img(".. /input/tgs-salt-identification-challenge/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
BLEND=sub_1.copy()
BLEND.iloc[:,1:] =(sub_1.iloc[:,1:] + sub_2.iloc[:,1:] + sub_3.iloc[:,1:] + sub_4.iloc[:,1:] + sub_5.iloc[:,1:] + sub_6.iloc[:,1:] + sub_7.iloc[:,1:] + sub_8.iloc[:,1:])/ 8
BLEND<save_to_csv>
train_df["masks"] = [np.array(load_img(".. /input/tgs-salt-identification-challenge/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
BLEND.to_csv('submission.csv', index=False )<set_options>
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2 )
%matplotlib inline
sns.set_style('whitegrid')
<define_variables>
def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class )
DATA_DIR = '.. /input/dog-breed-identification/'
TRAIN_DIR = DATA_DIR + 'train/'
TEST_DIR = DATA_DIR + 'test/'
BATCH_SIZE = 128
INPUT_SIZE = 299
NUM_CLASSES = 120
SEED = 42<count_values>
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state=1337 )
breeds_20 = train.breed.value_counts().head(NUM_CLASSES ).index
train = train[train.breed.isin(breeds_20)]<define_variables>
def conv_block(m, dim, acti, bn, res, do=0):
	n = Conv2D(dim, 3, activation=acti, padding='same' )(m)
 	n = BatchNormalization()(n)if bn else n
	n = Dropout(do )(n)if do else n
	n = Conv2D(dim, 3, activation=acti, padding='same' )(n)
 	n = BatchNormalization()(n)if bn else n
	return Concatenate()([m, n])if res else n
def level_block(m, dim, depth, inc, acti, do, bn, mp, up, res):
	if depth > 0:
		n = conv_block(m, dim, acti, bn, res)
 		m = MaxPooling2D()(n)if mp else Conv2D(dim, 3, strides=2, padding='same' )(n)
 		m = level_block(m, int(inc*dim), depth-1, inc, acti, do, bn, mp, up, res)
 		if up:
			m = UpSampling2D()(m)
 			m = Conv2D(dim, 2, activation=acti, padding='same' )(m)
 		else:
			m = Conv2DTranspose(dim, 3, strides=2, activation=acti, padding='same' )(m)
 		n = Concatenate()([n, m])
 		m = conv_block(n, dim, acti, bn, res)
 	else:
		m = conv_block(m, dim, acti, bn, res, do)
 	return m
def UNet(img_shape, out_ch=1, start_ch=64, depth=4, inc_rate=2., activation='relu',
		 dropout=0.5, batchnorm=False, maxpool=True, upconv=True, residual=False):
	i = Input(shape=img_shape)
 	o = level_block(i, start_ch, depth, inc_rate, activation, dropout, batchnorm, maxpool, upconv, residual)
 	o = Conv2D(out_ch, 1, activation='sigmoid' )(o)
 	return Model(inputs=i, outputs=o )
breed_labels = list(breeds_20.values )<choose_model_class>
model = UNet(( img_size_target,img_size_target,1),start_ch=16,depth=5,batchnorm=True )
def create_model(lr=0.0001, dropout=None):
model = Sequential()
base = InceptionResNetV2(weights='imagenet', include_top=False, input_shape=(INPUT_SIZE, INPUT_SIZE, 3))
for layer in base.layers:
layer.trainable = False
model.add(base)
 model.add(GlobalAveragePooling2D())
 model.add(Dense(NUM_CLASSES, activation='softmax'))
adam = Adam(lr=lr)
 sgd = SGD(lr=0.1, momentum=0.95, nesterov=False)
 model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['acc'],)
 return model<split>
sgd = SGD(lr=0.01, decay=1e-4, momentum=0.9, nesterov=True)
 model.compile(loss=weighted_bce_dice_loss, optimizer="adam", metrics=["accuracy",iouMetric] )
train_set, val_set = train_test_split(train, test_size=0.20, stratify=train['breed'], random_state=SEED )<train_model>
seq = iaa.Sequential([
iaa.Fliplr(0.5),
iaa.OneOf([
iaa.Noop() ,
iaa.PiecewiseAffine(scale=(0.01, 0.05), mode='edge', cval=(0)) ,
])
 ] )
gc.collect()
model = create_model()
train_generator.reset()
valid_generator.reset()
STEP_SIZE_TRAIN = train_set.shape[0] // BATCH_SIZE
STEP_SIZE_VALID = val_set.shape[0] // BATCH_SIZE
EPOCHS = 1
base_lr=0.0001
max_lr=1
step_size = EPOCHS * STEP_SIZE_TRAIN
clr = CyclicLR(base_lr=base_lr, max_lr=max_lr, step_size=step_size)
 model.fit_generator(train_generator,
epochs=EPOCHS,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
callbacks=[clr]
)<train_on_grid>
x_train_list=[]
for i in(x_train):
x_train_list.append(i)
 plt_images(x_train_list,5 )
gc.collect()
EPOCHS=40
STEP_SIZE_TRAIN = train_set.shape[0] // BATCH_SIZE
STEP_SIZE_VALID = val_set.shape[0] // BATCH_SIZE
train_generator.reset()
valid_generator.reset()
clr = CyclicLR(base_lr=0.04, max_lr=0.11, step_size=2*STEP_SIZE_TRAIN, mode='triangular')
 checkpoint = ModelCheckpoint('dog_breed_inceptionv3.hf5', monitor='val_acc', verbose=0, save_best_only=True, mode='max', save_weights_only=False)
 reduce = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, verbose=1, min_lr = 0.00001)
 history = model.fit_generator(train_generator,
epochs=EPOCHS,
steps_per_epoch=STEP_SIZE_TRAIN,
validation_data=valid_generator,
validation_steps=STEP_SIZE_VALID,
callbacks=[checkpoint, clr]
)<load_pretrained>
train_index_list=list(range(len(x_train)))
 epoch_list=[]
FileLink('dog_breed_inceptionv3.hf5' )<create_dataframe>
for i in range(1000):
shuffle(train_index_list)
 epoch_list+=train_index_list
model.load_weights('dog_breed_inceptionv3.hf5')
 test_datagen = ImageDataGenerator(rescale=1./255)
 test_generator = test_datagen.flow_from_dataframe(dataframe=submission,
directory=TEST_DIR,
x_col='id',
class_mode=None,
has_ext=False,
batch_size=BATCH_SIZE,
shuffle=False,
seed=SEED,
target_size=(INPUT_SIZE, INPUT_SIZE)
)
 predictions = model.predict_generator(test_generator, verbose=1 )<prepare_output>
def gen_flow(X,y):
batch_size=32
i=0
imgs=[]
masks=[]
while True:
for j in range(32):
imgs.append(x_train[i*32+j])
 masks.append(y_train[i*32+j])
 yield imgs,masks
i+=1
submission.loc[:,1:] = predictions
submission.head()<save_to_csv>
def gen_flow_for_two_inputs(X, y):
genX1 = gen.flow(X,y, batch_size=batch_size)
 while True:
X=genX1.next()
img_mask=np.concatenate([X[0],X[1]],axis=3)
 seq_det=seq.to_deterministic()
img_mask_aug=seq_det.augment_images(img_mask)
 img_mask_aug=crop_batch(img_mask_aug,0.1)
 img_aug=img_mask_aug[:,:,:,0]
mask_aug=np.round(img_mask_aug[:,:,:,1])
 yield np.expand_dims(img_aug,axis=3),np.expand_dims(mask_aug,axis=3 )
submission.to_csv('submission.csv', index=False)
 FileLink('submission.csv' )<import_modules>
sgd = SGD(lr=0.01, decay=1e-4, momentum=0.9, nesterov=True)
 model.compile(loss=bce_dice_loss, optimizer=sgd, metrics=["accuracy",iouMetric] )
import numpy as np
import pandas as pd
import os
from keras.applications import InceptionV3
from keras.models import Sequential
from keras.layers import Dense, Flatten, GlobalAveragePooling2D, Activation, Flatten, Dropout, BatchNormalization
from keras import optimizers, regularizers
from keras.applications import Xception
from keras.optimizers import sgd<load_from_csv>
early_stopping = EarlyStopping(patience=20, verbose=1)
 model_checkpoint = ModelCheckpoint("./keras.model", save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(factor=0.1, patience=20, min_lr=0.000001, verbose=1)
 epochs = 200
batch_size = 16
gen = ImageDataGenerator()
gen_flow = gen_flow_for_two_inputs(x_train, y_train)
 history = model.fit_generator(gen_flow,
validation_data=[x_valid, y_valid],
epochs=epochs,
steps_per_epoch=len(x_train)/ batch_size,
callbacks=[model_checkpoint, reduce_lr] )
df=pd.read_csv('.. /input/labels.csv')
 test_df=pd.read_csv('.. /input/sample_submission.csv')
 image_size = 299<choose_model_class>
model.load_weights("./keras.model",{"bce_dice_loss":bce_dice_loss,'iouMetric':iouMetric})
num_class = 120
model = Sequential()
model.add(InceptionV3(weights='imagenet', include_top=False,pooling='avg'))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_class, activation='softmax'))
model.layers[0].trainable = False
model.compile(optimizer='sgd',loss='categorical_crossentropy',
metrics=['accuracy'])
<prepare_x_and_y>
preds_valid = model.predict(x_valid ).reshape(-1, img_size_target, img_size_target)
 preds_valid = np.array([downsample(x)for x in preds_valid])
 y_valid_ori = np.array([train_df.loc[idx].masks for idx in ids_valid] )
datagen = ImageDataGenerator(preprocessing_function=preprocess_input, rescale=1./255., horizontal_flip=True,width_shift_range = 0.2,height_shift_range = 0.2,validation_split=0.15)
 train_generator=datagen.flow_from_dataframe(dataframe=df,directory=".. /input/train/",x_col="id",y_col="breed",has_ext=False,subset="training",batch_size=16,seed=1997,shuffle=True,class_mode="categorical",target_size=(image_size, image_size))
valid_generator=datagen.flow_from_dataframe(dataframe=df,directory=".. /input/train/",x_col="id",y_col="breed",has_ext=False,subset="validation",batch_size=1,seed=1997,shuffle=True,class_mode="categorical",target_size=(image_size, image_size))
test_datagen=ImageDataGenerator(preprocessing_function=preprocess_input,rescale=1./255.)
test_generator=test_datagen.flow_from_dataframe(dataframe=test_df,directory=".. /input/test/",x_col="id",y_col=None,has_ext=False,batch_size=1,seed=42,shuffle=False,class_mode=None,target_size=(image_size, image_size))<train_model>
thresholds = np.linspace(0, 1, 50)
 ious = np.array([iou_metric_batch(y_valid_ori, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)] )
model.fit_generator(generator=train_generator,
steps_per_epoch=train_generator.n,
validation_data=valid_generator,
validation_steps=train_generator.n,
epochs=1 )<predict_on_test>
threshold_best_index = np.argmax(ious[9:-10])+ 9
iou_best = ious[threshold_best_index]
threshold_best = thresholds[threshold_best_index]
pred=model.predict_generator(test_generator,steps=10357,verbose=1 )<save_to_csv>
def RLenc(img, order='F', format=True):

 bytes = img.reshape(img.shape[0] * img.shape[1], order=order)
 runs = []
r = 0
pos = 1
for c in bytes:
if(c == 0):
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
pos += 1
else:
r += 1
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
if format:
z = ''
for rr in runs:
z += '{} {} '.format(rr[0], rr[1])
 return z[:-1]
else:
return runs
labels =(train_generator.class_indices)
 labels = list(labels.keys())
 df = pd.DataFrame(data=pred,
columns=labels)
 columns = list(df)
 columns.sort()
df = df.reindex(columns=columns)
 filenames = test_df["id"]
df["id"] = filenames
cols = df.columns.tolist()
cols = cols[-1:] + cols[:-1]
df = df[cols]
df.head(5)
 df.to_csv("submission.csv",index=False )<import_modules>
preds_test = model.predict(x_test )
from tensorflow.python.keras.applications import InceptionV3
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense, Flatten, GlobalAveragePooling2D, Activation, Flatten, Dropout, BatchNormalization
from tensorflow.python.keras import optimizers, regularizers 
<load_from_csv>
pred_dict = {idx: RLenc(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }
<choose_model_class><EOS> 
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('submission.csv' )