Kaggle competition: TGS Salt Identification Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: tgs-salt-identification-challenge<create_dataframe>
plt.style.use('seaborn-white')
 sns.set_style("white")
n_comp = 463
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=1903 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
PCA_GENES = [f'pca_G-{i}' for i in range(n_comp)]
train2 = pd.DataFrame(train2, columns=PCA_GENES)
 test2 = pd.DataFrame(test2, columns=PCA_GENES)
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 60
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=1903 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
PCA_CELLS = [f'pca_C-{i}' for i in range(n_comp)]
train2 = pd.DataFrame(train2, columns=PCA_CELLS)
 test2 = pd.DataFrame(test2, columns=PCA_CELLS)
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
!unzip -o.. /input/tgs-salt-identification-challenge/train.zip -d./train/
var_thresh = VarianceThreshold(threshold=0.9)
 data = train_features.append(test_features)
 data_gene = data.copy()
data_cell = data.copy()
data_gene = data_gene[GENES+PCA_GENES]
data_cell = data_cell[CELLS+PCA_CELLS]
data_transformed_gene = var_thresh.fit_transform(data_gene)
 train_features_transformed_gene = data_transformed_gene[ : train_features.shape[0]]
test_features_transformed_gene = data_transformed_gene[-test_features.shape[0] : ]
train_features_transformed_gene = pd.DataFrame(train_features_transformed_gene)
 test_features_transformed_gene = pd.DataFrame(test_features_transformed_gene)
 train_features_transformed_gene.rename(columns=lambda x: 'vargen_' + str(x), inplace=True)
 test_features_transformed_gene.rename(columns=lambda x: 'vargen_' + str(x), inplace=True)
 data_transformed_cell = var_thresh.fit_transform(data_cell)
 train_features_transformed_cell = data_transformed_cell[ : train_features.shape[0]]
test_features_transformed_cell = data_transformed_cell[-test_features.shape[0] : ]
train_features_transformed_cell = pd.DataFrame(train_features_transformed_cell)
 test_features_transformed_cell = pd.DataFrame(test_features_transformed_cell)
 train_features_transformed_cell.rename(columns=lambda x: 'varcell_' + str(x), inplace=True)
 test_features_transformed_cell.rename(columns=lambda x: 'varcell_' + str(x), inplace=True)
 train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, train_features_transformed_gene, train_features_transformed_cell], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, test_features_transformed_gene, test_features_transformed_cell], axis=1)
 train_features<merge>
!unzip -o.. /input/tgs-salt-identification-challenge/test.zip -d./test/
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
<split>
img_size_ori = 101
img_size_target = 128
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True )
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int )<categorify>
train_df = pd.read_csv(".. /input/tgs-salt-identification-challenge/train.csv", index_col="id", usecols=[0])
 val_ids = pd.read_csv('.. /input/pseudolabelgen/validation_ids.csv')
 train_df = train_df.drop(val_ids.iloc[:,0].values)
 no_salt_ids = pd.read_csv(".. /input/pseudolabelgen/results(1)/no_salt_ids.csv", index_col="id", usecols=[0])
 no_salt_ids = no_salt_ids.sample(2340)
 some_salt_ids = pd.read_csv(".. /input/pseudolabelgen/results(1)/some_salt_ids.csv", index_col="id")
 some_salt_ids = some_salt_ids.sample(3660)
 depths_df = pd.read_csv(".. /input/tgs-salt-identification-challenge/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
class MoADataset:
def __init__(self, gen_features, cell_features, targets):
self.gen_features = gen_features
self.cell_features = cell_features
self.targets = targets
def __len__(self):
return(self.gen_features.shape[0])
 def __getitem__(self, idx):
dct = {
'gen_x' : torch.tensor(self.gen_features[idx, :], dtype=torch.float),
'cell_x' : torch.tensor(self.cell_features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, gen_features, cell_features):
self.gen_features = gen_features
self.cell_features = cell_features
def __len__(self):
return(self.gen_features.shape[0])
 def __getitem__(self, idx):
dct = {
'gen_x' : torch.tensor(self.gen_features[idx, :], dtype=torch.float),
'cell_x' : torch.tensor(self.cell_features[idx, :], dtype=torch.float),
}
return dct<train_model>
train_df["images"] = [np.array(load_img("./train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
train_df["masks"] = [np.array(load_img("./train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs_gen, inputs_cell, targets = data['gen_x'].to(device), data['cell_x'].to(device), data['y'].to(device)
 outputs = model(inputs_gen, inputs_cell)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs_gen, inputs_cell, targets = data['gen_x'].to(device), data['cell_x'].to(device), data['y'].to(device)
 outputs = model(inputs_gen, inputs_cell)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs_gen, inputs_cell = data['gen_x'].to(device), data['cell_x'].to(device)
 with torch.no_grad() :
outputs = model(inputs_gen, inputs_cell)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds 
<choose_model_class>
def rle_decode(rle_mask):

 if str(rle_mask)==str(np.nan):
return np.zeros(( 101,101))
s = rle_mask.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(101*101, dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(101,101 )
dropout_ratio = 0.25
class Model(nn.Module):
def __init__(self, num_features, num_gen_features, num_cell_features, num_targets, hidden_size):
super(Model, self ).__init__()

 self.gen_batch_norm1 = nn.BatchNorm1d(num_gen_features)
 self.gen_dropout1 = nn.Dropout(dropout_ratio)
 self.gen_dense1 = nn.utils.weight_norm(nn.Linear(num_gen_features, hidden_size))
self.cell_batch_norm1 = nn.BatchNorm1d(num_cell_features)
 self.cell_dropout1 = nn.Dropout(dropout_ratio)
 self.cell_dense1 = nn.utils.weight_norm(nn.Linear(num_cell_features, hidden_size))
self.gen_batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.gen_dropout2 = nn.Dropout(dropout_ratio)
 self.gen_dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.cell_batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.cell_dropout2 = nn.Dropout(dropout_ratio)
 self.cell_dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size * 2)
 self.dropout3 = nn.Dropout(dropout_ratio)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size * 2, num_targets))
def forward(self, gen_x, cell_x):

 gen_x = self.gen_batch_norm1(gen_x)
 gen_x = self.gen_dropout1(gen_x)
 gen_x = F.relu(self.gen_dense1(gen_x))
gen_x = self.gen_batch_norm2(gen_x)
 gen_x = self.gen_dropout2(gen_x)
 gen_x = F.relu(self.gen_dense2(gen_x))
cell_x = self.cell_batch_norm1(cell_x)
 cell_x = self.cell_dropout1(cell_x)
 cell_x = F.relu(self.cell_dense1(cell_x))
cell_x = self.cell_batch_norm2(cell_x)
 cell_x = self.cell_dropout2(cell_x)
 cell_x = F.relu(self.gen_dense2(cell_x))
x = torch.cat(( gen_x, cell_x), dim=1)
 x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<feature_engineering>
train_df_temp = pd.DataFrame()
train_df_temp['id'] = no_salt_ids.index
train_df_temp = train_df_temp.set_index('id')
 train_df_temp = train_df_temp.join(depths_df)
 train_df_temp["images"] = [np.array(load_img("./test/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(no_salt_ids.index)]
train_df_temp["masks"] = [np.zeros(( img_size_ori,img_size_ori)) for idx in tqdm_notebook(no_salt_ids.index)]
train_df = train_df.append(train_df_temp)
 del train_df_temp
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data<define_variables>
train_df_temp = pd.DataFrame()
train_df_temp['id'] = some_salt_ids.index
train_df_temp = train_df_temp.set_index('id')
 train_df_temp = train_df_temp.join(depths_df)
 train_df_temp["images"] = [np.array(load_img("./test/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(some_salt_ids.index)]
train_df_temp["masks"] = [np.fliplr(np.rot90(rle_decode(some_salt_ids.loc[idx,'rle_mask']),-1)) for idx in tqdm_notebook(some_salt_ids.index)]
train_df = train_df.append(train_df_temp)
 del train_df_temp
gc.collect()
feature_cols = [c for c in process_data(folds ).columns if c not in TARGETS]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
common_cols = ['cp_time_24', 'cp_time_48', 'cp_time_72', 'cp_dose_D1', 'cp_dose_D2']
gen_features = [c for c in feature_cols if(c.startswith('vargen_')or(c in common_cols)) ]
cell_features = [c for c in feature_cols if(c.startswith('varcell_')or(c in common_cols)) ]
print(len(feature_cols), len(gen_features), len(cell_features))<define_variables>
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2)
 def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
hidden_size=2048<prepare_x_and_y>
ids_valid = val_ids.iloc[:,0].values
temp_df = pd.DataFrame()
temp_df["images"] = [np.array(load_img("./train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(ids_valid)]
temp_df["masks"] = [np.array(load_img("./train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(ids_valid)]
x_valid = np.array(temp_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1)
 y_valid = np.array(temp_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1)
 del temp_df
gc.collect()
def run_training(fold, seed, target_cols):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train_gen, x_train_cell, y_train = train_df[gen_features].values, train_df[cell_features].values, train_df[target_cols].values
x_valid_gen, x_valid_cell, y_valid = valid_df[gen_features].values, valid_df[cell_features].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train_gen, x_train_cell, y_train)
 valid_dataset = MoADataset(x_valid_gen, x_valid_cell, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=len(feature_cols),
num_gen_features=len(gen_features),
num_cell_features=len(cell_features),
num_targets=len(target_cols),
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY, eps=0.00001)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), len(target_cols)))
 best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test_gen, x_test_cell = test_[gen_features].values, test_[cell_features].values
testdataset = TestDataset(x_test_gen, x_test_cell)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=len(feature_cols),
num_gen_features=len(gen_features),
num_cell_features=len(cell_features),
num_targets=len(target_cols),
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions<categorify>
ids_train,x_train,y_train = train_df.index.values,\
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1), \
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1 )
def run_k_fold(NFOLDS, seed, target_cols):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed, target_cols)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<feature_engineering>
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = 0.0
for batch in range(batch_size):
t, p = A[batch], B[batch]
true = np.sum(t)
 pred = np.sum(p)
 if true == 0:
metric +=(pred == 0)
 continue
intersection = np.sum(t * p)
 union = true + pred - intersection
iou = intersection / union
iou = np.floor(max(0,(iou - 0.45)*20)) / 10
metric += iou
metric /= batch_size
return metric
def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred > 0.5], tf.float64 )
for target_cols in TARGETS_SUB:
SEED = [1903, 1881]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed, target_cols)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 predictions[predictions<=0.0001] = 0.0001
predictions[predictions>=0.9999] = 0.9999
train[target_cols] = oof
test[target_cols] = predictions<load_from_csv>
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
 y_pred = K.cast(y_pred, 'float32')
 y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')
 intersection = y_true_f * y_pred_f
score = 2.* K.sum(intersection)/(K.sum(y_true_f)+ K.sum(y_pred_f))
return score
def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred)
 def bce_logdice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)- K.log(1.- dice_loss(y_true, y_pred))
train_targets_scored_orig = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 valid_results = train_targets_scored_orig.drop(columns=list(TARGETS)).merge(train[['sig_id']+list(TARGETS)], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored_orig[list(TARGETS)].values
y_pred = valid_results[list(TARGETS)].values
score = 0
for i in range(len(TARGETS)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )<save_to_csv>
class SnapshotCallbackBuilder:
def __init__(self, nb_epochs, nb_snapshots, init_lr=0.1):
self.T = nb_epochs
self.M = nb_snapshots
self.alpha_zero = init_lr
def get_callbacks(self, model_prefix='Model'):
callback_list = [
callbacks.ModelCheckpoint("./keras.model",monitor='val_my_iou_metric',
mode = 'max', save_best_only=True, verbose=1),
swa,
callbacks.LearningRateScheduler(schedule=self._cosine_anneal_schedule)
 ]
return callback_list
def _cosine_anneal_schedule(self, t):
cos_inner = np.pi *(t %(self.T // self.M))
cos_inner /= self.T // self.M
cos_out = np.cos(cos_inner)+ 1
return float(self.alpha_zero / 2 * cos_out )
sub = sample_submission.drop(columns=list(TARGETS)).merge(test[['sig_id']+list(TARGETS)], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<import_modules>
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 x = BatchNormalization()(x)
 if activation == True:
x = LeakyReLU(alpha=0.1 )(x)
 return x
def residual_block(blockInput, num_filters=16):
x = LeakyReLU(alpha=0.1 )(blockInput)
 x = BatchNormalization()(x)
 blockInput = BatchNormalization()(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 return x
sys.path.append('.. /input/iterative-stratification/iterative-stratification-master')
 try:
except:
warnings.filterwarnings('ignore')
 DEBUG = False
IS_TRAIN = False
DATA_PATH = '.. /input/lish-moa'
CACHE_PATH = '.. /input/mlp1-torch-weights1109'
if not os.path.exists(CACHE_PATH):
os.mkdir(CACHE_PATH)
 data_process_file = f'{CACHE_PATH}/data_process.pkl'
EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
hidden_size = 2048
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 SEED = [0, 1, 2, 3, 4, 5, 6]
if DEBUG:
NFOLDS = 2
EPOCHS = 2
SEED = [0]
CACHE_PATH = f'{CACHE_PATH}/debug'
if not os.path.exists(CACHE_PATH):
os.mkdir(CACHE_PATH)
 def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
def save_pickle(dic, save_path):
with open(save_path, 'wb')as f:
pickle.dump(dic, f)
 def load_pickle(load_path):
with open(load_path, 'rb')as f:
message_dict = pickle.load(f)
 return message_dict
def score_fn(y_true, y_pred):
losses = []
for idx in range(len(target_cols)) :
loss = log_loss(y_true[:, idx], y_pred[:, idx])
 losses.append(loss)
 return np.mean(losses)
 seed_everything(seed=42)
 selected_cols = ['sig_id','cp_type','cp_time','cp_dose'
,'g-0','g-2','g-3','g-5','g-6','g-7','g-8'
,'g-9','g-11','g-12','g-13','g-15','g-16','g-17','g-18','g-20','g-21'
,'g-22','g-24','g-25','g-26','g-27','g-28','g-29','g-30','g-31','g-32'
,'g-33','g-34','g-36','g-37','g-38','g-39','g-41','g-42','g-43','g-45'
,'g-47','g-48','g-49','g-50','g-51','g-52','g-53','g-54','g-55','g-56'
,'g-57','g-58','g-60','g-61','g-62','g-63','g-65','g-66','g-67','g-68'
,'g-69','g-70','g-71','g-72','g-73','g-75','g-76','g-77','g-78','g-79'
,'g-80','g-81','g-83','g-84','g-85','g-86','g-87','g-89','g-90','g-91'
,'g-92','g-93','g-94','g-96','g-97','g-98','g-100','g-101','g-102','g-103'
,'g-104','g-105','g-106','g-107','g-108','g-109','g-110','g-111','g-112'
,'g-113','g-114','g-115','g-116','g-117','g-118','g-119','g-120','g-121'
,'g-122','g-123','g-124','g-125','g-126','g-127','g-129','g-130','g-131'
,'g-132','g-133','g-134','g-135','g-136','g-137','g-138','g-139','g-140'
,'g-141','g-142','g-143','g-144','g-146','g-147','g-148','g-149','g-150'
,'g-151','g-152','g-154','g-156','g-157','g-158','g-160','g-161','g-162'
,'g-163','g-164','g-165','g-166','g-167','g-169','g-170','g-172','g-173'
,'g-174','g-175','g-177','g-178','g-179','g-180','g-181','g-183','g-184'
,'g-185','g-186','g-187','g-188','g-189','g-190','g-192','g-194','g-195'
,'g-196','g-199','g-200','g-202','g-203','g-205','g-206','g-207','g-208'
,'g-209','g-210','g-211','g-212','g-215','g-216','g-217','g-218','g-219'
,'g-221','g-222','g-224','g-225','g-226','g-227','g-228','g-229','g-230'
,'g-231','g-233','g-235','g-236','g-237','g-238','g-239','g-240','g-241'
,'g-242','g-243','g-245','g-246','g-247','g-248','g-250','g-251','g-252'
,'g-253','g-254','g-255','g-256','g-257','g-258','g-260','g-262','g-263'
,'g-265','g-267','g-268','g-269','g-270','g-272','g-273','g-274','g-276'
,'g-279','g-280','g-283','g-284','g-285','g-286','g-287','g-291','g-292'
,'g-293','g-294','g-296','g-297','g-298','g-299','g-300','g-301','g-302'
,'g-303','g-305','g-306','g-307','g-308','g-309','g-310','g-312','g-313'
,'g-314','g-317','g-318','g-319','g-321','g-322','g-323','g-324','g-325'
,'g-326','g-327','g-328','g-329','g-330','g-331','g-332','g-335','g-336'
,'g-337','g-338','g-340','g-341','g-342','g-343','g-344','g-346','g-347'
,'g-348','g-349','g-350','g-352','g-353','g-354','g-355','g-356','g-357'
,'g-358','g-359','g-360','g-361','g-362','g-363','g-365','g-366','g-367'
,'g-368','g-369','g-371','g-372','g-373','g-374','g-375','g-376','g-377'
,'g-379','g-380','g-381','g-382','g-383','g-384','g-385','g-386','g-387'
,'g-388','g-389','g-390','g-391','g-392','g-394','g-395','g-396','g-397'
,'g-398','g-400','g-402','g-403','g-404','g-405','g-407','g-408','g-409'
,'g-410','g-411','g-412','g-414','g-415','g-416','g-417','g-418','g-419'
,'g-420','g-421','g-422','g-423','g-424','g-425','g-426','g-427','g-428'
,'g-429','g-430','g-431','g-432','g-433','g-434','g-435','g-438','g-439'
,'g-440','g-441','g-442','g-443','g-444','g-445','g-446','g-447','g-449'
,'g-450','g-451','g-453','g-454','g-455','g-456','g-457','g-458','g-459'
,'g-460','g-461','g-462','g-463','g-465','g-466','g-468','g-469','g-470'
,'g-471','g-472','g-473','g-474','g-475','g-476','g-479','g-480','g-482'
,'g-483','g-484','g-485','g-486','g-488','g-489','g-491','g-492','g-493'
,'g-497','g-498','g-499','g-500','g-502','g-503','g-504','g-506','g-507'
,'g-508','g-509','g-510','g-511','g-513','g-514','g-515','g-516','g-518'
,'g-520','g-522','g-523','g-524','g-525','g-526','g-527','g-528','g-529'
,'g-530','g-531','g-533','g-534','g-535','g-536','g-537','g-538','g-539'
,'g-540','g-541','g-542','g-543','g-544','g-546','g-547','g-550','g-551'
,'g-552','g-553','g-554','g-555','g-556','g-557','g-558','g-559','g-560'
,'g-561','g-562','g-563','g-564','g-566','g-567','g-568','g-569','g-570'
,'g-571','g-572','g-574','g-577','g-578','g-579','g-580','g-583','g-584'
,'g-587','g-588','g-589','g-590','g-592','g-593','g-594','g-595','g-596'
,'g-597','g-598','g-599','g-600','g-602','g-604','g-605','g-606','g-608'
,'g-609','g-610','g-611','g-612','g-613','g-614','g-616','g-619','g-620'
,'g-622','g-624','g-627','g-628','g-629','g-630','g-631','g-632','g-634'
,'g-635','g-636','g-639','g-640','g-641','g-642','g-643','g-644','g-646'
,'g-647','g-648','g-649','g-651','g-652','g-655','g-656','g-657','g-658'
,'g-659','g-660','g-661','g-663','g-664','g-665','g-666','g-667','g-669'
,'g-671','g-672','g-673','g-674','g-675','g-677','g-678','g-679','g-681'
,'g-682','g-683','g-684','g-685','g-686','g-688','g-689','g-691','g-692'
,'g-693','g-694','g-696','g-697','g-698','g-699','g-700','g-701','g-702'
,'g-704','g-705','g-706','g-708','g-709','g-710','g-711','g-712','g-713'
,'g-714','g-720','g-722','g-724','g-725','g-726','g-727','g-728','g-729'
,'g-731','g-733','g-734','g-735','g-736','g-737','g-738','g-739','g-740'
,'g-741','g-742','g-743','g-744','g-745','g-746','g-747','g-748','g-749'
,'g-750','g-751','g-752','g-753','g-755','g-756','g-757','g-758','g-759'
,'g-760','g-761','g-762','g-763','g-764','g-766','g-767','g-768','g-769'
,'g-771','c-0','c-5','c-6','c-7','c-8','c-9','c-10','c-12','c-13','c-15'
,'c-18','c-20','c-22','c-24','c-25','c-26','c-30','c-33','c-34','c-36'
,'c-37','c-38','c-41','c-44','c-45','c-46','c-47','c-48','c-50','c-51'
,'c-52','c-54','c-56','c-57','c-58','c-59','c-60','c-62','c-63','c-64'
,'c-65','c-66','c-67','c-69','c-70','c-71','c-72','c-73','c-75','c-76'
,'c-77','c-79','c-80','c-81','c-83','c-85','c-86','c-87','c-89','c-92'
,'c-93','c-95','c-96','c-98','c-99']
if IS_TRAIN:
data_features = pd.read_csv(f'{DATA_PATH}/train_features.csv')[selected_cols]
test_features = pd.read_csv(f'{DATA_PATH}/test_features.csv')[selected_cols]
train_targets_scored = pd.read_csv(f'{DATA_PATH}/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(f'{DATA_PATH}/train_targets_nonscored.csv')
 print(f'Train: {len(data_features)}')
 target_cols = [col for col in train_targets_scored.columns if col != 'sig_id']
data_process_dict = {}
data_process_dict['target_cols'] = target_cols
else:
data_features = pd.read_csv(f'{DATA_PATH}/test_features.csv')[selected_cols]
sample_submission = pd.read_csv(f'{DATA_PATH}/sample_submission.csv')
 print(f'Test: {len(data_features)}')
 data_process_dict = load_pickle(data_process_file)
 target_cols = data_process_dict['target_cols']
id_col = 'sig_id'
GENES = [col for col in data_features.columns if col.startswith('g-')]
CELLS = [col for col in data_features.columns if col.startswith('c-')]
feature_cols = GENES + CELLS
vec_len = len(data_features)
 if IS_TRAIN:
rankGauss_dict = {}
for col in tqdm(GENES + CELLS, desc='GENES + CELLS cols rankgauss fitting'):
transformer = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution="normal")
 transformer.fit(data_features[col].values.reshape(vec_len, 1))
rankGauss_dict[col] = transformer
data_process_dict['rankGauss_dict'] = rankGauss_dict
else:
rankGauss_dict = data_process_dict['rankGauss_dict']
for col in tqdm(GENES + CELLS, desc='GENES + CELLS cols rankgauss transforming'):
raw_vec = data_features[col].values.reshape(vec_len, 1)
 data_features[col] = rankGauss_dict[col].transform(raw_vec ).reshape(1, vec_len)[0]
n_comp = 50
if IS_TRAIN:
pca = PCA(n_components=n_comp, random_state=42)
 pca.fit(data_features[GENES])
 data_process_dict['g_pca'] = pca
pca = data_process_dict['g_pca']
data2 = pca.transform(data_features[GENES])
 data2 = pd.DataFrame(data2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 feature_cols.extend([f'pca_G-{i}' for i in range(n_comp)])
 data_features = pd.concat(( data_features, data2), axis=1)
 n_comp = 15
if IS_TRAIN:
pca = PCA(n_components=n_comp, random_state=42)
 pca.fit(data_features[CELLS])
 data_process_dict['c_pca'] = pca
pca = data_process_dict['c_pca']
data2 = pca.transform(data_features[CELLS])
 data2 = pd.DataFrame(data2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 feature_cols.extend([f'pca_C-{i}' for i in range(n_comp)])
 data_features = pd.concat(( data_features, data2), axis=1)
 n_comp = 50
if IS_TRAIN:
svd = TruncatedSVD(n_components=n_comp, random_state=42)
 svd.fit(data_features[GENES])
 data_process_dict['g_svd'] = svd
svd = data_process_dict['g_svd']
data2 = svd.transform(data_features[GENES])
 data2 = pd.DataFrame(data2, columns=[f'TruncatedSVD_G-{i}' for i in range(n_comp)])
 feature_cols.extend([f'TruncatedSVD_G-{i}' for i in range(n_comp)])
 data_features = pd.concat(( data_features, data2), axis=1)
 n_comp = 15
if IS_TRAIN:
svd = TruncatedSVD(n_components=n_comp, random_state=42)
 svd.fit(data_features[CELLS])
 data_process_dict['c_svd'] = svd
svd = data_process_dict['c_svd']
data2 = svd.transform(data_features[CELLS])
 data2 = pd.DataFrame(data2, columns=[f'TruncatedSVD_C-{i}' for i in range(n_comp)])
 feature_cols.extend([f'TruncatedSVD_C-{i}' for i in range(n_comp)])
 data_features = pd.concat(( data_features, data2), axis=1)
 n_comp = 50
if IS_TRAIN:
fa = FactorAnalysis(n_components=n_comp, random_state=42)
 fa.fit(data_features[GENES])
 data_process_dict['g_fa'] = fa
fa = data_process_dict['g_fa']
data2 = fa.transform(data_features[GENES])
 data2 = pd.DataFrame(data2, columns=[f'FA_G-{i}' for i in range(n_comp)])
 feature_cols.extend([f'FA_G-{i}' for i in range(n_comp)])
 data_features = pd.concat(( data_features, data2), axis=1)
 n_comp = 15
if IS_TRAIN:
fa = FactorAnalysis(n_components=n_comp, random_state=42)
 fa.fit(data_features[CELLS])
 data_process_dict['c_fa'] = fa
fa = data_process_dict['c_fa']
data2 = fa.transform(data_features[CELLS])
 data2 = pd.DataFrame(data2, columns=[f'FA_C-{i}' for i in range(n_comp)])
 feature_cols.extend([f'FA_C-{i}' for i in range(n_comp)])
 data_features = pd.concat(( data_features, data2), axis=1)
 if IS_TRAIN:
cate_dict = {}
for col in ['cp_time', 'cp_dose']:
cate_dict[col] = []
for mod in data_features[col].unique() :
cate_dict[col].append(mod)
 data_process_dict['cate_dict'] = cate_dict
cate_dict = data_process_dict['cate_dict']
for col in cate_dict:
for mod in cate_dict[col]:
data_features[str(mod)] =(data_features[col] == mod ).astype(int)
 feature_cols.append(str(mod))
class MoADataset:
def __init__(self, features, targets, isTrain=True):
self.features = features
self.targets = targets
self.isTrain = isTrain
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x': torch.tensor(self.features[idx, :], dtype=torch.float),
}
if self.isTrain:
dct['y'] = torch.tensor(self.targets[idx, :], dtype=torch.float)
 return dct
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
if scheduler:
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
def MultilabelStratifiedKFold_run(
 model_fn,
train_feat,
train_labels,
test_feat,
seed,
model_name,
input_dim,
output_dim,
nfolds=NFOLDS,
verbose=0,
isTrain=IS_TRAIN, 
):
seed_everything(seed)
 if IS_TRAIN:
train_pred = np.zeros(( len(train_feat), output_dim))
kf = MultilabelStratifiedKFold(n_splits=nfolds, shuffle=True, random_state=seed)
 cluster_num = 250
kmeans = KMeans(n_clusters=cluster_num, random_state=0 ).fit(train_labels)
 label_cluster = np.eye(cluster_num)[kmeans.predict(train_labels)]
for _fold,(trn_idx, val_idx)in enumerate(kf.split(train_feat, label_cluster)) :
start_time = time.time()
train_dataset = MoADataset(train_feat[trn_idx], train_labels[trn_idx], True)
 valid_dataset = MoADataset(train_feat[val_idx], train_labels[val_idx], True)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = model_fn(
 num_features=input_dim,
num_targets=output_dim,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = SmoothBCEwLogits(smoothing=1e-3)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
best_loss = np.inf
model_weights = f"{CACHE_PATH}/{model_name}_SEED{seed}_FOLD{_fold}.pth"
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer, scheduler, loss_fn, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 if verbose:
print(f"SEED: {seed}, FOLD: {_fold}, EPOCH: {epoch:3}, train_loss: {train_loss:.5f}, valid_loss: {valid_loss:.5f}, "
f"time: {(time.time() - start_time)/ 60:.2f}min")
 if valid_loss < best_loss:
best_loss = valid_loss
torch.save(model.state_dict() , model_weights)
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
model.load_state_dict(torch.load(model_weights))
train_pred[val_idx] = inference_fn(model, validloader, DEVICE)
 return train_pred
else:
test_pred = np.zeros(( len(test_feat), output_dim))
for _fold in range(nfolds):
model = Model(
 num_features=input_dim,
num_targets=output_dim,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 model_weights = f"{CACHE_PATH}/{model_name}_SEED{seed}_FOLD{_fold}.pth"
model.load_state_dict(torch.load(model_weights))
testdataset = MoADataset(test_feat, None, False)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 test_pred += inference_fn(model, testloader, DEVICE)/ nfolds
return test_pred
if IS_TRAIN:
data = data_features.merge(train_targets_scored, on='sig_id')
 data = data[data['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 data_labels = data[target_cols].values
else:
data = data_features[data_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 feature_cols = pd.read_csv('.. /input/feature-selecting1105/chosen_feat_df.csv' ).top_feat_cols.values.tolist()
data_feat = data[feature_cols].values<choose_model_class>
def UXception(input_shape=(None, None, 3)) :
backbone = Xception(input_shape=input_shape,weights='imagenet',include_top=False)
 input = backbone.input
start_neurons = 16
conv4 = backbone.layers[121].output
conv4 = LeakyReLU(alpha=0.1 )(conv4)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(0.1 )(pool4)
 convm = Conv2D(start_neurons * 32,(3, 3), activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 32)
 convm = residual_block(convm,start_neurons * 32)
 convm = LeakyReLU(alpha=0.1 )(convm)
 deconv4 = Conv2DTranspose(start_neurons * 16,(3, 3), strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(0.1 )(uconv4)
 uconv4 = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 16)
 uconv4 = residual_block(uconv4,start_neurons * 16)
 uconv4 = LeakyReLU(alpha=0.1 )(uconv4)
 deconv3 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(uconv4)
 conv3 = backbone.layers[31].output
uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(0.1 )(uconv3)
 uconv3 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 8)
 uconv3 = residual_block(uconv3,start_neurons * 8)
 uconv3 = LeakyReLU(alpha=0.1 )(uconv3)
 deconv2 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 conv2 = backbone.layers[21].output
conv2 = ZeroPadding2D(((1,0),(1,0)) )(conv2)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(0.1 )(uconv2)
 uconv2 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 4)
 uconv2 = residual_block(uconv2,start_neurons * 4)
 uconv2 = LeakyReLU(alpha=0.1 )(uconv2)
 deconv1 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv2)
 conv1 = backbone.layers[11].output
conv1 = ZeroPadding2D(((3,0),(3,0)) )(conv1)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(0.1 )(uconv1)
 uconv1 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 2)
 uconv1 = residual_block(uconv1,start_neurons * 2)
 uconv1 = LeakyReLU(alpha=0.1 )(uconv1)
 uconv0 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="same" )(uconv1)
 uconv0 = Dropout(0.1 )(uconv0)
 uconv0 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv0)
 uconv0 = residual_block(uconv0,start_neurons * 1)
 uconv0 = residual_block(uconv0,start_neurons * 1)
 uconv0 = LeakyReLU(alpha=0.1 )(uconv0)
 uconv0 = Dropout(0.1/2 )(uconv0)
 output_layer = Conv2D(1,(1,1), padding="same", activation="sigmoid" )(uconv0)
 model = Model(input, output_layer)
 model.name = 'u-xception'
return model
CACHE_PATH = '.. /input/mlp1-torch-weights1109'
hidden_size = 2048
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.4)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.4)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
self.Relu = nn.ReLU(inplace=True)
 self.PReLU = nn.PReLU()
self.LeakyReLU = nn.LeakyReLU(negative_slope=0.01, inplace=True)
 self.GeLU = nn.GELU()
self.RReLU = nn.RReLU()
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dense1(x)
 x = self.PReLU(x)
 x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = self.dense2(x)
 x = self.PReLU(x)
 x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
oof_pred1 = pd.read_csv(f'{CACHE_PATH}/oof_pred.csv')[target_cols].values
npy_pred1 = np.zeros(( len(data), len(target_cols)))
 for seed in SEED:
if IS_TRAIN:
seed_npy_pred = MultilabelStratifiedKFold_run(
 model_fn=Model,
train_feat=data_feat,
train_labels=data_labels,
test_feat=None,
seed=seed,
model_name='mlp',
input_dim=len(feature_cols),
output_dim=len(target_cols),
nfolds=NFOLDS,
verbose=1,
isTrain=IS_TRAIN, 
)
 else:
seed_npy_pred = MultilabelStratifiedKFold_run(
 model_fn=Model,
train_feat=None,
train_labels=None,
test_feat=data_feat,
seed=seed,
model_name='mlp',
input_dim=len(feature_cols),
output_dim=len(target_cols),
nfolds=NFOLDS,
verbose=1,
isTrain=IS_TRAIN, 
)
 npy_pred1 += seed_npy_pred / len(SEED )<choose_model_class>
K.clear_session()
model = UXception(input_shape=(img_size_target,img_size_target,3))
CACHE_PATH = '.. /input/mlp2-torch-weights1110'
hidden_size = 1024
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
hidden_size1 = 1024
hidden_size2 = 1024
hidden_size3 = 512
self.l1_tanh = nn.Sequential(
 nn.BatchNorm1d(num_features),
nn.Dropout(0.2),
nn.Linear(num_features, hidden_size3, bias=False),
nn.Tanh())
 self.l1_sigm = nn.Sequential(
 nn.BatchNorm1d(num_features),
nn.Dropout(0.2),
nn.Linear(num_features, hidden_size3, bias=False),
nn.RReLU())
 self.l1_nn = nn.Sequential(
 nn.BatchNorm1d(hidden_size3),
nn.LayerNorm(hidden_size3),
nn.Dropout(0.5),
nn.Linear(hidden_size3, hidden_size3, bias=False),
nn.RReLU())
 self.l1 = nn.Sequential(
 nn.BatchNorm1d(num_features),
nn.LayerNorm(num_features),
nn.Dropout(0.2),
nn.Linear(num_features, hidden_size1),
nn.PReLU(hidden_size1))
self.tanh = nn.Sequential(
 nn.BatchNorm1d(hidden_size1),
nn.Dropout(0.5),
nn.Linear(hidden_size1, hidden_size2, bias=False),
nn.Tanh())
 self.sigm = nn.Sequential(
 nn.BatchNorm1d(hidden_size1),
nn.Dropout(0.5),
nn.Linear(hidden_size1, hidden_size2, bias=False),
nn.RReLU())
 self.f_layer = nn.Sequential(
 nn.BatchNorm1d(hidden_size2 + hidden_size3),
nn.Dropout(0.3),
nn.RReLU() ,
nn.Linear(hidden_size2 + hidden_size3, num_targets)
)
 def forward(self, x):
x_tanh_l1 = self.l1_tanh(x)
 x_sigm_l1 = self.l1_sigm(x)
 x_att_l1 = x_tanh_l1 * x_sigm_l1
x_att_l1 = self.l1_nn(x_att_l1)
 x = self.l1(x)
 x_tanh = self.tanh(x)
 x_sigm = self.sigm(x)
 x = x_tanh * x_sigm
x = torch.cat(( x, x_att_l1), dim=1)
 x = self.f_layer(x)
 return x
oof_pred2 = pd.read_csv(f'{CACHE_PATH}/oof_pred.csv')[target_cols].values
npy_pred2 = np.zeros(( len(data), len(target_cols)))
 for seed in SEED:
if IS_TRAIN:
seed_npy_pred = MultilabelStratifiedKFold_run(
 model_fn=Model,
train_feat=data_feat,
train_labels=data_labels,
test_feat=None,
seed=seed,
model_name='mlp',
input_dim=len(feature_cols),
output_dim=len(target_cols),
nfolds=NFOLDS,
verbose=1,
isTrain=IS_TRAIN, 
)
 else:
seed_npy_pred = MultilabelStratifiedKFold_run(
 model_fn=Model,
train_feat=None,
train_labels=None,
test_feat=data_feat,
seed=seed,
model_name='mlp',
input_dim=len(feature_cols),
output_dim=len(target_cols),
nfolds=NFOLDS,
verbose=1,
isTrain=IS_TRAIN, 
)
 npy_pred2 += seed_npy_pred / len(SEED )<choose_model_class>
def l2_norm(grad):
norm = K.sqrt(K.sum(K.square(grad)))+ K.epsilon()
return norm
class OptimizerWrapper(optimizers.Optimizer):
def __init__(self, optimizer):
self.optimizer = optimizers.get(optimizer)
 self._optimizer_get_gradients = self.optimizer.get_gradients
def get_gradients(self, loss, params):
grads = self._optimizer_get_gradients(loss, params)
 return grads
@interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
self.optimizer.get_gradients = self.get_gradients
self.optimizer.get_updates(loss, params)
 self.optimizer.get_gradients = self._optimizer_get_gradients
return self.updates
def set_weights(self, weights):
self.optimizer.set_weights(weights)
 def get_weights(self):
return self.optimizer.get_weights()
def get_config(self):
config = {'optimizer_name': self.optimizer.__class__.__name__.lower() }
optimizer_config = {'optimizer_config': self.optimizer.get_config() }
return dict(list(optimizer_config.items())+ list(config.items()))
@property
def weights(self):
return self.optimizer.weights
@property
def updates(self):
return self.optimizer.updates
@classmethod
def from_config(cls, config):
raise NotImplementedError
@classmethod
def set_normalization_function(cls, name, func):
global _NORMS
_NORMS[name] = func
@classmethod
def get_normalization_functions(cls):
global _NORMS
return sorted(list(_NORMS.keys()))
class NormalizedOptimizer(OptimizerWrapper):
def __init__(self, optimizer, normalization='l2'):
super(NormalizedOptimizer, self ).__init__(optimizer)
 if normalization not in _NORMS:
raise ValueError('`normalization` must be one of %s.
 '
'Provided was "%s".' %(str(sorted(list(_NORMS.keys()))), normalization))
self.normalization = normalization
self.normalization_fn = _NORMS[normalization]
self.lr = K.variable(1e-3, name='lr')
 def get_gradients(self, loss, params):
grads = super(NormalizedOptimizer, self ).get_gradients(loss, params)
 grads = [grad / self.normalization_fn(grad)for grad in grads]
return grads
def get_config(self):
config = {'normalization': self.normalization}
base_config = super(NormalizedOptimizer, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
@classmethod
def from_config(cls, config):
optimizer_config = {'class_name': config['optimizer_name'],
'config': config['optimizer_config']}
optimizer = optimizers.get(optimizer_config)
 normalization = config['normalization']
return cls(optimizer, normalization=normalization)
 _NORMS = {
'l2': l2_norm,
}
get_custom_objects().update({'NormalizedOptimizer': NormalizedOptimizer} )
CACHE_PATH = '.. /input/mlp3-torch-weights1110'
hidden_size = 1024
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.BN0 = nn.BatchNorm1d(num_features)
 self.dropout0 = nn.Dropout(0.2)
 drop_rate = 0.4
self.dense1 = nn.Linear(num_features, 2048)
 self.BN1 = nn.BatchNorm1d(2048)
 self.dropout1 = nn.Dropout(drop_rate)
 self.dense2 = nn.Linear(2048, 2048)
 self.BN2 = nn.BatchNorm1d(2048)
 self.dropout2 = nn.Dropout(drop_rate)
 self.dense3 = nn.Linear(2048, 2048)
 self.BN3 = nn.BatchNorm1d(2048)
 self.dropout3 = nn.Dropout(drop_rate)
 self.dense4 = nn.Linear(2048, num_targets)
 self.Relu = nn.ReLU(inplace=True)
 self.PReLU = nn.PReLU()
def forward(self, inp):
x = self.BN0(inp)
 x = self.dropout0(x)
 x = self.dense1(x)
 x = self.PReLU(x)
 x = self.BN1(x)
 x = self.dropout1(x)
 x = self.dense2(x)
 x = self.PReLU(x)
 x = self.BN2(x)
 x = self.dropout2(x)
 x = self.dense3(x)
 x = self.PReLU(x)
 x = self.BN3(x)
 x = self.dropout3(x)
 x = self.dense4(x)
 return x
oof_pred3 = pd.read_csv(f'{CACHE_PATH}/oof_pred.csv')[target_cols].values
npy_pred3 = np.zeros(( len(data), len(target_cols)))
 for seed in SEED:
if IS_TRAIN:
seed_npy_pred = MultilabelStratifiedKFold_run(
 model_fn=Model,
train_feat=data_feat,
train_labels=data_labels,
test_feat=None,
seed=seed,
model_name='mlp',
input_dim=len(feature_cols),
output_dim=len(target_cols),
nfolds=NFOLDS,
verbose=1,
isTrain=IS_TRAIN, 
)
 else:
seed_npy_pred = MultilabelStratifiedKFold_run(
 model_fn=Model,
train_feat=None,
train_labels=None,
test_feat=data_feat,
seed=seed,
model_name='mlp',
input_dim=len(feature_cols),
output_dim=len(target_cols),
nfolds=NFOLDS,
verbose=1,
isTrain=IS_TRAIN, 
)
 npy_pred3 += seed_npy_pred / len(SEED )<choose_model_class>
class SWA(keras.callbacks.Callback):
def __init__(self, filepath, swa_epoch):
super(SWA, self ).__init__()
self.filepath = filepath
self.swa_epoch = swa_epoch
def on_train_begin(self, logs=None):
self.nb_epoch = self.params['epochs']
print('Stochastic weight averaging selected for last {} epochs.' 
.format(self.nb_epoch - self.swa_epoch))
def on_epoch_end(self, epoch, logs=None):
if epoch == self.swa_epoch:
self.swa_weights = self.model.get_weights()
elif epoch > self.swa_epoch:
for i in range(len(self.swa_weights)) :
self.swa_weights[i] =(self.swa_weights[i] * 
(epoch - self.swa_epoch)+ self.model.get_weights() [i])/(( epoch - self.swa_epoch)+ 1)
 else:
pass
def on_train_end(self, logs=None):
self.model.set_weights(self.swa_weights)
 print('Final model parameters set to stochastic weight average.')
 self.model.save_weights(self.filepath)
 print('Final stochastic averaged weights saved to file.' )
CACHE_PATH = '.. /input/lstm-torch1116-weights'
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
hidden_size = 200
self.Lstm = nn.LSTM(num_features, hidden_size // 2, bidirectional=True, batch_first=True, dropout=0.2)
 self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.4)
 self.dense3 = nn.Linear(hidden_size, num_targets)
 self.Relu = nn.ReLU(inplace=True)
 self.PReLU = nn.PReLU()
self.LeakyReLU = nn.LeakyReLU(negative_slope=0.01, inplace=True)
 self.GeLU = nn.GELU()
self.RReLU = nn.RReLU()
self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.batch_norm1(x)
 inp = x.unsqueeze(-2)
 self.Lstm.flatten_parameters()
lstm, _ = self.Lstm(inp)
 max_pool, _ = torch.max(lstm, 1)
 avg_pool = torch.mean(lstm, 1)
 pooled_output = self.dropout(max_pool)
 pooled_output = self.RReLU(pooled_output)
 x = self.batch_norm3(pooled_output)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
oof_pred4 = pd.read_csv(f'{CACHE_PATH}/oof_pred.csv')[target_cols].values
npy_pred4 = np.zeros(( len(data), len(target_cols)))
 for seed in SEED:
if IS_TRAIN:
seed_npy_pred = MultilabelStratifiedKFold_run(
 model_fn=Model,
train_feat=data_feat,
train_labels=data_labels,
test_feat=None,
seed=seed,
model_name='mlp',
input_dim=len(feature_cols),
output_dim=len(target_cols),
nfolds=NFOLDS,
verbose=1,
isTrain=IS_TRAIN, 
)
 else:
seed_npy_pred = MultilabelStratifiedKFold_run(
 model_fn=Model,
train_feat=None,
train_labels=None,
test_feat=data_feat,
seed=seed,
model_name='mlp',
input_dim=len(feature_cols),
output_dim=len(target_cols),
nfolds=NFOLDS,
verbose=1,
isTrain=IS_TRAIN, 
)
 npy_pred4 += seed_npy_pred / len(SEED )<normalization>
sgd = SGD(0.01, momentum=0.9, nesterov=True)
 sgd = NormalizedOptimizer(sgd, normalization='l2' )
CACHE_PATH = '.. /input/cnn-torch1120-weights'
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class SEModule(nn.Module):
def __init__(self, in_channels, out_channels, kernel_size, padding, reduction=2):
super(SEModule, self ).__init__()
self.conv0 = nn.Conv1d(in_channels, out_channels, kernel_size=kernel_size, padding=padding)
 self.conv1 = nn.Conv1d(in_channels, in_channels//reduction, kernel_size=kernel_size, padding=padding)
 self.conv2 = nn.Conv1d(in_channels//reduction, out_channels, kernel_size=kernel_size, padding=padding)
 def forward(self, x):
s = F.adaptive_avg_pool1d(x, 1)
 s = self.conv0(s)
 x *= torch.sigmoid(s)
 return x
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 conv_size = 128
dropout_rate = 0.2
self.conv1d_1 = nn.Conv1d(num_features, conv_size, kernel_size=1, stride=1, dilation=1, padding=0, padding_mode='replicate')
 self.batch_norm_conv_1 = nn.BatchNorm1d(conv_size)
 self.dropout_conv_1 = nn.Dropout(dropout_rate)
 self.conv1d_2 = nn.Conv1d(num_features, conv_size, kernel_size=3, stride=1, dilation=1, padding=1, padding_mode='replicate')
 self.batch_norm_conv_2 = nn.BatchNorm1d(conv_size)
 self.dropout_conv_2 = nn.Dropout(dropout_rate)
 self.conv1d_3 = nn.Conv1d(num_features, conv_size, kernel_size=5, stride=1, dilation=1, padding=2, padding_mode='replicate')
 self.batch_norm_conv_3 = nn.BatchNorm1d(conv_size)
 self.dropout_conv_3 = nn.Dropout(dropout_rate)
 self.conv1d_4 = nn.Conv1d(num_features, conv_size, kernel_size=15, stride=1, dilation=1, padding=7, padding_mode='replicate')
 self.batch_norm_conv_4 = nn.BatchNorm1d(conv_size)
 self.dropout_conv_4 = nn.Dropout(dropout_rate)
 self.conv1d_5 = nn.Conv1d(num_features, conv_size, kernel_size=31, stride=1, dilation=1, padding=15, padding_mode='replicate')
 self.batch_norm_conv_5 = nn.BatchNorm1d(conv_size)
 self.dropout_conv_5 = nn.Dropout(dropout_rate)
 self.batch_norm3 = nn.BatchNorm1d(conv_size * 5)
 self.dropout3 = nn.Dropout(0.5)
 self.dense3 = nn.Linear(conv_size * 5, num_targets)
 self.ReLU = nn.ReLU(inplace=True)
 self.PReLU = nn.PReLU()
self.LeakyReLU = nn.LeakyReLU(negative_slope=0.01, inplace=True)
 self.RReLU = nn.RReLU()
self.dropout = nn.Dropout(0.2)
 self.spatial_dropout = SpatialDropout(0.2)
 def forward(self, x):
x = x.unsqueeze(-1)
 x1 = self.conv1d_1(x)
 x1 = F.relu(x1)
 x1 = self.batch_norm_conv_1(x1)
 x1 = self.dropout_conv_1(x1)
 x1 = x1.squeeze(-1)
 x2 = self.conv1d_2(x)
 x2 = F.relu(x2)
 x2 = self.batch_norm_conv_2(x2)
 x2 = self.dropout_conv_2(x2)
 x2 = x2.squeeze(-1)
 x3 = self.conv1d_3(x)
 x3 = F.relu(x3)
 x3 = self.batch_norm_conv_3(x3)
 x3 = self.dropout_conv_3(x3)
 x3 = x3.squeeze(-1)
 x4 = self.conv1d_4(x)
 x4 = F.relu(x4)
 x4 = self.batch_norm_conv_4(x4)
 x4 = self.dropout_conv_4(x4)
 x4 = x4.squeeze(-1)
 cross1 = x1 * x2
conv_output = torch.cat(( x1, x2, x3, x4, cross1), 1)
 x = self.batch_norm3(conv_output)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
oof_pred5 = pd.read_csv(f'{CACHE_PATH}/oof_pred.csv')[target_cols].values
npy_pred5 = np.zeros(( len(data), len(target_cols)))
 for seed in SEED:
if IS_TRAIN:
seed_npy_pred = MultilabelStratifiedKFold_run(
 model_fn=Model,
train_feat=data_feat,
train_labels=data_labels,
test_feat=None,
seed=seed,
model_name='mlp',
input_dim=len(feature_cols),
output_dim=len(target_cols),
nfolds=NFOLDS,
verbose=1,
isTrain=IS_TRAIN, 
)
 else:
seed_npy_pred = MultilabelStratifiedKFold_run(
 model_fn=Model,
train_feat=None,
train_labels=None,
test_feat=data_feat,
seed=seed,
model_name='mlp',
input_dim=len(feature_cols),
output_dim=len(target_cols),
nfolds=NFOLDS,
verbose=1,
isTrain=IS_TRAIN, 
)
 npy_pred5 += seed_npy_pred / len(SEED )<compute_test_metric>
model.compile(loss=bce_dice_loss, optimizer=sgd, metrics=[my_iou_metric] )
oof_pred = oof_pred1 * 0.3 + oof_pred2 * 0.25 + oof_pred3 * 0.2 + oof_pred4 * 0.15 + oof_pred5 * 0.1
train_targets_scored = pd.read_csv(f'{DATA_PATH}/train_targets_scored.csv')
 y_true = train_targets_scored[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], oof_pred[:, i])
 score += score_ / len(target_cols)
 print("OOF log_loss: ", score )<save_to_csv>
x_train = np.append(x_train, [np.fliplr(x)for x in x_train[0:3200]], axis=0)
 y_train = np.append(y_train, [np.fliplr(x)for x in y_train[0:3200]], axis=0)
 x_train = np.repeat(x_train,3,axis=3)
 x_valid = np.repeat(x_valid,3,axis=3 )
npy_pred = npy_pred1 * 0.3 + npy_pred2 * 0.25 + npy_pred3 * 0.2 + npy_pred4 * 0.15 + npy_pred5 * 0.1
test_pred = pd.DataFrame(npy_pred, columns=target_cols)
 test_pred[id_col] = data[id_col].values
sub = sample_submission.drop(columns=target_cols ).merge(test_pred[['sig_id'] + target_cols], on='sig_id',
how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False)
 print(sub.shape )<set_options>
epochs = 40
snapshot = SnapshotCallbackBuilder(nb_epochs=epochs,nb_snapshots=1,init_lr=1e-3)
 batch_size = 32
swa = SWA('./keras_swa.model',35)
 history = model.fit(x_train, y_train,
validation_data=[x_valid, y_valid],
epochs=epochs,
batch_size=batch_size,
callbacks=snapshot.get_callbacks() ,shuffle=True,verbose=2 )
warnings.filterwarnings('ignore')
 pd.set_option('max_columns', 2000 )<load_from_csv>
del ids_train, x_train, y_train,some_salt_ids,depths_df
gc.collect()
train_features = pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv' )<define_search_space>
try:
print('using swa weight model')
 model.load_weights('./keras_swa.model')
 except:
model.load_weights('./keras.model' )
n_comp_GENES = int(0.6*610)
 n_comp_CELLS = int(0.6*55)
 VarianceThreshold_for_FS = 0.82<define_variables>
def predict_result(model,x_test,img_size_target,batch_size):
preds_test1 = model.predict([x_test],batch_size=batch_size ).reshape(-1, img_size_target, img_size_target)
 x_test_reflect2 = np.array([np.fliplr(x)for x in x_test])
 preds_test2_refect = model.predict([x_test_reflect2],batch_size=batch_size ).reshape(-1, img_size_target, img_size_target)
 preds_test2 = np.array([ np.fliplr(x)for x in preds_test2_refect])
 x_test_reflect3 = np.array([np.flipud(x)for x in x_test])
 preds_test3_refect = model.predict([x_test_reflect3],batch_size=batch_size ).reshape(-1, img_size_target, img_size_target)
 preds_test3 = np.array([ np.flipud(x)for x in preds_test3_refect])
 x_test_reflect4 = np.array([np.fliplr(np.flipud(x)) for x in x_test])
 preds_test4_refect = model.predict([x_test_reflect4],batch_size=batch_size ).reshape(-1, img_size_target, img_size_target)
 preds_test4 = np.array([ np.fliplr(np.flipud(x)) for x in preds_test4_refect])
 preds_avg =(preds_test1+preds_test2+preds_test3+preds_test4)/4
return preds_avg
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<concatenate>
preds_valid = predict_result(model,x_valid,img_size_target,batch_size)
 preds_valid = np.array([downsample(x)for x in preds_valid])
 y_valid_ori = np.array([downsample(x)for x in y_valid] )
train_len = len(train_features.values)
 test_len = len(test_features.values)
 for col in(GENES + CELLS):
epsilon = 1e-6
train_1col = train_features[col].values.reshape(train_len, 1)
 test_1col = test_features[col].values.reshape(test_len, 1)
 np_arr_total = np.vstack(( train_1col, test_1col))
res = np_arr_total[:,0].argsort().argsort()
res =(res/res.max() -0.5)*2
res = np.clip(res,-1+epsilon,1-epsilon)
 res = erfinv(res)
 np_arr_total_r = np_arr_total
np_arr_total_r[:,0] = res * np.sqrt(2)
 train_features[col] =(np_arr_total_r[:train_len] ).reshape(1, train_len)[0]
test_features[col] =(np_arr_total_r[train_len:] ).reshape(1, test_len)[0]<set_options>
thresholds = np.linspace(0.3, 0.7, 31)
 ious = np.array([iou_metric_batch(y_valid_ori, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)] )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<create_dataframe>
del x_valid, y_valid,preds_valid,y_valid_ori,train_df
gc.collect()
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp_GENES, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<concatenate>
train_df = pd.read_csv(".. /input/tgs-salt-identification-challenge/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/tgs-salt-identification-challenge/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp_CELLS, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
batch_size = 500
preds_test = []
i = 0
while i < test_df.shape[0]:
index_val = test_df.index[i:i+batch_size]
x_test = np.array([upsample(np.array(load_img("./test/images/{}.png".format(idx), grayscale=True)))/ 255 for idx in(index_val)] ).reshape(-1, img_size_target, img_size_target, 1)
 x_test = np.repeat(x_test,3,axis=3)
 preds_test_temp = predict_result(model,x_test,img_size_target,32)
 if i==0:
preds_test = preds_test_temp
else:
preds_test = np.concatenate([preds_test,preds_test_temp],axis=0)
 if i%2000==0:
print('Images Processed:',i)
 i += batch_size
print('Done!' )
var_thresh = VarianceThreshold(VarianceThreshold_for_FS)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape<merge>

 def rle_encode(im):

 pixels = im.flatten(order = 'F')
 pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]<drop_column>
t1 = time.time()
pred_dict = {idx: rle_encode(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }
t2 = time.time()
print(f"Usedtime = {t2-t1} s" )
<drop_column><EOS> 
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('swa_xce_submission.csv' )