Kaggle competition: Mechanisms of Action (MoA) Prediction
label_cols = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
p_res = p_lstm.copy()
p_res[label_cols] =(1*p_tfidf[label_cols] + 2*p_lstm[label_cols])/ 3<save_to_csv>
warnings.filterwarnings('ignore')
 def seed_everything(seed=1903):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
p_res.to_csv('submission_ensemble.csv', index=False )<define_variables>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
count = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 scheduler.step()
return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
f_lstm = '.. /input/improved-lstm-baseline-glove-dropout/submission.csv'
f_nbsvm = '.. /input/nb-svm-strong-linear-baseline/submission.csv'<load_from_csv>
data_loc = ".. /input/lish-moa/"
train_features = pd.read_csv(data_loc + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_loc + 'train_targets_scored.csv')
 test_features = pd.read_csv(data_loc + 'test_features.csv')
 target_cols = [x for x in train_targets_scored.columns.to_list() if x != 'sig_id']
print(train_features.shape, test_features.shape )
p_lstm = pd.read_csv(f_lstm)
 p_nbsvm = pd.read_csv(f_nbsvm )<feature_engineering>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
label_cols = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
p_res = p_lstm.copy()
p_res[label_cols] =(p_nbsvm[label_cols] + p_lstm[label_cols])/ 2<save_to_csv>
feature_to_scale = ["cp_time"] + GENES + CELLS
print(len(feature_to_scale))
data_scaler = MinMaxScaler()
data_entire[feature_to_scale ] = data_scaler.fit_transform(data_entire[feature_to_scale ] )
p_res.to_csv('submission.csv', index=False )<import_modules>
print(data_entire.shape)
 cols_ohe = ['cp_type', 'cp_dose']
data_entire = ohe(data_entire, cols_ohe)
import pandas as pd
import numpy as np
from tqdm import tqdm<define_variables>
data_train = data_entire.iloc[0:23814, :].copy(deep=True)
 data_test = data_entire.iloc[23814:, :].copy(deep=True )
INPUT = '.. /input/unet34-submission-tta-0-699-new-public-lb/submission.csv'
OUTPUT = 'submission.csv'<categorify>
tmp_data_dir = "/kaggle/working/tmp_data/"
try:
os.makedirs(tmp_data_dir)
 except FileExistsError:
pass
data_train.to_pickle(tmp_data_dir + "data_train.pkl")
 data_test.to_pickle(tmp_data_dir + "data_test.pkl" )
def get_mask(img_id, df, shape =(768,768)) :
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 px = df.loc[img_id]['EncodedPixels']
if(type(px)== float): return None
elif(type(px)== str): px = [px]
count = 1
for mask in px:
if(type(mask)== float):
if len(px)== 1: return None
else: continue
s = mask.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = count*(img[start:start+length] == 0)
 count+=1
return img.reshape(shape ).T
def decode_mask(mask, shape=(768, 768)) :
pixels = mask.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
if(len(runs)== 0): return np.nan
runs[runs > shape[0]*shape[1]] = shape[0]*shape[1]
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def set_masks(mask):
n = mask.max()
result = []
for i in range(1,n+1):
result.append(decode_mask(mask == i))
return result<categorify>
seed_everything(seed=1903)
 tmp_data_dir = "/kaggle/working/tmp_data/"
data_train_x = pd.read_pickle(tmp_data_dir + "data_train.pkl")
 data_test_x = pd.read_pickle(tmp_data_dir + "data_test.pkl")
 target_cols = [x for x in train_targets_scored.columns.to_list() if x != 'sig_id']
feature_cols = [c for c in data_train_x.columns.to_list() if c != 'sig_id']
names = list(set(pred_df.index))
ship_list_dict = []
for name in tqdm(names):
mask = get_mask(name, pred_df)
 if(not isinstance(mask, np.ndarray)and mask == None)\
or mask.sum() == 0:
ship_list_dict.append({'ImageId':name,'EncodedPixels':np.nan})
 else:
encodings = set_masks(mask)
 if(len(encodings)== 0):
ship_list_dict.append({'ImageId':name,'EncodedPixels':np.nan})
 continue
buf =[]
for e in encodings:
if e == e: buf.append(e)
 encodings = buf
if len(encodings)== 0 : encodings = [np.nan]
for encoding in encodings:
ship_list_dict.append({'ImageId':name,'EncodedPixels':encoding} )<save_to_csv>
train_entire = data_train_x.merge(train_targets_scored, on="sig_id")
 NFOLDS = 5
seed = 1903
mskf = MultilabelStratifiedKFold(n_splits=NFOLDS, random_state=seed)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=data_train_x, y=train_targets_scored[target_cols])) :
train_entire.loc[v_idx, 'kfold'] = int(f)
 train_entire['kfold'] = train_entire['kfold'].astype(int )
pred_df_cor = pd.DataFrame(ship_list_dict)
 pred_df_cor.to_csv(OUTPUT, index=False )<groupby>
train_entire.to_pickle(tmp_data_dir + "data_train_process2_5fold_1903.pkl" )
gr = pred_df_cor.groupby("ImageId")["EncodedPixels"].apply(lambda x: x.isnull().any() and len(x)> 1)
 print(gr.value_counts())
 print(gr[gr] )<import_modules>
def run_training(train, valid, feature_cols, target_cols, model_path, seed, param_provided=None):
seed_everything(seed)
 x_train, y_train = train[feature_cols].values, train[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 if valid is not None:
x_valid, y_valid = valid[feature_cols].values, valid[target_cols].values
valid_dataset = MoADataset(x_valid, y_valid)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 if param_provided is not None:
EPOCHS = param_provided['epoch']
hidden_size = param_provided['hidden_size']
LEARNING_RATE = param_provided['lr']
print("hidden_size: ", hidden_size, ", learning_rate: ", LEARNING_RATE)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=4, gamma=0.5)
 loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
best_loss = np.inf
train_losses = []; valid_losses = []
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer, scheduler, loss_fn, trainloader, DEVICE)
 train_losses.append(train_loss)
 if valid is not None:
valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 valid_losses.append(valid_loss)
 if epoch % 5 == 0:
print(f"EPOCH: {epoch}, train_loss: {train_loss}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
torch.save(model.state_dict() , model_path)
 elif EARLY_STOP == True:
early_step += 1
if early_step >= early_stopping_steps:
break
else:
if epoch % 10 == 0:
print(f"EPOCH: {epoch}, train_loss: {train_loss}")
 print("early stop with epoch: ", epoch)
 print(f"LAST EPOCH: {epoch}, train_loss: {train_loss}")
 if valid is None:
torch.save(model.state_dict() , model_path)
 return {"train_losses": train_losses, "valid_losses": valid_losses}
from fastai.conv_learner import *
from fastai.dataset import *
import pandas as pd
import numpy as np
import os
from PIL import Image
from sklearn.model_selection import train_test_split
from tqdm import tnrange, tqdm_notebook
from scipy import ndimage<define_variables>
def run_training_tune(trial, train, valid, feature_cols, target_cols, model_path, seed):
seed_everything(seed)
 x_train, y_train = train[feature_cols].values, train[target_cols].values
x_valid, y_valid = valid[feature_cols].values, valid[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 hidden_size = trial.suggest_int("hidden_size", 200, 500, step=50)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 lr = trial.suggest_float("lr", 1e-5, 1e-1, log=True)
 optimizer = torch.optim.Adam(model.parameters() , lr=lr, weight_decay=WEIGHT_DECAY)
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=4, gamma=0.5)
 loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
best_loss = np.inf
train_losses = []; valid_losses = []
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer, scheduler, loss_fn, trainloader, DEVICE)
 train_losses.append(train_loss)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 valid_losses.append(valid_loss)
 if epoch % 5 == 0:
print(f"EPOCH: {epoch}, train_loss: {train_loss}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
elif EARLY_STOP == True:
early_step += 1
if early_step >= early_stopping_steps:
break
return valid_losses[-1]
PATH = './'
TRAIN = '.. /input/airbus-ship-detection/train/'
TEST = '.. /input/airbus-ship-detection/test/'
SEGMENTATION = '.. /input/airbus-ship-detection/train_ship_segmentations.csv'
PRETRAINED_DETECTION_PATH = '.. /input/fine-tuning-resnet34-on-ship-detection/models/'
PRETRAINED_SEGMENTATION_PATH = '.. /input/unet34-dice-0-87/models/'
DETECTION_TEST_PRED = '.. /input/fine-tuning-resnet34-on-ship-detection/ship_detection.csv'
exclude_list = ['6384c3e78.jpg','13703f040.jpg', '14715c06d.jpg', '33e0ff2d5.jpg',
'4d4e09f2a.jpg', '877691df8.jpg', '8b909bb20.jpg', 'a8d99130e.jpg',
'ad55c3143.jpg', 'c8260c541.jpg', 'd6c7f17c7.jpg', 'dc3e7c901.jpg',
'e44dffe88.jpg', 'ef87bad36.jpg', 'f083256d8.jpg']<choose_model_class>
def print_model_results(train_loss, valid_loss, train_len, valid_len):
total_num_inst = train_len + valid_len
train_loss_final = train_loss
valid_loss_final = valid_loss
entire_train_loss = train_len / total_num_inst * train_loss_final \
+ valid_len / total_num_inst * valid_loss_final
print("train loss: ", train_loss_final, " valid loss: ", valid_loss_final,
"entire loss: ", entire_train_loss )
nw = 2
arch = resnet34<load_from_csv>
start_time = datetime.datetime.now()
time.sleep(5)
 end_time = datetime.datetime.now()
print("time elapsed: ", end_time - start_time )
train_names = [f for f in os.listdir(TRAIN)]
test_names = [f for f in os.listdir(TEST)]
for el in exclude_list:
if(el in train_names): train_names.remove(el)
 if(el in test_names): test_names.remove(el)
 tr_n, val_n = train_test_split(train_names, test_size=0.05, random_state=42)
 segmentation_df = pd.read_csv(os.path.join(PATH, SEGMENTATION)).set_index('ImageId' )<categorify>
def cut_empty(names):
return [name for name in names
if(type(segmentation_df.loc[name]['EncodedPixels'])!= float)]
tr_n_cut = cut_empty(tr_n)
 val_n_cut = cut_empty(val_n )<categorify>
best_param_with_epoch = {'kfold_0': {'hidden_size': 450, 'lr': 0.0008925929114331024, 'epoch': 30},
'kfold_1': {'hidden_size': 400, 'lr': 0.0051852480304641155, 'epoch': 17},
'kfold_2': {'hidden_size': 200, 'lr': 0.0020302278497303823, 'epoch': 30},
'kfold_3': {'hidden_size': 300, 'lr': 0.0029643441803447576, 'epoch': 20},
'kfold_4': {'hidden_size': 250, 'lr': 0.006931570763035235, 'epoch': 16}}
def get_mask(img_id, df):
shape =(768,768)
 img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 masks = df.loc[img_id]['EncodedPixels']
if(type(masks)== float): return img.reshape(shape)
 if(type(masks)== str): masks = [masks]
for mask in masks:
s = mask.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1
return img.reshape(shape ).T<prepare_x_and_y>
seed_everything(seed=1903)
 DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 print(DEVICE)
 EPOCHS = 100
BATCH_SIZE = 64
LEARNING_RATE = 1e-2
WEIGHT_DECAY = 1e-5
EARLY_STOPPING_STEPS = 11
EARLY_STOP = True
num_features = len(feature_cols)
 num_targets = len(target_cols)
 hidden_size = 50
if_tune = False
SEED = [1903]
prefix = "process_2"
models_dir = "/kaggle/working/trained_models/"
try:
os.makedirs(models_dir)
 except FileExistsError:
pass
for seed in SEED:
NFOLDS = 5
train_entire = pd.read_pickle(tmp_data_dir + "data_train_process2_5fold_1903.pkl")
 best_param = {}
for k_fold in np.arange(NFOLDS):
print("runing fold ", k_fold)
 train = train_entire
valid = None
data_test_x = pd.read_pickle(tmp_data_dir + "data_test.pkl")
 model_path = models_dir + prefix + "_model_fold_{0:d}.pth".format(k_fold)
 model_result = None
if if_tune:
start_time = datetime.datetime.now()
study = optuna.create_study(direction="minimize")
 study.optimize(lambda trial: run_training_tune(trial, train, valid, feature_cols, target_cols, model_path,
seed), n_trials=20)
 pruned_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.PRUNED]
complete_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]
print("Study statistics: ")
 print(" Number of finished trials: ", len(study.trials))
print(" Number of pruned trials: ", len(pruned_trials))
print(" Number of complete trials: ", len(complete_trials))
print("Best trial:")
 trial = study.best_trial
print(" Value: ", trial.value)
 print(" Params: ")
 for key, value in trial.params.items() :
print(" {}: {}".format(key, value))
best_param["kfold_" + str(k_fold)] = trial.params
end_time = datetime.datetime.now()
print("time elapsed: ", end_time - start_time)
 else:
start_time = datetime.datetime.now()
model_result = run_training(train, valid, feature_cols, target_cols,
model_path, seed,
param_provided = best_param_with_epoch["kfold_" + str(k_fold)])
 end_time = datetime.datetime.now()
print("time elapsed: ", end_time - start_time)
 if valid is None:
print("loss on the entire training data", model_result["train_losses"][-1])
 else:
print_model_results(model_result["train_losses"][-1], model_result["valid_losses"][-1],
train.shape[0], valid.shape[0] )
class pdFilesDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.segmentation_df = pd.read_csv(SEGMENTATION ).set_index('ImageId')
 super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_image(os.path.join(self.path, self.fnames[i]))
if self.sz == 768: return img
else: return cv2.resize(img,(self.sz, self.sz))
def get_y(self, i):
mask = np.zeros(( 768,768), dtype=np.uint8)if(self.path == TEST)\
else get_mask(self.fnames[i], self.segmentation_df)
 img = Image.fromarray(mask ).resize(( self.sz, self.sz)).convert('RGB')
 return np.array(img ).astype(np.float32)
 def get_c(self): return 0<create_dataframe>
x_test = data_test_x[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 for k_fold in np.arange(NFOLDS):
if best_param_with_epoch is not None:
hidden_size = best_param_with_epoch["kfold_" + str(k_fold)]['hidden_size']
model_path = models_dir + prefix + "_model_fold_{0:d}.pth".format(k_fold)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(model_path))
model.to(DEVICE)
 predictions = inference_fn(model, testloader, DEVICE)
 pred_test = pd.DataFrame(data=predictions, columns=target_cols ).fillna(0)
 pred_test = pd.concat([data_test_x[["sig_id"]], pred_test], axis=1)
 pred_result_f = models_dir + prefix + "_fold_{0:d}_prediction.csv".format(k_fold)
 pred_test.to_csv(pred_result_f, index=False )
def get_data(sz,bs):
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.CLASS)
 tr_names = tr_n if(len(tr_n_cut)%bs == 0)else tr_n[:-(len(tr_n_cut)%bs)]
ds = ImageData.get_ds(pdFilesDataset,(tr_names,TRAIN), 
(val_n_cut,TRAIN), tfms, test=(test_names,TEST))
md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)
 md.is_multi = False
return md<find_best_params>
folds_to_use = np.arange(NFOLDS)
 for i_count, k_fold in enumerate(folds_to_use):
pred_result_f = models_dir + prefix + "_fold_{0:d}_prediction.csv".format(k_fold)
 pred_test = pd.read_csv(pred_result_f, header=0)
 pred_test[target_cols] = pred_test[target_cols]/len(folds_to_use)
 if i_count == 0:
pred_test_average = pred_test.copy(deep=True)
 else:
pred_test_average[target_cols] = pred_test_average[target_cols] + pred_test[target_cols]
<choose_model_class><EOS> 
sample_submission = pd.read_csv(".. /input/lish-moa/" + 'sample_submission.csv')
 sub = sample_submission.drop(columns=target_cols ).merge(pred_test_average, on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )