Kaggle competition: Mechanisms of Action (MoA) Prediction
for col in df.columns.drop('target'):
unknowns = set(test[col].astype(str ).unique())- set(df[col].astype(str ).unique())
 print('The predictor ', col, ' contains ', len(unknowns), ' value present only in the test set.' )<split>
%matplotlib inline
start = datetime.now()
train_scores = []
val_scores = []
y_validations = []
y_predictions = []
X_nb = df.drop(['bin_3', 'nom_6', 'target'], axis=1 ).astype(str)
 for index_train, index_val in StratifiedKFold(n_splits=5, random_state=42, shuffle=True ).split(X_nb, y):
X_train = X_nb.iloc[index_train].copy()
y_train = y.iloc[index_train].copy()
X_valid = X_nb.iloc[index_val].copy()
y_valid = y.iloc[index_val].copy()
for col in ['nom_5', 'nom_7', 'nom_8', 'nom_9']:
rare_categories = X_train[col].value_counts().iloc[np.where(X_train[col].value_counts() <100)].index
X_train.loc[:,col] = X_train.loc[:,col].replace(rare_categories, 'nan')
 X_valid.loc[:,col] = X_valid.loc[:,col].replace(rare_categories, 'nan')
 oe = OrdinalEncoder()
X_train_oe = oe.fit_transform(X_train)
 X_val_oe = oe.transform(X_valid)
 X_train_oe += 2
X_val_oe += 2
nb = CategoricalNB(alpha = 3.4)
 nb.fit(X_train_oe, y_train)
 y_pred = nb.predict_proba(X_train_oe)
 train_score = roc_auc_score(y_train, y_pred[:,1])
 train_scores.append(train_score)
 y_pred = nb.predict_proba(X_val_oe)
 val_score = roc_auc_score(y_valid, y_pred[:,1])
 val_scores.append(val_score)
 y_validations.append(y_valid)
 y_predictions.append(y_pred[:,1])
 display_scores(train_scores, val_scores, y_validations, y_predictions)
 runtime = datetime.now() - start
print('
Runtime ', runtime )<categorify>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
oe = OrdinalEncoder()
X_nb = df.drop(['bin_3', 'nom_6', 'target'], axis=1 ).astype(str)
 X_test_nb = test.drop(['bin_3', 'nom_6'], axis=1 ).astype(str)
 for col in ['nom_5', 'nom_7', 'nom_8', 'nom_9']:
rare_categories = X_nb[col].value_counts().iloc[np.where(X_nb[col].value_counts() <100)].index
X_nb[col] = X_nb[col].replace(rare_categories, 'nan')
 X_test_nb[col] = X_test_nb[col].replace(rare_categories, 'nan')
 X_oe = oe.fit_transform(X_nb)
 X_test_oe = oe.transform(X_test_nb)
 X_oe += 2
X_test_oe += 2<train_on_grid>
def preprocess(df):

 df = df.copy()
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 df.loc[:, 'cp_time'] = df.loc[:, 'cp_time'].map({24: 0, 48: 1, 72: 2})
 del df['sig_id']
return df
start = datetime.now()
nb = CategoricalNB(alpha = 3.4)
 nb.fit(X_oe, df.target)
 y_pred = nb.predict_proba(X_oe)
 train_score = roc_auc_score(df.target, y_pred[:,1])
 print('AUROC on the entire training set: ', train_score)
 runtime = datetime.now() - start
print('
Runtime ', runtime )<predict_on_test>
train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']
y_test = nb.predict_proba(X_test_oe)[:,1]
output = pd.DataFrame({'id':sample_submission.index, 'target':y_test})
 output<save_to_csv>
train_targets = train_targets.loc[train['cp_type']==0].reset_index(drop=True)
 train = train.loc[train['cp_type']==0].reset_index(drop=True )
output.to_csv('output_nb_3rd_strategy.csv', index=False )<load_from_csv>
scaler = MinMaxScaler()
scaler.fit(train)
 train = scaler.transform(train)
 test = scaler.transform(test )
output_gbrf = pd.read_csv('output_gbrf_1st_strategy.csv')
 output_lr = pd.read_csv('output_lr_2nd_strategy.csv')
 output_nb = pd.read_csv('output_nb_3rd_strategy.csv' )<feature_engineering>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return self.features.shape[0]
def __getitem__(self, idx):
return {
'input': torch.tensor(self.features[idx, :], dtype=torch.float),
'target': torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return self.features.shape[0]
def __getitem__(self, idx):
return {
'input': torch.tensor(self.features[idx, :], dtype=torch.float)
 }
mean = output_nb
mean.target =(output_gbrf.target * 0.78560 + output_lr.target * 0.78552 + output_nb.target * 0.78477)/(0.78560 + 0.78552 + 0.78477 )<save_to_csv>
import torch
import torch.nn as nn
import torch.nn.functional as F
mean.to_csv('ouput_ensemble.csv', index=False )<install_modules>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
device = 'cuda' if torch.cuda.is_available() else 'cpu'
self.weight = torch.tensor([0.5] ).to(device)
 def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.prelu(self.dense1(x), self.weight)
 x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.prelu(self.dense2(x), self.weight)
 x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
os.mkdir("./working")
 os.environ['USER'] = 'root'
os.system('pip install.. /input/xlearn/xlearn/xlearn-0.40a1/')
<set_options>
train_targets = train_targets.values
warnings.filterwarnings('ignore')
 pd.options.display.max_columns = 50
BIN_COL = [f'bin_{i}' for i in range(5)]
NOM_COL = [f'nom_{i}' for i in range(10)]
ORD_COL = [f'ord_{i}' for i in range(6)]
NOM_5_9 = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']
NOM_0_4 = ['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4']
DATE_COL = ['day','month']
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
for dirname, _, filenames in os.walk('/kaggle/working/libffm-binaries'):
for filename in filenames:
print(os.path.join(dirname, filename))<load_from_csv>
def set_seed(seed):
os.environ['PYTHONHASHSEED'] = str(seed)
 torch.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(seed )
%%time
def read_csv() :
train = pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat-ii/test.csv')
 train_id = train['id']
test_id = test['id']
train.drop('id', axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 return train, test, train_id, test_id
train, test, train_id, test_id = read_csv()
def preprocessing(df):
df['bin_missing'] =(df[['bin_0', 'bin_1', 'bin_2', 'bin_4']].isnull().sum(axis=1)>0 ).replace({True:1, False:0})
 df['nom_missing'] =(df[NOM_COL].isnull().sum(axis=1)>0 ).replace({True:1, False:0})
 df['ord_missing'] =(df[ORD_COL].isnull().sum(axis=1)>0 ).replace({True:1, False:0})
 df['date_missing'] =(df[DATE_COL].isnull().sum(axis=1)>0 ).replace({True:1, False:0})
 df.day = df.day.replace({3:5,2:6,1:7})
 df.loc[df.ord_5.notnull() , 'ord_5_1'] = df.loc[df.ord_5.notnull() , 'ord_5'].apply(lambda x: x[0])
 df.loc[df.ord_5.notnull() , 'ord_5_2'] = df.loc[df.ord_5.notnull() , 'ord_5'].apply(lambda x: x[1])
 return df
train, test, train_id, test_id = read_csv()
train = preprocessing(train)
 test = preprocessing(test)
 print(f'train day unique value:{train.day.unique() }')
 print(f'test day unique value:{test.day.unique() }')
 for col in test.columns:
if len(set(train[col].dropna().unique().tolist())^ set(test[col].dropna().unique().tolist())) >0:
train_only = list(set(train[col].dropna().unique().tolist())- set(test[col].dropna().unique().tolist()))
test_only = list(set(test[col].dropna().unique().tolist())- set(train[col].dropna().unique().tolist()))
print(col, '(train only)', train_only, '(test only)', test_only)
 train.loc[train[col].isin(train_only), col] = np.NaN
test.loc[test[col].isin(test_only), col] = np.NaN
test.insert(1, 'target', 0)
 drop_cols = ['bin_3', 'ord_5']
train.drop(columns=drop_cols, inplace=True)
 test.drop(columns=drop_cols, inplace=True )<categorify>
def get_dataloaders(num_workers, batch_size, x_train, y_train, x_valid, y_valid):

 train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, num_workers=num_workers)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, num_workers=num_workers)
 loader = {
"train": train_loader,
"valid": valid_loader
}
return loader
%%time
features = [_f for _f in train if _f not in ['id', 'target']]
def factor_encoding(train, test):
assert sorted(train.columns)== sorted(test.columns)
 full = pd.concat([train, test], axis=0, sort=False)
 for f in full:
full[f], _ = pd.factorize(full[f])
 full[f] += 1
return full.iloc[:len(train)], full.iloc[len(train):]
train_f, test_f = factor_encoding(train[features], test[features])
 print(train_f,'-'*20)
 print(train_f.head(10))
print(test_f,'-'*20)
 print(test_f.head(10))
class LibFFMEncoder(object):
def __init__(self):
self.encoder = 1
self.encoding = {}
def encode_for_libffm(self, row):
txt = f"{row[0]}"
for i, r in enumerate(row[1:]):
try:
txt += f' {i+1}:{self.encoding[(i, r)]}:1'
except KeyError:
self.encoding[(i, r)] = self.encoder
self.encoder += 1
txt += f' {i+1}:{self.encoding[(i, r)]}:1'
return txt
SPLITS = 5
fold_ids = [
[trn_, val_] for(trn_, val_)in KFold(SPLITS,True,1 ).split(train)
 ]
for fold_,(trn_, val_)in enumerate(fold_ids):
encoder = LibFFMEncoder()
libffm_format_trn = pd.concat([train['target'].iloc[trn_], train_f.iloc[trn_]], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 libffm_format_val = pd.concat([train['target'].iloc[val_], train_f.iloc[val_]], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 print(train['target'].iloc[trn_].shape, train['target'].iloc[val_].shape, libffm_format_val.shape)
 libffm_format_trn.to_csv(f'libffm_trn_fold_{fold_+1}.txt', index=False, header=False)
 libffm_format_val.to_csv(f'libffm_val_fold_{fold_+1}.txt', index=False, header=False)
 encoder = LibFFMEncoder()
libffm_format_trn = pd.concat([train['target'], train_f], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 libffm_format_tst = pd.concat([test['target'], test_f], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 libffm_format_trn.to_csv(f'libffm_trn.txt', index=False, header=False)
 libffm_format_tst.to_csv(f'libffm_tst.txt', index=False, header=False )<choose_model_class>
def get_testloaders(num_workers, batch_size, x_test):

 test_dataset = TestDataset(x_test)
 test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, num_workers=num_workers)
 loader = {
'test': test_loader
}
return loader
%%time
ffm_model = xl.create_ffm()
outputs = []
param = {'task':'binary',
'lr':0.2,
'k':4,
'lambda':0.0002,
'metric':'auc',
'epoch': 15
}
for fold_ in range(1, SPLITS+1):
print(f'fold: {fold_}')
 model = f"libffm_fold_{fold_}_model"
trn_fold_txt = f"libffm_trn_fold_{fold_}.txt"
val_fold_txt = f"libffm_val_fold_{fold_}.txt"
val_preds_fold_txt = f"val_preds_fold_{fold_}.txt"
ffm_model.setTrain(trn_fold_txt)
 ffm_model.setValidate(val_fold_txt)
 print(' fitting...')
 ffm_model.fit(param, 'model.output')
 print(' make predictions...')
 ffm_model.setTest(val_fold_txt)
 ffm_model.setSigmoid()
ffm_model.predict("model.output", val_preds_fold_txt)
 gc.collect()
print(' auc score:',
roc_auc_score(
 train['target'].iloc[fold_ids[fold_-1][1]],
pd.read_csv(val_preds_fold_txt, header=None ).values[:,0])) 
<set_options>
def train_model(n_epochs, loaders, model, optimizer, criterion, device, save_path):

 scheduler = StepLR(optimizer, step_size=2, gamma=0.96)
 valid_loss_min = np.Inf
print(valid_loss_min)
 for epoch in range(1, n_epochs + 1):
scheduler.step()
train_loss = 0.0
valid_loss = 0.0
model.train()
for data in loaders['train']:
data_input, data_target = data['input'].to(device), data['target'].to(device)
 optimizer.zero_grad()
output = model(data_input)
 loss = criterion(output, data_target)
 loss.backward()
optimizer.step()
train_loss +=(loss.item() / len(loaders['train']))
model.eval()
for data in loaders['valid']:
data_input, data_target = data['input'].to(device), data['target'].to(device)
 output = model(data_input)
 loss = criterion(output, data_target)
 valid_loss +=(loss.item() / len(loaders['valid']))
if valid_loss < valid_loss_min:
print(f'Epoch: \t{epoch}\tValidation loss decreased({valid_loss_min} -> {valid_loss} ).Saving the model...')
 torch.save(model.state_dict() , save_path)
 valid_loss_min = valid_loss
return model
gc.collect()<load_from_csv>
def run_training(seed, kfold, batch_size, epochs, learning_rate, weight_decay):
set_seed(seed)
 for fold,(train_idx, valid_idx)in enumerate(kfold.split(X=train, y=train_targets)) :
x_train, x_valid = train[train_idx], train[valid_idx]
y_train, y_valid = train_targets[train_idx], train_targets[valid_idx]
dataloaders = get_dataloaders(0, batch_size, x_train, y_train, x_valid, y_valid)
 device = 'cuda' if torch.cuda.is_available() else 'cpu'
model = Model(875, 206, 1024 ).to(device)
 criterion_moa = nn.BCEWithLogitsLoss()
optimizer_moa = torch.optim.Adam(model.parameters() , lr=learning_rate, weight_decay=weight_decay)
 train_model(epochs, dataloaders, model, optimizer_moa, criterion_moa, device, f'models/model_seed_{seed}_fold_{fold}.pt' )
oof_preds = np.zeros(train.shape[0])
 for fold_,(_, val_)in enumerate(fold_ids):
oof_preds[val_] = pd.read_csv(f'val_preds_fold_{fold_+1}.txt', header=None ).values[:, 0]
print(roc_auc_score(train['target'], oof_preds))<train_model>
FOLDS = 5
WORKERS = 0
BATCH_SIZE = 128
EPOCHS = 50
LEARNING_RATE = 0.0002
WEIGHT_DECAY = 0.00001
SEED = 42
%%time
ffm_model = xl.create_ffm()
ffm_model.setTrain("libffm_trn.txt")
 ffm_model.fit(param, 'model.output' )<predict_on_test>
%mkdir models
%%time
ffm_model.setTest('libffm_tst.txt')
 ffm_model.setSigmoid()
ffm_model.predict('model.output', 'tst_preds.txt' )<set_options>
mskf = MultilabelStratifiedKFold(n_splits=FOLDS)
 for seed in range(40, 45):
run_training(seed, mskf, BATCH_SIZE, EPOCHS, LEARNING_RATE, WEIGHT_DECAY )
gc.collect()<save_to_csv>
def inference(loaders, model, device):

 model.eval()
preds = []
for data in loaders['test']:
data_input = data['input'].to(device)
 with torch.no_grad() :
output = model(data_input)
 pred = output.sigmoid().detach().cpu().numpy()
preds.append(pred)
 return np.concatenate(preds )
submission = pd.DataFrame()
submission['id'] = test_id
submission['target'] = pd.read_csv('tst_preds.txt', header=None ).values[:,0]
submission.to_csv('xlearn_prediction.csv', index=False )<load_from_csv>
def run_inferencing(seed):
set_seed(seed)
 preds = np.zeros(( len(test), 206))
for i in range(0, FOLDS):
dataloaders = get_testloaders(0, 128, test)
 device = 'cuda' if torch.cuda.is_available() else 'cpu'
model = Model(875, 206, 1024)
 model.load_state_dict(torch.load(f'models/model_seed_{seed}_fold_{i}.pt'))
model.to(device)
 pred = inference(dataloaders, model, device)
 preds += pred
preds = preds / FOLDS
return preds
submission = pd.read_csv('.. /input/cat-in-the-dat-ii/sample_submission.csv', index_col='id' )<load_from_csv>
preds = np.zeros(( len(test), 206))
for seed in range(40, 45):
preds += run_inferencing(seed)
 preds = preds / 5
<load_from_csv><EOS> 
sample_submission[targets] = preds
sample_submission.loc[test_features['cp_type']=='ctl_vehicle', targets] = 0
sample_submission.to_csv('submission.csv', index=False )