Kaggle competition: Mechanisms of Action (MoA) Prediction
important_predictors = [ "kills",
"maxPlace",
"numGroups",
"distance",
"killStreaks",
"weaponsAcquired",
"winPoints",
"killPlace",
"killPoints",
"longestKill"
]
x_train = train[important_predictors]
x_train.head()<train_model>
warnings.filterwarnings("ignore")
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )
regressor = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 20, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.8)
 regressor.fit(x_train, y_train )<predict_on_test>
def get_logger(filename='log'):
logger = getLogger(__name__)
 logger.setLevel(INFO)
 handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
handler2 = FileHandler(filename=f"{filename}.log")
 handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
 logger.addHandler(handler2)
 return logger
logger = get_logger()
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
y_predict = regressor.predict(x_test)
 print(y_predict )<groupby>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
y_predict[y_predict > 1] = 1
test['winPlacePercPredictions'] = y_predict
aux = test.groupby(['matchId','groupId'])['winPlacePercPredictions'].agg('mean' ).groupby('matchId' ).rank(pct=True ).reset_index()<merge>
train = train_features.merge(train_targets_scored, on='sig_id')
 target_cols = [c for c in train_targets_scored.columns if c not in ['sig_id']]
cols = target_cols + ['cp_type']
train[cols].groupby('cp_type' ).sum().sum(1 )
aux.columns = ['matchId','groupId','winPlacePerc']
test = test.merge(aux, how='left', on=['matchId','groupId'])
 test.head()<compute_train_metric>
folds = train.copy()
Fold = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[target_cols])) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int)
 print(folds.shape )
scores = cross_val_score(regressor, x_train, y_train, cv=10)
 print(scores )<compute_test_metric>
class TrainDataset(Dataset):
def __init__(self, df, num_features, cat_features, labels):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
self.labels = labels
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 label = torch.tensor(self.labels[idx] ).float()
return cont_x, cate_x, label
class TestDataset(Dataset):
def __init__(self, df, num_features, cat_features):
self.cont_values = df[num_features].values
self.cate_values = df[cat_features].values
def __len__(self):
return len(self.cont_values)
 def __getitem__(self, idx):
cont_x = torch.FloatTensor(self.cont_values[idx])
 cate_x = torch.LongTensor(self.cate_values[idx])
 return cont_x, cate_x
accuracy = scores.mean()
print(accuracy )<set_options>
cat_features = ['cp_time', 'cp_dose']
num_features = [c for c in train.columns if train.dtypes[c] != 'object']
num_features = [c for c in num_features if c not in cat_features]
num_features = [c for c in num_features if c not in target_cols]
target = train[target_cols].values
def cate2num(df):
df['cp_time'] = df['cp_time'].map({24: 0, 48: 1, 72: 2})
 df['cp_dose'] = df['cp_dose'].map({'D1': 3, 'D2': 4})
 return df
train = cate2num(train)
 test = cate2num(test )
print(os.listdir(".. /input"))
sns.set(style='white', context='notebook', palette='Set2')
 %matplotlib inline
warnings.simplefilter(action='ignore', category=FutureWarning )<load_from_csv>
class CFG:
max_grad_norm=1000
gradient_accumulation_steps=1
hidden_size=512
dropout=0.5
lr=1e-2
weight_decay=1e-6
batch_size=32
epochs=20
num_features=num_features
cat_features=cat_features
target_cols=target_cols
train_df = pd.read_csv(".. /input/train_V2.csv")
 test_df = pd.read_csv(".. /input/test_V2.csv" )<count_missing_values>
class TabularNN(nn.Module):
def __init__(self, cfg, num_units = 5):
super().__init__()

 self.mlp = nn.Sequential(
 nn.Linear(len(cfg.num_features), cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU() ,
nn.Linear(cfg.hidden_size, cfg.hidden_size),
nn.BatchNorm1d(cfg.hidden_size),
nn.Dropout(cfg.dropout),
nn.PReLU())
 self.fc1 = nn.ModuleList([nn.Linear(cfg.hidden_size,len(cfg.target_cols)) for i in range(num_units)])
 def maxout(self, x, layer_list):
max_output = layer_list[0](x)
 for _, layer in enumerate(layer_list, start=1):
max_output = torch.max(max_output, layer(x))
return max_output
def forward(self, cont_x, cate_x):
x = self.mlp(cont_x)
 x = self.maxout(x, self.fc1)
 return x
pd.DataFrame({'train':train_df.isna().sum() , 'test':test_df.isna().sum() } )<correct_missing_values>
def train_fn(train_loader, model, optimizer, epoch, scheduler, device):
losses = AverageMeter()
model.train()
for step,(cont_x, cate_x, y)in enumerate(train_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
loss.backward()
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters() , CFG.max_grad_norm)
 if(step + 1)% CFG.gradient_accumulation_steps == 0:
scheduler.step()
optimizer.step()
optimizer.zero_grad()
return losses.avg
def validate_fn(valid_loader, model, device):
losses = AverageMeter()
model.eval()
val_preds = []
for step,(cont_x, cate_x, y)in enumerate(valid_loader):
cont_x, cate_x, y = cont_x.to(device), cate_x.to(device), y.to(device)
 batch_size = cont_x.size(0)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 loss = nn.BCEWithLogitsLoss()(pred, y)
 losses.update(loss.item() , batch_size)
 val_preds.append(pred.sigmoid().detach().cpu().numpy())
 if CFG.gradient_accumulation_steps > 1:
loss = loss / CFG.gradient_accumulation_steps
val_preds = np.concatenate(val_preds)
 return losses.avg, val_preds
def inference_fn(test_loader, model, device):
model.eval()
preds = []
for step,(cont_x, cate_x)in enumerate(test_loader):
cont_x, cate_x = cont_x.to(device), cate_x.to(device)
 with torch.no_grad() :
pred = model(cont_x, cate_x)
 preds.append(pred.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
train_df = train_df.dropna()<concatenate>
def run_single_nn(cfg, train, test, folds, num_features, cat_features, target, device, fold_num=0, seed=42):
logger.info(f'Set seed {seed}')
 seed_everything(seed=seed)
 trn_idx = folds[folds['fold'] != fold_num].index
val_idx = folds[folds['fold'] == fold_num].index
train_folds = train.loc[trn_idx].reset_index(drop=True)
 valid_folds = train.loc[val_idx].reset_index(drop=True)
 train_target = target[trn_idx]
valid_target = target[val_idx]
train_dataset = TrainDataset(train_folds, num_features, cat_features, train_target)
 valid_dataset = TrainDataset(valid_folds, num_features, cat_features, valid_target)
 train_loader = DataLoader(train_dataset, batch_size=cfg.batch_size, shuffle=True,
num_workers=4, pin_memory=True, drop_last=True)
 valid_loader = DataLoader(valid_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True, drop_last=False)
 model = TabularNN(cfg)
 model.to(device)
 optimizer = optim.Adam(model.parameters() , lr=cfg.lr, weight_decay=cfg.weight_decay)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=cfg.epochs, steps_per_epoch=len(train_loader))
log_df = pd.DataFrame(columns=(['EPOCH']+['TRAIN_LOSS']+['VALID_LOSS']))
best_loss = np.inf
for epoch in range(cfg.epochs):
train_loss = train_fn(train_loader, model, optimizer, epoch, scheduler, device)
 valid_loss, val_preds = validate_fn(valid_loader, model, device)
 log_row = {'EPOCH': epoch,
'TRAIN_LOSS': train_loss,
'VALID_LOSS': valid_loss,
}
log_df = log_df.append(pd.DataFrame(log_row, index=[0]), sort=False)
 if valid_loss < best_loss:
logger.info(f'epoch{epoch} save best model...{valid_loss}')
 best_loss = valid_loss
oof = np.zeros(( len(train), len(cfg.target_cols)))
 oof[val_idx] = val_preds
torch.save(model.state_dict() , f"fold{fold_num}_seed{seed}.pth")
 test_dataset = TestDataset(test, num_features, cat_features)
 test_loader = DataLoader(test_dataset, batch_size=cfg.batch_size, shuffle=False,
num_workers=4, pin_memory=True)
 model = TabularNN(cfg)
 model.load_state_dict(torch.load(f"fold{fold_num}_seed{seed}.pth"))
model.to(device)
 predictions = inference_fn(test_loader, model, device)
 torch.cuda.empty_cache()
return oof, predictions
def run_kfold_nn(cfg, train, test, folds, num_features, cat_features, target, device, n_fold=5, seed=42):
oof = np.zeros(( len(train), len(cfg.target_cols)))
 predictions = np.zeros(( len(test), len(cfg.target_cols)))
 for _fold in range(n_fold):
logger.info("Fold {}".format(_fold))
_oof, _predictions = run_single_nn(cfg,
train,
test,
folds,
num_features,
cat_features,
target,
device,
fold_num=_fold,
seed=seed)
 oof += _oof
predictions += _predictions / n_fold
score = 0
for i in range(target.shape[1]):
_score = log_loss(target[:,i], oof[:,i])
 score += _score / target.shape[1]
logger.info(f"CV score: {score}")
 return oof, predictions
dataset = pd.concat([train_df, test_df], sort=True)
 dataset = reduce_mem_usage(dataset )<merge>
oof = np.zeros(( len(train), len(CFG.target_cols)))
 predictions = np.zeros(( len(test), len(CFG.target_cols)))
 SEED = [0, 1, 2]
for seed in SEED:
_oof, _predictions = run_kfold_nn(CFG,
train, test, folds,
num_features, cat_features, target,
device,
n_fold=5, seed=seed)
 oof += _oof / len(SEED)
 predictions += _predictions / len(SEED)
 score = 0
for i in range(target.shape[1]):
_score = log_loss(target[:,i], oof[:,i])
 score += _score / target.shape[1]
logger.info(f"Seed Averaged CV score: {score}" )
def min_by_team(df):
cols_to_drop = ['Id', 'groupId', 'matchId', 'matchType', 'winPlacePerc']
features = [col for col in df.columns if col not in cols_to_drop]
agg = df.groupby(['matchId','groupId'])[features].min()
return df.merge(agg, suffixes=['', '_min'], how='left', on=['matchId', 'groupId'])
 def max_by_team(df):
cols_to_drop = ['Id', 'groupId', 'matchId', 'matchType', 'winPlacePerc']
features = [col for col in df.columns if col not in cols_to_drop]
agg = df.groupby(['matchId', 'groupId'])[features].max()
return df.merge(agg, suffixes=['', '_max'], how='left', on=['matchId', 'groupId'])
 def sum_by_team(df):
cols_to_drop = ['Id', 'groupId', 'matchId', 'matchType', 'winPlacePerc']
features = [col for col in df.columns if col not in cols_to_drop]
agg = df.groupby(['matchId', 'groupId'])[features].sum()
return df.merge(agg, suffixes=['', '_sum'], how='left', on=['matchId', 'groupId'])
 def median_by_team(df):
cols_to_drop = ['Id', 'groupId', 'matchId', 'matchType', 'winPlacePerc']
features = [col for col in df.columns if col not in cols_to_drop]
agg = df.groupby(['matchId', 'groupId'])[features].median()
return df.merge(agg, suffixes=['', '_median'], how='left', on=['matchId', 'groupId'])
 def mean_by_team(df):
cols_to_drop = ['Id', 'groupId', 'matchId', 'matchType', 'winPlacePerc']
features = [col for col in df.columns if col not in cols_to_drop]
agg = df.groupby(['matchId', 'groupId'])[features].mean()
return df.merge(agg, suffixes=['', '_mean'], how='left', on=['matchId', 'groupId'])
 def rank_by_team(df):
cols_to_drop = ['Id', 'groupId', 'matchId', 'matchType', 'winPlacePerc']
features = [col for col in df.columns if col not in cols_to_drop]
agg = df.groupby(['matchId', 'groupId'])[features].mean()
agg = agg.groupby('matchId')[features].rank(pct=True)
 return df.merge(agg, suffixes=['', '_mean_rank'], how='left', on=['matchId', 'groupId'] )<import_modules>
train[target_cols] = oof
train[['sig_id']+target_cols].to_csv('oof.csv', index=False)
 test[target_cols] = predictions
test[['sig_id']+target_cols].to_csv('pred.csv', index=False )
from sklearn.model_selection import KFold, StratifiedKFold, train_test_split
from sklearn.metrics import mean_absolute_error<create_dataframe>
result = train_targets_scored.drop(columns=target_cols)\ 
.merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = result[target_cols].values
score = 0
for i in range(y_true.shape[1]):
_score = log_loss(y_true[:,i], y_pred[:,i])
 score += _score / y_true.shape[1]
logger.info(f"Final result: {score}" )
def oof_model_preds(df, model, num_folds, params):
train_df = df[df['winPlacePerc'].notnull() ]
test_df = df[df['winPlacePerc'].isnull() ]
print("Starting LightGBM.Train shape: {}, test shape: {}".format(train_df.shape, test_df.shape))
del df
gc.collect()
oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feature_importance_df = pd.DataFrame()
drop_features = ['Id', 'groupId', 'matchId', 'matchType', 'winPlacePerc']
feats = [f for f in train_df.columns if f not in drop_features]
if num_folds == 1:
train_x, valid_x, train_y, valid_y = train_test_split(train_df[feats], train_df['winPlacePerc'], test_size=0.2, random_state=1001)
 model.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)],
eval_metric= 'mae', verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'])
 oof_preds = model.predict(train_df[feats])
 sub_preds = model.predict(test_df[feats])
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = model.feature_importances_
fold_importance_df["fold"] = 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('MAE : %.6f' %(mean_absolute_error(train_df['winPlacePerc'], oof_preds)))
 del train_x, train_y, valid_x, valid_y
gc.collect()
elif num_folds > 1:
folds = KFold(n_splits= num_folds, shuffle=True, random_state=1001)
 for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df[feats], train_df['winPlacePerc'])) :
train_x, train_y = train_df[feats].iloc[train_idx], train_df['winPlacePerc'].iloc[train_idx]
valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df['winPlacePerc'].iloc[valid_idx]
model.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)],
eval_metric= 'mae', verbose=params['verbose'], early_stopping_rounds=params['early_stopping_rounds'])
 oof_preds[valid_idx] = model.predict(valid_x)
 sub_preds += model.predict(test_df[feats])/ folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = model.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d MAE : %.6f' %(n_fold + 1, mean_absolute_error(valid_y, oof_preds[valid_idx])))
 del train_x, train_y, valid_x, valid_y
gc.collect()
print('Full MAE score %.6f' % mean_absolute_error(train_df['winPlacePerc'], oof_preds))
return oof_preds, sub_preds, feature_importance_df<choose_model_class>
sub = submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False)
 sub.head()