Kaggle competition: Mechanisms of Action (MoA) Prediction
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('xgb_test.pickle', 'wb'))<load_from_csv>
warnings.filterwarnings('ignore' )
warnings.filterwarnings('ignore')
 path = '/kaggle/input/competitive-data-science-predict-future-sales/'
items = pd.read_csv(path+'items.csv')
 sales_train = pd.read_csv(path+'sales_train.csv')
 item_categories = pd.read_csv(path+'item_categories.csv')
 test = pd.read_csv(path+'test.csv')
 shops = pd.read_csv(path+'shops.csv')
<data_type_conversions>
from sklearn.preprocessing import QuantileTransformer
sales_train.date = pd.to_datetime(sales_train.date, format="%d.%m.%Y")
<data_type_conversions>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
def downcast_dtypes(df):
float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype in ["int64", "int32"]]
df[float_cols] = df[float_cols].astype(np.float32)
 df[int_cols] = df[int_cols].astype(np.int16)
 return df
sales_train = downcast_dtypes(sales_train)
 print(sales_train.info() )<define_variables>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
target_range = [0 ,20]<merge>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
sales = pd.merge(sales_train, items, on='item_id', how='left')
 sales = sales.drop('item_name', axis=1)
 sales.head(10 )<define_variables>
def make_PCA(train_features, test_features, columns=GENES, n_comps=600):
data = pd.concat([pd.DataFrame(train_features[columns]),
pd.DataFrame(test_features[columns])])
 data_pca =(PCA(n_components=n_comps, random_state=42 ).fit_transform(data[columns]))
train_pca = data_pca[:train_features.shape[0]]
test_pca = data_pca[-test_features.shape[0]:]
train_pca = pd.DataFrame(train_pca, columns=[f'pca_{columns[0][0]}-{i}' for i in range(n_comps)])
 test_pca = pd.DataFrame(test_pca, columns=[f'pca_{columns[0][0]}-{i}' for i in range(n_comps)])
 train_features = pd.concat(( train_features, train_pca), axis=1)
 test_features = pd.concat(( test_features, test_pca), axis=1)
 return train_features, test_features
index_cols = ['shop_id', 'item_id', 'date_block_num'] 
<groupby>
train_features, test_features = make_PCA(train_features, test_features)
 train_features, test_features = make_PCA(train_features, test_features, columns=CELLS, n_comps=50 )
mean_sales = sales.groupby(['date_block_num', 'shop_id', 'item_id'] ).agg({'item_cnt_day': 'sum', 'item_price': np.mean} ).reset_index()<merge>
var_thresh = VarianceThreshold(0.9)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
mean_sales = pd.merge(grid, mean_sales, on=['date_block_num', 'shop_id', 'item_id'], how='left' ).fillna(0)
 mean_sales = pd.merge(mean_sales, items, on='item_id', how='left' )<merge>
q_list = [0.1, 0.2, 0.25, 0.3, 0.4, 0.5, 0.6, 0.7, 0.75, 0.8, 0.9]
for type_id in ['item_id', 'shop_id', 'item_category_id']:
for column_id, aggregator, aggtype in [('item_price',np.mean,'avg'),('item_cnt_day',np.sum,'sum'),('item_cnt_day',np.mean,'avg')]:
mean_df = sales.groupby([type_id,'date_block_num'] ).aggregate(aggregator ).reset_index() [[column_id,type_id,'date_block_num']]
mean_df.columns = [type_id+'_'+aggtype+'_'+column_id,type_id,'date_block_num']
mean_sales = pd.merge(mean_sales, mean_df, on=['date_block_num',type_id], how='left' )<merge>
def make_knn_features(features, pre_name='G'):
q_list = [0.1, 0.2, 0.25, 0.3, 0.4, 0.5, 0.6, 0.7, 0.75, 0.8, 0.9]
kNN_features = {}
for q in q_list:
kNN_features[f'q-{q}'] = features.quantile(q, axis=1 ).values
kNN_features['min'] = features.min(axis=1 ).values
kNN_features['max'] = features.max(axis=1 ).values
kNN_features['std'] = features.std(axis=1 ).values
kNN_features['mean'] = features.mean(axis=1 ).values
return pd.DataFrame(kNN_features, columns=kNN_features.keys() )
lag_variables = list(mean_sales.columns[7:])+['item_cnt_day']
lags = [1, 2, 3, 6]
for lag in tqdm_notebook(lags):
sales_new_df = mean_sales.copy()
sales_new_df.date_block_num += lag
sales_new_df = sales_new_df[['date_block_num', 'shop_id', 'item_id']+lag_variables]
sales_new_df.columns = ['date_block_num', 'shop_id', 'item_id'] + [lag_feat+ '_lag_'+str(lag)for lag_feat in lag_variables]
mean_sales = pd.merge(mean_sales, sales_new_df, on=['date_block_num', 'shop_id', 'item_id'], how='left')
<filter>
TRAIN_MASK =(train_features['cp_type'] != 'ctl_vehicle')
 TEST_MASK =(test_features['cp_type'] != 'ctl_vehicle')
 g_knn_train_features = make_knn_features(train_features[GENES][TRAIN_MASK])
 g_knn_test_features = make_knn_features(test_features[GENES][TEST_MASK])
 c_knn_train_features = make_knn_features(train_features[CELLS][TRAIN_MASK])
 c_knn_test_features = make_knn_features(test_features[CELLS][TEST_MASK])
 knn_train_features = pd.concat([g_knn_train_features, c_knn_train_features], axis=1)
 knn_test_features = pd.concat([g_knn_test_features, c_knn_test_features], axis=1 )
mean_sales = mean_sales[mean_sales['date_block_num']>12]<data_type_conversions>
target_cols = train_targets_scored.drop('sig_id', axis=1 ).columns.values.tolist()
for feat in mean_sales.columns:
if 'item_cnt' in feat:
mean_sales[feat] = mean_sales[feat].fillna(0)
 elif 'item_price' in feat:
mean_sales[feat] = mean_sales[feat].fillna(mean_sales[feat].median())
<define_variables>
clfr = RandomForestClassifier(10)
 clfr.fit(X=knn_train_features, y=train_targets_scored[target_cols][train_features['cp_type']!='ctl_vehicle'] )
cols_to_drop = lag_variables[:-1] + ['item_price', 'item_name']<drop_column>
clfr_res = clfr.predict(knn_train_features )
training = mean_sales.drop(cols_to_drop, axis=1 )<prepare_x_and_y>
( clfr_res == train_targets_scored[target_cols][train_features['cp_type']!='ctl_vehicle'].values ).mean()
xgbtrain = xgb.DMatrix(training.iloc[:, training.columns != 'item_cnt_day'].values, training.iloc[:, training.columns == 'item_cnt_day'].values )<train_model>
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape
params = {
'max_depth': 10,
'subsample': 1,
'min_child_weight': 0.5,
'eta': 0.3,
'num_round': 1000,
'seed': 1,
'silent': 0,
'eval_metric': 'rmse'
}
boost = xgb.train(params, xgbtrain )<drop_column>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
train.shape
cols = list(training.columns)
 del cols[cols.index('item_cnt_day')]<define_variables>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
[cols[x] for x in [2, 0, 1, 3, 5]]<load_from_csv>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
test = pd.read_csv(path+'test.csv' )<merge>
def set_train_folds(train, target, fold_number_col_name="kfold", n_splits=7, shuffle=True, random_state=42):
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=n_splits,
shuffle=shuffle,
random_state=random_state)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, fold_number_col_name] = int(f)
 folds[fold_number_col_name] = folds[fold_number_col_name].astype(int)
 return folds
test['date_block_num'] = 34
test = pd.merge(test, items, on='item_id', how='left' )<merge>
folds = set_train_folds(train, target)
 folds.head()
for lag in tqdm_notebook(lags):
sales_new_df = mean_sales.copy()
sales_new_df.date_block_num += lag
sales_new_df = sales_new_df[['date_block_num', 'shop_id', 'item_id']+lag_variables]
sales_new_df.columns = ['date_block_num', 'shop_id', 'item_id'] + [lag_feat+ '_lag_'+str(lag)for lag_feat in lag_variables]
test = pd.merge(test, sales_new_df, on=['date_block_num', 'shop_id', 'item_id'], how='left')
<drop_column>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
df_test = set(test.drop(['ID', 'item_name'], axis=1 ).columns)
 df_training = set(training.drop('item_cnt_day', axis=1 ).columns)
 for i in df_test:
assert i in df_training
for j in df_training:
assert j in df_test<filter>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0.
 for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0.
 valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
assert df_training == df_test<data_type_conversions>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
test = test.drop(['ID', 'item_name'], axis=1)
 for feat in test.columns:
if 'item_cnt' in feat:
test[feat]=test[feat].fillna(0)
 elif 'item_price' in feat:
test[feat] = test[feat].fillna(test[feat].median() )<predict_on_test>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2619422201258426)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2619422201258426)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
xgbpredict = xgb.DMatrix(test.values)
 pred = boost.predict(xgbpredict)
 pd.Series(pred ).describe()<prepare_output>
def encode_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
submission_df = pd.DataFrame({'ID': test.index, 'item_cnt_month': pred})
 submission_df.head(10 )<save_to_csv>
feature_cols = [c for c in encode_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )
submission_df.to_csv('submission_xgboost.csv', index=False )<load_from_csv>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1500
df_train_raw = pd.read_csv('.. /input/train.csv' )<count_unique_values>
def run_training(fold, seed):
seed_everything(seed)
 train = encode_data(folds)
 test_ = encode_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
print(( '{m:d} unique matches' ).format(m=len(df_train_raw['matchId'].unique())))
 print(( '{tk:d} data entries in {m:d} different matches with exactly 3 team kills' ).format(
 tk=len(df_train_raw[df_train_raw['teamKills']==3]),
m=len(df_train_raw[df_train_raw['teamKills']==3]['matchId'].unique())))
 print(( '{tk:d} data entries in {m:d} different matches with more than 3 team kills' ).format(
 tk=len(df_train_raw[df_train_raw['teamKills']>3]),
m=len(df_train_raw[df_train_raw['teamKills']>3]['matchId'].unique())))
 print(( '{g:d} data entries in {m:d} different matches with exactly 1 group' ).format(
 g=len(df_train_raw[df_train_raw['numGroups']==1]),
m=len(df_train_raw[df_train_raw['numGroups']==1]['matchId'].unique())))
 n_group = 10
print(( '{g:d} data entries in {m:d} different matches with less than {n:d} groups' ).format(
 g=len(df_train_raw[df_train_raw['numGroups']<n_group]),
m=len(df_train_raw[df_train_raw['numGroups']<n_group]['matchId'].unique()),
n=n_group))<drop_column>
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
df_train = df_train_raw
del df_train_raw<merge>
SEED = [21,42]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
def FeatureEngineering(df):
df_size = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_mean = df.groupby(['matchId','groupId'] ).mean().reset_index()
df_sum = df.groupby(['matchId','groupId'] ).sum().reset_index()
df_max = df.groupby(['matchId','groupId'] ).max().reset_index()
df_min = df.groupby(['matchId','groupId'] ).min().reset_index()
df_match_mean = df.groupby(['matchId'] ).mean().reset_index()
df = pd.merge(df, df_size, how='left', on=['matchId', 'groupId'])
 del df_size
df = pd.merge(df, df_mean, suffixes=["", "_mean"], how='left', on=['matchId', 'groupId'])
 del df_mean
df = pd.merge(df, df_sum, suffixes=["", "_sum"], how='left', on=['matchId', 'groupId'])
 del df_sum
df = pd.merge(df, df_max, suffixes=["", "_max"], how='left', on=['matchId', 'groupId'])
 del df_max
df = pd.merge(df, df_min, suffixes=["", "_min"], how='left', on=['matchId', 'groupId'])
 del df_min
df = pd.merge(df, df_match_mean, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 del df_match_mean
columns = list(df.columns)
 columns.remove("Id")
 columns.remove("matchId")
 columns.remove("groupId")
 columns.remove("Id_mean")
 columns.remove("Id_sum")
 columns.remove("Id_max")
 columns.remove("Id_min")
 columns.remove("Id_match_mean")
 df = df[columns]
return df 
<compute_test_metric>
predictions2 = clfr.predict(knn_train_features)
 final_model = RandomForestClassifier(3)
 final_model.fit(np.hstack([train[target_cols], predictions2]), train_targets_scored[target_cols][TRAIN_MASK] )
def MAE(y_estimate, y_true):
return sum(abs(y_estimate-y_true)) /len(y_estimate )<prepare_x_and_y>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score)
matchId = df_train['matchId'].unique()
matchIdTrain = np.random.choice(matchId, int(0.80*len(matchId)))
 df_train2 = df_train[df_train['matchId'].isin(matchIdTrain)]
df_test = df_train[~df_train['matchId'].isin(matchIdTrain)]
y_train = df_train2['winPlacePerc']
X_train = df_train2.drop(columns=['winPlacePerc'])
 y_test = df_test['winPlacePerc']
X_test = df_test.drop(columns=['winPlacePerc'])
 X_train = FeatureEngineering(X_train)
 X_test = FeatureEngineering(X_test)
 print(( 'Training set size: {train:d}, test set size: {test:d}' ).format(train=len(X_train), test=len(X_test)))
 print(X_train.describe())
 del df_train2, df_test, df_train<load_from_csv>
def log_loss_metric(y_true, y_pred):
metrics = []
for _target in train_targets_scored.columns:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels = [0,1]))
return np.mean(metrics )
def RandomForestModel() :
print('
Creating and training random forest regressor')
 rfr = RandomForestRegressor(n_jobs=4, n_estimators=10)
 rfr.fit(X_train, y_train)
 y_rfr = rfr.predict(X_test)
 score_rfr = MAE(y_rfr, y_test)
 print(( 'Random Forest training testset score: {s:.3f}' ).format(s=score_rfr))
X_submit = pd.read_csv('.. /input/test.csv')
 df_submit = X_submit[['Id', 'matchId', 'groupId']]
X_submit = FeatureEngineering(X_submit)
 y_submit = rfr.predict(X_submit)
 df_submit['prediction'] = y_submit
return df_submit<feature_engineering>
predictions_knn = clfr.predict(knn_test_features)
 test[target_cols] = final_model.predict(np.hstack([test[target_cols], predictions_knn]))
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )