Kaggle competition: Mechanisms of Action (MoA) Prediction
level0 = [('rf',rf),('xgb',xgb),('lgb',lgb),('knn',knn),('gbr',gbr),('abr',abr),('cat',cat),('svr',svr),('vr',VR)]
level1 = LinearRegression()
stack = StackingRegressor(estimators=level0,final_estimator=level1, cv=5,n_jobs=-1)
 stack.fit(train1,targets)
 sqrt(mean_squared_error(targets, stack.predict(train1)) )<predict_on_test>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_features.head()
stack_pred = np.expm1(stack.predict(test1))
vr_pred = np.expm1(VR.predict(test1))
rf_pred = np.expm1(rf.predict(test1))
xgb_pred = np.expm1(xgb.predict(test1))
lgb_pred = np.expm1(lgb.predict(test1))
cat_pred = np.expm1(cat.predict(test1))
svr_pred = np.expm1(svr.predict(test1))
abr_pred = np.expm1(abr.predict(test1))
gbr_pred = np.expm1(gbr.predict(test1))
knn_pred = np.expm1(knn.predict(test1))<save_to_csv>
test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 test_features.head()
submission = pd.DataFrame({
"Id": test["Id"],
"SalePrice": stack_pred
})
 submission.to_csv('submission.csv', index=False )<compute_test_metric>
train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets.head()
t = [105000.0,164370.59375,199657.75,201696.609375,187367.46875]
t1 = [122242.0,163769.0,184763.0,199176.0,191727.0]
m_new = 8945.36427302845
m_again = submission.iloc[0:5,1]
sqrt(mean_squared_error(t, m_again))<load_from_csv>
cp_type = np.array(train_features['cp_type'])
 print('Train')
 print('count of trt_cp:',(cp_type == 'trt_cp' ).sum())
 print('count of ctl_vehicle:',(cp_type == 'ctl_vehicle' ).sum())
 cp_type = np.array(test_features['cp_type'])
 print('Test')
 print('count of trt_cp:',(cp_type == 'trt_cp' ).sum())
 print('count of ctl_vehicle:',(cp_type == 'ctl_vehicle' ).sum() )
X_train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv', index_col='Id')
 X_test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv', index_col='Id')
 X_train.drop(X_train[X_train.GrLivArea > 4500].index, inplace=True)
 y = np.log1p(X_train['SalePrice'])
 X_train.drop(columns=['SalePrice'], inplace=True )<define_variables>
cp_dose = np.array(train_features['cp_dose'])
 print('Train')
 print('count of D1:',(cp_dose == 'D1' ).sum())
 print('count of D2:',(cp_dose == 'D2' ).sum())
 cp_dose = np.array(test_features['cp_dose'])
 print('Test')
 print('count of D1:',(cp_dose == 'D1' ).sum())
 print('count of D2:',(cp_dose == 'D2' ).sum() )
cat_cols = [cname for cname in X_train.columns if X_train[cname].dtype == 'object']
num_cols = [cname for cname in X_train.columns if X_train[cname].dtype in ['int64', 'float64']]
num_cols_zeros = []
num_cols_median = []
cat_cols_freq = []
cat_cols_none = []
for column in num_cols:
if X_train[column].min() == 0:
num_cols_zeros.append(column)
 else:
num_cols_median.append(column)
 for column in cat_cols:
if X_train[column].isnull().sum() < 30:
cat_cols_freq.append(column)
 else:
cat_cols_none.append(column)
 zero_imputer = Pipeline([ 
('impute', SimpleImputer(strategy='constant', fill_value=0.0)) , 
('scale', StandardScaler())
 ])
 medi_imputer = Pipeline([ 
('impute', SimpleImputer(strategy='median')) , 
('scale', StandardScaler())
 ])
 none_imputer = Pipeline([ 
('impute', SimpleImputer(strategy='constant', fill_value='none')) , 
('OH_encode', OneHotEncoder(handle_unknown='ignore', sparse=False))
])
 freq_imputer = Pipeline([ 
('impute', SimpleImputer(strategy='most_frequent')) , 
('OH_encode', OneHotEncoder(handle_unknown='ignore', sparse=False))
])
 target_encoder = Pipeline([ 
('encode', ce.TargetEncoder(cols=cat_cols)) , 
('scale', StandardScaler())
 ])
 count_encoder = Pipeline([ 
('encode', ce.CountEncoder()), 
('scale', StandardScaler())
 ])
 preprocessor = make_column_transformer(
(zero_imputer, num_cols_zeros), 
(medi_imputer, num_cols_median), 
(none_imputer, cat_cols_none), 
(freq_imputer, cat_cols_freq), 
(count_encoder, cat_cols), 
(target_encoder, cat_cols)
)
 def full_area(X):
X_copy = X.copy()
X_copy['full_area'] = X['TotalBsmtSF'] + X['1stFlrSF'] + X['2ndFlrSF'] + X['GarageArea'] + X['MasVnrArea'] + X['WoodDeckSF'] + X['OpenPorchSF'] + X['ScreenPorch']
return X_copy
pre_pipe = Pipeline([ 
('features', FunctionTransformer(full_area)) , 
('preprocessing', preprocessor)
 ] )<import_modules>
train = train_features.merge(train_targets, on = 'sig_id' )
random_state=8
models = [
XGBRegressor(random_state=random_state),
SVR() ,
DecisionTreeRegressor(random_state=random_state),
AdaBoostRegressor(DecisionTreeRegressor(random_state=random_state)) ,
RandomForestRegressor(random_state=random_state),
ExtraTreesRegressor(random_state=random_state),
GradientBoostingRegressor(random_state=random_state),
MLPRegressor(random_state=random_state),
KNeighborsRegressor() ,
Ridge(random_state=random_state),
]
def evaluate_model(X, y, models, pre_pipe):
performance = {}
for model in models:
X_transformed = pre_pipe.fit_transform(X, y)
 score = cross_val_score(model, X_transformed, y, cv=5 ).mean()
print("Model: {}, mean cross validated score: {}".format(type(model ).__name__, score))
performance[type(model ).__name__] = score
return performance
model_scores = evaluate_model(X_train, y, models, pre_pipe)
 top_models = sorted(model_scores, key=model_scores.get, reverse=True)[:5]
print("The best models are: {}".format(top_models))<init_hyperparams>
d = ['cp_type']
d.extend(train_targets.columns[1:])
 train1 = train[d]
ar = np.array(train1[train1['cp_type'] == 'ctl_vehicle'])[:, 1:] 
(ar != 0 ).sum()
GBR_params = {'model__n_estimators' : np.arange(100,500,50),
'model__learning_rate': np.arange(0.01, 0.1, 0.01),
'model__max_depth': np.arange(2,10,1),
'model__min_samples_leaf': np.arange(100,200,10),
'model__alpha': np.arange(0.7,0.9,0.05)}
Ridge_params = {'model__alpha': np.arange(0.5, 1.5, 0.1)}
ExtraT_params = {'model__n_estimators' : np.arange(50,1000,50)}
Ada_params = {'model__n_estimators' : np.arange(50,1000,50),
'model__learning_rate': np.arange(0.01, 0.1, 0.01)}
XGB_params = {'model__n_estimators' : np.arange(100,2000,100),
'model__learning_rate': np.arange(0.01, 0.1, 0.01),
'model__max_depth': np.arange(1,10,1)}
SVR_params = {'model__epsilon': np.arange(0.1, 0.5, 0.1),
'model__C': np.arange(1, 20, 1)}
KNR_params = {'model__leaf_size': np.arange(5, 50, 5),
'model__n_neighbors': np.arange(3,10,1),
'model__p': np.arange(1,4,1)}
model_list = [GradientBoostingRegressor() , Ridge() , ExtraTreesRegressor() , AdaBoostRegressor() , XGBRegressor() ]
param_list = [GBR_params, Ridge_params, ExtraT_params, Ada_params, XGB_params]
iterations = 250
best_estimators = {}
for i in range(len(model_list)) :
model_name = type(model_list[i] ).__name__
full_pipeline = Pipeline([ 
('preprocessing', pre_pipe), 
('model', model_list[i])])
 fitted_model = RandomizedSearchCV(full_pipeline, param_distributions = param_list[i], n_iter=iterations, cv=5, scoring="neg_mean_squared_error", verbose=0)
 fitted_model.fit(X_train, y)
 print("Model: {}, best cross validated score: {}".format(model_name, -1*fitted_model.best_score_))
best_estimators[model_name] = fitted_model.best_estimator_
voting = VotingRegressor([ 
('GradientBoostingRegressor', best_estimators['GradientBoostingRegressor']), 
('Ridge', best_estimators['Ridge']), 
('ExtraTreesRegressor',best_estimators['ExtraTreesRegressor']), 
('AdaBoostRegressor', best_estimators['AdaBoostRegressor']), 
('XGBRegressor', best_estimators['XGBRegressor'])
 ])
 voting = voting.fit(X_train, y)
 preds = np.expm1(voting.predict(X_test))<save_to_csv>
d = ['cp_type']
d.extend(train_targets.columns[1:])
 train1 = train[d]
ar = np.array(train1[train1['cp_type'] == 'trt_cp'])[:, 1:] 
(ar != 0 ).sum() /(ar.shape[0] * ar.shape[1] )
output = pd.DataFrame({'Id': X_test.index,
'SalePrice': preds})
 output.to_csv('submission.csv', index=False)
<load_from_csv>
d = ['cp_type', 'cp_time']
d.extend(train_targets.columns[1:])
 train1 = train[d]
ar = np.array(train1[(train1['cp_type'] == 'trt_cp')&(train1['cp_time'] == 24)])[:, 2:] 
(ar != 0 ).sum() /(ar.shape[0] * ar.shape[1] )
train_all = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test_all = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 sample_submission = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/sample_submission.csv' )<sort_values>
d = ['cp_type', 'cp_time']
d.extend(train_targets.columns[1:])
 train1 = train[d]
ar = np.array(train1[(train1['cp_type'] == 'trt_cp')&(train1['cp_time'] == 48)])[:, 2:] 
(ar != 0 ).sum() /(ar.shape[0] * ar.shape[1] )
train_all.corr() ['SalePrice'].sort_values(ascending=False)[1:]<feature_engineering>
d = ['cp_type', 'cp_time']
d.extend(train_targets.columns[1:])
 train1 = train[d]
ar = np.array(train1[(train1['cp_type'] == 'trt_cp')&(train1['cp_time'] == 72)])[:, 2:] 
(ar != 0 ).sum() /(ar.shape[0] * ar.shape[1] )
train_all['SalePrice'] = np.log1p(train_all['SalePrice'])
 cont_plot(train_all, 'SalePrice' )<concatenate>
d = ['cp_type', 'cp_dose']
d.extend(train_targets.columns[1:])
 train1 = train[d]
ar = np.array(train1[(train1['cp_type'] == 'trt_cp')&(train1['cp_dose'] == 'D1')])[:, 2:] 
(ar != 0 ).sum() /(ar.shape[0] * ar.shape[1] )
all_dataset = pd.concat(( train_all, test_all)).reset_index(drop = True)
 all_dataset.drop(['SalePrice'], axis = 1, inplace = True)
 all_dataset.shape<define_variables>
d = ['cp_type', 'cp_dose']
d.extend(train_targets.columns[1:])
 train1 = train[d]
ar = np.array(train1[(train1['cp_type'] == 'trt_cp')&(train1['cp_dose'] == 'D2')])[:, 2:] 
(ar != 0 ).sum() /(ar.shape[0] * ar.shape[1] )
none_features = ["Alley",
"PoolQC",
"MiscFeature",
"Fence",
"FireplaceQu",
"GarageType",
"GarageFinish",
"GarageQual",
"GarageCond",
'BsmtQual',
'BsmtCond',
'BsmtExposure',
'BsmtFinType1',
'BsmtFinType2',
'MasVnrType']
for feature in none_features:
all_dataset[feature] = all_dataset[feature].fillna('None' )<count_missing_values>
d = ['cp_type', 'cp_time', 'cp_dose']
d.extend(train_targets.columns[1:])
 train1 = train[d]
ar = np.array(train1[(train1['cp_type'] == 'trt_cp')&(train1['cp_time'] == 72)&(train1['cp_dose'] == 'D2')])[:, 3:] 
(ar != 0 ).sum() /(ar.shape[0] * ar.shape[1] )
print('If the BsmtFinType1 is {}, then the BsmtFinSF1 should be 0 instead of {}'.format(
(all_dataset[all_dataset['BsmtFinSF1'].isnull() == True]['BsmtFinType1'].iloc[0]),
all_dataset[all_dataset['BsmtFinSF1'].isnull() == True]['BsmtFinSF1'].iloc[0])) 
<define_variables>
d = ['cp_type', 'cp_time']
d.extend(train_targets.columns[1:])
 train1 = train[d]
ar = np.array(train1[(train1['cp_type'] == 'trt_cp')])[:, 2:] 
(ar != 0 ).sum() /(ar.shape[0] * ar.shape[1] )
zero_features =['BsmtFinSF1',
'BsmtFinSF2',
'BsmtUnfSF',
'TotalBsmtSF',
'BsmtFullBath',
'BsmtHalfBath',
'GarageYrBlt',
'GarageArea',
'GarageCars',
'MasVnrArea']
for feature in zero_features:
all_dataset[feature] = all_dataset[feature].fillna(0 )<groupby>
d = ['cp_type', 'cp_time']
d.extend(train_targets.columns[1:])
 train1 = train[d]
ar = np.array(train1[(train1['cp_type'] == 'trt_cp')])[:, 2:] 
(ar != 1 ).sum() /(ar.shape[0] * ar.shape[1] )
all_dataset['LotFrontage'] = all_dataset.groupby('LotShape')['LotFrontage'].transform(lambda x: x.fillna(x.median()))<categorify>
features_g = ['g-' + str(i)for i in range(772)]
features_c = ['c-' + str(i)for i in range(100)]
all_dataset['MSSubClass'] = all_dataset['MSSubClass'].astype(str)
 all_dataset['MSZoning'] = all_dataset.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))<categorify>
for df in train_features, test_features:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_median'] = df[features_g].median(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_median'] = df[features_c].median(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['gc_median'] = df[features_g + features_c].median(axis = 1)
null_cat = ['Utilities', 'Functional', 'SaleType',
'Exterior1st', 'Electrical', 'KitchenQual',
'Exterior2nd']
for feature in null_cat:
all_dataset[feature] = all_dataset[feature].fillna(statistics.mode(all_dataset[feature]))<data_type_conversions>
train_targets = train_targets[train_features['cp_type'] == 'trt_cp']
train_features = train_features[train_features['cp_type'] == 'trt_cp']
train_features = train_features.drop(columns = ['sig_id', 'cp_type'])
 test1_features = test_features.copy()
test_features = test_features.drop(columns = ['sig_id', 'cp_type'])
 cat_columns = ['cp_time', 'cp_dose']
cat_train_features = train_features[cat_columns]
cat_test_features = test_features[cat_columns]
train_features = train_features.drop(columns = cat_columns)
 test_features = test_features.drop(columns = cat_columns)
 train = np.array(train_features)
 test = np.array(test_features)
 targets = train_targets.columns
train_targets = np.array(train_targets[train_targets.columns[1:]] )
features_to_object = ['YrSold', 'MoSold']
for feature in features_to_object:
all_dataset[feature] = all_dataset[feature].astype(str )<feature_engineering>
batch_size = 128
val_batch_size = batch_size * 4
def box_transform(df):
numerical_features = df.dtypes[df.dtypes != 'object'].index.drop('Id')
 for feature in numerical_features:
df[feature] = scipy.special.boxcox1p(df[feature], scipy.stats.boxcox_normmax(df[feature] + 1))<categorify>
one_hot = OneHotEncoder()
X2_train = one_hot.fit_transform(cat_train_features)
 X2_test = one_hot.transform(cat_test_features )
final = pd.get_dummies(all_dataset ).reset_index(drop=True ).drop('Id', axis=1)
<prepare_x_and_y>
train = np.concatenate(( X2_train.toarray() , train), 1)
 test = np.concatenate(( X2_test.toarray() , test), 1 )
X_train = final.iloc[:len(train_all),:]
X_test = final.iloc[len(train_all):,:]
Y_train = train_all['SalePrice']<split>
n_folds = 7
n_starts = 1
n_epochs = 70
n_targets = train_targets.shape[1]
X_sub_train, X_sub_test, Y_sub_train, Y_sub_test = train_test_split(X_train, Y_train, test_size = 0.3, random_state=42)
 print(X_sub_train.shape, Y_sub_train.shape)
 print(X_sub_test.shape, Y_sub_test.shape )<train_on_grid>
criterion = nn.BCELoss()
kfold = MultilabelStratifiedKFold(n_splits = 7, random_state = 42, shuffle = True )
ridge = Ridge(normalize=True)
 ridge_params = {'alpha':np.linspace(0.1,1,10),'tol':np.linspace(0.001,0.01,10)}
ridge_search = GridSearchCV(ridge, ridge_params, cv=5)
 ridge_search.fit(X_train, Y_train )<train_model>
class SimpleNet(nn.Module):
def __init__(self, num_columns):
super(SimpleNet, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_columns)
 self.dropout1 = nn.Dropout(0.4)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_columns, 1024))
self.batch_norm2 = nn.BatchNorm1d(1024)
 self.dropout2 = nn.Dropout(0.4)
 self.dense2 = nn.utils.weight_norm(nn.Linear(1024, 1024))
self.batch_norm3 = nn.BatchNorm1d(1024)
 self.dropout3 = nn.Dropout(0.4)
 self.dense3 = nn.utils.weight_norm(nn.Linear(1024, 206))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.sigmoid(self.dense3(x))
return x
alpha_ridge = list(ridge_search.best_params_.values())[0]
tol_ridge = list(ridge_search.best_params_.values())[1]
ridge_test = Ridge(alpha = alpha_ridge, tol = tol_ridge, normalize=True)
 ridge_test.fit(X_sub_train, Y_sub_train)
 Y_fit_ridge = ridge_test.predict(X_sub_test)
 rmse_ridge = np.sqrt(mean_squared_error(Y_sub_test, Y_fit_ridge))
rmse_ridge<train_on_grid>
class Dataset1(Dataset):
def __init__(self, df, targets, mode='train'):
self.mode = mode
self.df = df
if mode == 'train':
self.targets = targets
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
if self.mode == 'train':
return torch.FloatTensor(self.df[index]), torch.FloatTensor(self.targets[index])
 elif self.mode == 'test':
return torch.FloatTensor(self.df[index]), 0
lasso = Lasso(normalize=True)
 lasso_params = {'alpha':np.linspace(0.00001,0.001,100)}
lasso_search = GridSearchCV(lasso, lasso_params, cv=5)
 lasso_search.fit(X_train, Y_train )<compute_train_metric>
if torch.cuda.is_available() :
device = 'cuda'
device
alpha_lasso = list(lasso_search.best_params_.values())[0]
lasso_test = Lasso(alpha=alpha_lasso, normalize=True)
 lasso_test.fit(X_sub_train, Y_sub_train)
 Y_fit_lasso = lasso_test.predict(X_sub_test)
 rmse_lasso = np.sqrt(mean_squared_error(Y_sub_test, Y_fit_lasso))
rmse_lasso<train_on_grid>
submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
lgbm = LGBMRegressor()
lgbm_params = {'num_leaves':[5,10,20], 'max_depth':[3,5,8]
,'learning_rate':[0.1,0.2,0.3], 'n_estimators':[75,100,125]}
lgbm_search = GridSearchCV(lgbm, lgbm_params, cv=5)
 lgbm_search.fit(X_train, Y_train )<train_model>
for n,(tr, te)in enumerate(kfold.split(train_targets, train_targets)) :
print('Train fold:', n + 1)
 xtrain, xval = train[tr], train[te]
ytrain, yval = train_targets[tr], train_targets[te]
train_set = Dataset1(xtrain, ytrain)
 val_set = Dataset1(xval, yval)
 dataloaders = {
'train': DataLoader(train_set, batch_size=batch_size, shuffle=True),
'val': DataLoader(val_set, batch_size=val_batch_size, shuffle=False)
 }
model = SimpleNet(895 ).to(device)
 checkpoint_path = f'repeat:{0}_Fold:{n+1}.pt'
optimizer = optim.Adam(model.parameters() , weight_decay=1e-5)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=3, eps=1e-4, verbose=True)
 best_loss = {'train': np.inf, 'val': np.inf}
for epoch in range(n_epochs):
epoch_loss = {'train': 0.0, 'val': 0.0}
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
running_loss = 0.0
for i,(x, y)in enumerate(dataloaders[phase]):
x, y = x.to(device), y.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase=='train'):
preds = model(x)
 loss = criterion(preds, y)
 if phase=='train':
loss.backward()
optimizer.step()
running_loss += loss.item() / len(dataloaders[phase])
 epoch_loss[phase] = running_loss
print("Epoch {}/{} - loss: {:5.5f} - val_loss: {:5.5f}".format(epoch + 1, n_epochs, epoch_loss['train'], epoch_loss['val']))
scheduler.step(epoch_loss['val'])
 if epoch_loss['val'] < best_loss['val']:
best_loss = epoch_loss
torch.save(model.state_dict() , checkpoint_path )
learning_rate_lgbm = list(lgbm_search.best_params_.values())[0]
max_depth_lgbm = list(lgbm_search.best_params_.values())[1]
n_estimators_lgbm = list(lgbm_search.best_params_.values())[2]
num_leaves_lgbm = list(lgbm_search.best_params_.values())[3]
lgbm_test = LGBMRegressor(learning_rate = learning_rate_lgbm, max_depth = max_depth_lgbm,
n_estimators = n_estimators_lgbm, num_leaves= num_leaves_lgbm)
 lgbm_test.fit(X_sub_train, Y_sub_train)
 Y_fit_lgbm = lgbm_test.predict(X_sub_test)
 rmse_lgbm = np.sqrt(mean_squared_error(Y_sub_test, Y_fit_lgbm))
rmse_lgbm<train_on_grid>
oof = np.zeros(( train.shape[0], n_targets - 1))
oof_targets = np.zeros(( train.shape[0], n_targets - 1))
preds = np.zeros(( test.shape[0], n_targets - 1))
elastic = ElasticNet()
elastic_params = {'alpha':np.linspace(0.005,0.01,5), 'l1_ratio':[0.01,0.05]}
elastic_search = GridSearchCV(elastic, elastic_params, cv=5)
 elastic_search.fit(X_train,Y_train )<train_on_grid>
def mean_log_loss(y_true, y_pred):
metrics = []
for i, target in enumerate(targets[1:]):
metrics.append(log_loss(y_true[:, i], y_pred[:, i].astype(float), labels=[0,1]))
return np.mean(metrics )
alpha_elastic = list(elastic_search.best_params_.values())[0]
l1_ratio_elastic = list(elastic_search.best_params_.values())[1]
elastic_test = ElasticNet(alpha = alpha_elastic
, l1_ratio = l1_ratio_elastic)
 elastic_test.fit(X_sub_train, Y_sub_train)
 Y_fit_elastic = elastic_test.predict(X_sub_test)
 rmse_elastic = np.sqrt(mean_squared_error(Y_sub_test, Y_fit_elastic))
rmse_elastic<init_hyperparams>
seed_targets = []
seed_oof = []
seed_preds = np.zeros(( test.shape[0], n_targets, n_folds))
for n,(tr, te)in enumerate(kfold.split(train_targets, train_targets)) :
xval, yval = train[te], train_targets[te]
fold_preds = []
val_set = Dataset1(xval, yval)
 test_set = Dataset1(test, None, mode='test')
 dataloaders = {
'val': DataLoader(val_set, batch_size=val_batch_size, shuffle=False),
'test': DataLoader(test_set, batch_size=val_batch_size, shuffle=False)
 }
checkpoint_path = f'repeat:{0}_Fold:{n+1}.pt'
model = SimpleNet(895 ).to(device)
 model.load_state_dict(torch.load(checkpoint_path))
model.eval()
for phase in ['val', 'test']:
for i,(x, y)in enumerate(dataloaders[phase]):
if phase == 'val':
x, y = x.to(device), y.to(device)
 elif phase == 'test':
x = x.to(device)
 with torch.no_grad() :
batch_preds = model(x)
 if phase == 'val':
seed_targets.append(y)
 seed_oof.append(batch_preds)
 elif phase == 'test':
fold_preds.append(batch_preds)
 fold_preds = torch.cat(fold_preds, dim=0 ).cpu().numpy()
seed_preds[:, :, n] = fold_preds
seed_targets = torch.cat(seed_targets, dim=0 ).cpu().numpy()
seed_oof = torch.cat(seed_oof, dim=0 ).cpu().numpy()
seed_preds = np.mean(seed_preds, axis=2)
 print("Overall score is {:5.5f}".format(mean_log_loss(seed_targets, seed_oof)) )
<choose_model_class>
submission[targets[1:]] = seed_preds
submission.loc[test1_features['cp_type']=='ctl_vehicle', targets[1:]] = 0
submission.to_csv('submission.csv', index=False )