Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<choose_model_class>
torch.__version__
<compute_test_metric>
if False:
loader, optimizer, model, loss_fn =...
 swa_model = AveragedModel(model)
 scheduler = CosineAnnealingLR(optimizer, T_max=100)
 swa_start = 5
swa_scheduler = SWALR(optimizer, swa_lr=0.05)
 for epoch in range(100):
for input, target in loader:
optimizer.zero_grad()
loss_fn(model(input), target ).backward()
optimizer.step()
if epoch > swa_start:
swa_model.update_parameters(model)
 swa_scheduler.step()
else:
scheduler.step()
torch.optim.swa_utils.update_bn(loader, swa_model)
 preds = swa_model(test_input )
cv_score = np.mean(cross_val_score(final_model,X,y,scoring = 'neg_root_mean_squared_error', cv=k_fold,n_jobs = -1,verbose =0))
print('RMSE for training set :', round(np.sqrt(mean_squared_error(y, train_pred)) ,2))
print('Cross Validation RMSE for training set :', round(-cv_score,2))<define_variables>
warnings.filterwarnings('ignore' )
final_pred = sum(test_pred)/5<save_to_csv>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
submission = pd.DataFrame({"Id": test.ID, "SalePrice": final_pred})
 submission.to_csv("submission.csv", index = None)
 submission<import_modules>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import preprocessing
from sklearn import linear_model
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from lightgbm import LGBMRegressor
from xgboost import XGBRegressor
import sklearn.metrics as metrics
import math
import seaborn as sns
from scipy.stats import skew
from scipy.special import boxcox1p
from scipy.stats import boxcox_normmax
from sklearn.model_selection import cross_val_score
from sklearn.metrics import mean_squared_error
from math import sqrt
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import RepeatedKFold
from sklearn.linear_model import LinearRegression
from sklearn.neighbors import KNeighborsRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.svm import SVR
from sklearn.ensemble import StackingRegressor
from sklearn.linear_model import LassoCV
from sklearn.linear_model import RidgeCV
from sklearn.linear_model import ElasticNetCV
from sklearn.ensemble import VotingRegressor
from scipy.special import boxcox1p
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import AdaBoostRegressor
from catboost import CatBoostRegressor
from sklearn.pipeline import make_pipeline<load_from_csv>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
train_data = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test_data = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', 1000)
 train = train_data.copy()
test = test_data.copy()<drop_column>
train_targets_scored.sum() [1:].sort_values()
test = test.drop(['Alley','PoolQC','Fence','MiscFeature'],axis=1)
 test.info()<drop_column>
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
<count_missing_values>
n_comp = 15
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
test_obj.isna().sum()<count_values>
var_thresh = VarianceThreshold(threshold=0.5)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features
test['SaleType'].value_counts()<categorify>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
dataset = [train,test]
columns = ['BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','FireplaceQu','GarageType','GarageFinish','GarageQual','GarageCond','Electrical','MSZoning','Utilities','Exterior1st','Exterior2nd','MasVnrType','KitchenQual','Functional','SaleType']
for set in dataset:
for col in columns:
set[col].fillna(set[col].mode() [0],inplace=True)
<count_missing_values>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
test_num.isna().sum()<data_type_conversions>
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=5)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds
col1 = ['LotFrontage','GarageYrBlt','MasVnrArea','BsmtUnfSF','BsmtFinSF2','BsmtFinSF1','TotalBsmtSF','BsmtFullBath','BsmtHalfBath','GarageCars','GarageArea']
for set in dataset:
for c in col1:
set[c] = set[c].fillna(set[c].median() )<drop_column>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
x = torch.tensor(self.features[idx, :], dtype=torch.float)
 y = torch.tensor(self.targets[idx, :], dtype=torch.float)
 return x, y
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
x = torch.tensor(self.features[idx, :], dtype=torch.float)
 return x
train = train.drop(['Heating','RoofMatl','Condition2','Street','Utilities'],axis=1)
 test = test.drop(['Heating','RoofMatl','Condition2','Street','Utilities'],axis=1 )<feature_engineering>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.Linear(num_features, hidden_size)
 self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.5)
 self.dense2 = nn.Linear(hidden_size, hidden_size)
 self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.5)
 self.dense3 = nn.Linear(hidden_size, num_targets)
 def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
dataset = [train,test]
for set in dataset:
set.loc[set['YrSold']<set['YearBuilt'],'YrSold'] = 2009
set['House_Age'] = set['YrSold'] - set['YearBuilt']<filter>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
train.iloc[1295:1301,:]<feature_engineering>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
data1 = [train,test]
for set in data1:
set['YrBltAndRemod']=set['YearBuilt']+set['YearRemodAdd']
set['TotalSF']=set['TotalBsmtSF'] + set['1stFlrSF'] + set['2ndFlrSF']
set['Total_sqr_footage'] =(set['BsmtFinSF1'] + set['BsmtFinSF2'] +
set['1stFlrSF'] + set['2ndFlrSF'])
 set['Total_Bathrooms'] =(set['FullBath'] +(0.5 * set['HalfBath'])+
set['BsmtFullBath'] +(0.5 * set['BsmtHalfBath']))
set['Total_porch_sf'] =(set['OpenPorchSF'] + set['3SsnPorch'] +
set['EnclosedPorch'] + set['ScreenPorch'] +
set['WoodDeckSF'])
 set['haspool'] = set['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 set['has2ndfloor'] = set['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 set['hasgarage'] = set['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 set['hasbsmt'] = set['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 set['hasfireplace'] = set['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 col1 = ['MSZoning','BldgType','Exterior2nd'] 
<prepare_x_and_y>
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )
train1 = train.copy()
test1 = test.copy()
dataset1 = [train1,test1]
bin_map = {'TA':2,'Gd':3, 'Fa':1,'Ex':4,'Po':1,'None':0,'Y':1,'N':0,'Reg':3,'IR1':2,'IR2':1,'IR3':0,"None" : 0,
"No" : 2, "Mn" : 2, "Av": 3,"Gd" : 4,"Unf" : 1, "LwQ": 2, "Rec" : 3,"BLQ" : 4, "ALQ" : 5, "GLQ" : 6
}
Paved = {"N" : 0, "P" : 1, "Y" : 2,'1.5Unf':0,'SFoyer':1,'1.5Fin':2,'2.5Unf':3,'SLvl':4,'1Story':5,'2Story':6,'2.5Fin':7}
MSZ = {'C':0,'RM':1,'RH':2,'RL':3,'FV':4}
LC = {'Bnk':0,'Lvl':1,'Low':2,'HLS':3}
LC1 = {'Inside':0,'FR2':1,'Corner':2,'CulDSac':3,'FR3':4}
LS = {'Gtl':0,'Mod':1,'Sev':2}
N = {'MeadowV':0,'IDOTRR':1,'BrDale':2,'BrkSide':3,'OldTown':4,'Edwards':5,'Blueste':6,'Sawyer':6,'SWISU':7,'NPkVill':7,'NAmes':7,'Mitchel':8,'SawyerW':9,'NWAmes':9,'Gilbert':9,'Blmngtn':9,'ClearCr':10,'CollgCr':10,'Crawfor':11,'Somerst':12,'Veenker':13,'Timber':13,'NridgHt':14,'StoneBr':14,'NoRidge':15}
C1 = {'Artery':0,'RRAe':1,'Feedr':2,'RRAn':3,'RRNe':4,'RRNn':5,'PosN':6,'PosA':7,'Norm':4}
BT = {'2FmCon':0,'Duplx':1,'TwnhsI':2,'TwnhsE':3,'1Fam':4}
MSV = {'BrkCmn':0,'None':1,'BrkFace':2,'Stone':3}
F={'Slab':0,'BrkTil':1,'CBlock':2,'Stone':3,'Wood':4,'PConc':5}
EX2 = {'CBlock':0,'AsbShng':1,'BrkComm':2,'AsphShn':3,'Stone':4,'Stucco':4,'Wd Sdng':4,'MetalSd':4,
'WdShing':5,'Plywood':6,'HdBoard':6,'BrkFace':7,'VinylSd':8,'CemntBd':9,'ImStucc':10,'Other':11}
rest = {'Gambrel':0,'Gable':1,'Mansard':2,'Flat':3,'Hip':4,'Shed':5,'BrkComm':0,'AsphShn':1,
'CBlock':2,'AsbShng':3,'WdShing':4,'Wd Sdng':5,'MetalSd':5,'HdBoard':6,'Stucco':6,'Plywood':7,
'BrkFace':8,'VinylSd':9,'CemntBd':10,'Stone':11,'ImStucc':12,'Mix':0,'FuseP':1,'FuseF':2,
'FuseA':3,'SBrkr':4,'Maj2':0,'Sev':1,'Min1':2,'Min2':2,'Maj1':3,'Mod':4,'Typ':5,'CarPort':0,
'Detchd':1,'2Types':2,'Attchd':4,'BuiltIn':5,'Basment':3,'Unf':0,'RFn':1,'Fin':2,'Oth':0,
'ConLD':1,'COD':2,'ConLw':3,'WD':4,'ConLI':5,'CWD':6,'New':7,'Con':8,'AdjLand':0,'Abnorml':1,
'Family':2,'Normal':3,'Alloca':4,'Partial':5}
for set in dataset1:
set['ExterQual'] = set['ExterQual'].map(bin_map)
 set['ExterCond'] = set['ExterCond'].map(bin_map)
 set['BsmtCond'] = set['BsmtCond'].map(bin_map)
 set['BsmtQual'] = set['BsmtQual'].map(bin_map)
 set['HeatingQC'] = set['HeatingQC'].map(bin_map)
 set['KitchenQual'] = set['KitchenQual'].map(bin_map)
 set['FireplaceQu'] = set['FireplaceQu'].map(bin_map)
 set['GarageQual'] = set['GarageQual'].map(bin_map)
 set['GarageCond'] = set['GarageCond'].map(bin_map)
 set['CentralAir'] = set['CentralAir'].map(bin_map)
 set['LotShape'] = set['LotShape'].map(bin_map)
 set['BsmtExposure'] = set['BsmtExposure'].map(bin_map)
 set['BsmtFinType1'] = set['BsmtFinType1'].map(bin_map)
 set['BsmtFinType2'] = set['BsmtFinType2'].map(bin_map)
 set['PavedDrive'] = set['PavedDrive'].map(Paved)
 set['MSZoning'] = set['MSZoning'].map(MSZ)
 set['LandContour'] = set['LandContour'].map(LC)
 set['LotConfig'] = set['LotConfig'].map(LC1)
 set['LandSlope'] = set['LandSlope'].map(LS)
 set['Neighborhood'] = set['Neighborhood'].map(N)
 set['Condition1'] = set['Condition1'].map(C1)
 set['BldgType'] = set['BldgType'].map(BT)
 set['HouseStyle'] = set['HouseStyle'].map(Paved)
 set['RoofStyle'] = set['RoofStyle'].map(rest)
 set['Exterior1st'] = set['Exterior1st'].map(rest)
 set['Exterior2nd'] = set['Exterior2nd'].map(EX2)
 set['MasVnrType'] = set['MasVnrType'].map(MSV)
 set['Foundation'] = set['Foundation'].map(F)
 set['Electrical'] = set['Electrical'].map(rest)
 set['Functional'] = set['Functional'].map(rest)
 set['GarageType'] = set['GarageType'].map(rest)
 set['GarageFinish'] = set['GarageFinish'].map(rest)
 set['SaleType'] = set['SaleType'].map(rest)
 set['SaleCondition'] = set['SaleCondition'].map(rest)
 col1 = ['MSZoning','BldgType','Exterior2nd']
for set in dataset1:
for c in col1:
set[c] = set[c].fillna(set[c].median() )<drop_column>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 45
SWA_START_EPOCH = 30
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 5
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1024
log_price = np.log1p(train1['SalePrice'])
 targets = log_price
col = ['Id','SalePrice','Electrical_Mix', 'GarageQual_Ex', 'Exterior1st_Stone', 'Exterior1st_ImStucc', 'Exterior2nd_Other', 'HouseStyle_2.5Fin']
drop = ['Id','SalePrice']
train1.drop(drop,axis=1,inplace=True)
 test1.drop(['Id'],axis=1,inplace=True)
<count_values>
def train_fn(swa_model, model, optimizer, scheduler, swa_scheduler, epoch, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
data_x, data_y = data
inputs, targets = data_x.to(device), data_y.to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
if epoch > SWA_START_EPOCH:
swa_model.update_parameters(model)
 swa_scheduler.step()
else:
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
data_x, data_y = data
inputs, targets = data_x.to(device), data_y.to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data.to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
train1['MoSold'].value_counts()<sort_values>
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 swa_model = AveragedModel(model)
 model = model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1,
div_factor=1e3, max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
swa_scheduler = SWALR(optimizer, swa_lr=LEARNING_RATE, anneal_strategy="cos", anneal_epochs=10)
 loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(swa_model, model, optimizer,scheduler, swa_scheduler, epoch, loss_fn, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
oof[val_idx] = valid_preds
torch.optim.swa_utils.update_bn(trainloader, swa_model)
 torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 break
oof[val_idx] = valid_preds
torch.optim.swa_utils.update_bn(trainloader, swa_model)
 torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
numeric_feats = train1.dtypes[train1.dtypes != "object"].index
skewed_feats = train1[numeric_feats].apply(lambda x: skew(x)).sort_values(ascending=False)
 skewed_feats<define_variables>
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
cols = ['MSSubClass', 'LotFrontage', 'LotArea', 'OverallQual',
'OverallCond', 'YearBuilt', 'MasVnrArea','TotalBsmtSF', 'BsmtFinSF1', 'GarageYrBlt', 'YearRemodAdd', 'GarageArea',
'BsmtFinSF2', 'BsmtUnfSF', '1stFlrSF',
'2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath',
'BsmtHalfBath', 'FullBath', 'HalfBath', 'BedroomAbvGr',
'KitchenAbvGr', 'TotRmsAbvGrd', 'Fireplaces',
'GarageCars', 'WoodDeckSF', 'OpenPorchSF',
'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal',
'House_Age','YrBltAndRemod','TotalSF','Total_sqr_footage','Total_porch_sf','YrSold','MoSold']
scaler = preprocessing.StandardScaler()
scaler1 = preprocessing.RobustScaler()
scaler.fit(train1)
 train1 = scaler.transform(train1)
 test1 = scaler.transform(test1 )<choose_model_class>
SEED = 777
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 oof_, predictions_ = run_k_fold(NFOLDS, SEED)
 oof += oof_
predictions += predictions_
train[target_cols] = oof
test[target_cols] = predictions
rf = RandomForestRegressor(bootstrap=True,criterion='mse',random_state=42,max_depth=35,
max_features=35,n_estimators=2500,n_jobs=-1)
 xgb = XGBRegressor(booster='gbtree', colsample_bylevel=1,colsample_bynode=1, colsample_bytree=0.6, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=6, min_child_weight=4, n_estimators=4500,
n_jobs=4, nthread=None, objective='reg:linear',
reg_alpha=0.6, reg_lambda=0.6, scale_pos_weight=1,
silent=None, subsample=0.8, verbosity=1)
 lgb = LGBMRegressor(max_depth=25,num_leaves=120,learning_rate=0.01,n_jobs=-1,boosting_type='gbdt',
objective='regression',mertic='rmse', verbosity=1,bagging_fraction=0.7,
feature_fraction=0.5,bagging_frequency=6,bagging_seed=42,seed=42,
colsample_bylevel=1,colsample_bynode=1, colsample_bytree=1)
 knn = KNeighborsRegressor(n_neighbors=2,leaf_size=2,p=1,metric='manhattan',weights='distance')
 lasso = LassoCV(alphas=[5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008], max_iter=50000)
 ridge = make_pipeline(scaler1,RidgeCV(alphas=[14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5],cv=5))
svr = make_pipeline(scaler1, SVR(C= 20, epsilon= 0.008, gamma=0.0003))
gbr = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.01, max_depth=4,
max_features='sqrt', min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =42)
 dtr = DecisionTreeRegressor(criterion='mse',random_state=42,max_depth=35,
max_features='sqrt', min_samples_leaf=15, min_samples_split=10)
 abr = AdaBoostRegressor(dtr,n_estimators=4500,random_state=42,learning_rate=0.01)
 cat = CatBoostRegressor(depth=6,learning_rate=0.1,n_estimators=2500,eval_metric = 'RMSE' )<choose_model_class>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0 )
def get_stack() :
level0 = list()
level0.append(( 'rf',rf))
level0.append(( 'xgb',xgb))
level0.append(( 'knn',knn))
level0.append(( 'lgb',lgb))
level0.append(( 'svr',svr))
level1 = LinearRegression()
model = StackingRegressor(estimators=level0, final_estimator=level1, cv=5)
 return model
def get_models() :
models = {'rf':rf,'xgb':xgb,'knn':knn,'lgb':lgb,'svr':svr,'Stacking':get_stack() }
return models
def evaluate_model(model, X, y):
cv = RepeatedKFold(n_splits=5, n_repeats=3, random_state=1)
 scores = cross_val_score(model, X, y, scoring='neg_mean_absolute_error', cv=cv, n_jobs=-1, error_score='raise')
 return scores 
<train_model>
y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
svr.fit(train1,targets)
 rf.fit(train1,targets)
 xgb.fit(train1,targets)
 gbr.fit(train1,targets)
 abr.fit(train1,targets)
 lgb.fit(train1,targets)
 knn.fit(train1,targets)
 cat.fit(train1,targets )<train_on_grid>
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score)
VR = VotingRegressor([('rf',rf),('xgb',xgb),('knn',knn),('lgb',lgb),('gbr',gbr),('abr',abr),('cat',cat),('svr',svr)],n_jobs=-1)
 VR.fit(train1,targets)
 sqrt(mean_squared_error(targets, VR.predict(train1)) )<train_on_grid>
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )