Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<import_modules>
%matplotlib inline
color = sns.color_palette()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
print(check_output(["ls", ".. /input/house-prices-advanced-regression-techniques/"] ).decode("utf8"))
from sklearn.metrics import log_loss, roc_auc_score
from datetime import datetime
import datatable as dt
from datatable.models import Ftrl<load_from_csv>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
%%time
train = dt.fread('.. /input/train.csv' )<load_from_csv>
lam = 0.0
train["SalePrice"] = boxcox1p(train["SalePrice"], lam )
%%time
test = dt.fread('.. /input/test.csv' )<count_unique_values>
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))
train.nunique()<count_unique_values>
all_data.isnull()
test.nunique()<count_unique_values>
all_data.isnull().sum()
train[:, 'EngineVersion'].nunique1()<count_unique_values>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head(20 )
train_unique = dt.unique(train[:, 'EngineVersion'] ).to_list() [0]
len(train_unique )<count_unique_values>
all_data["PoolQC"] = all_data["PoolQC"].fillna("None" )
test_unique = dt.unique(test[:, 'EngineVersion'] ).to_list() [0]
len(test_unique )<count_unique_values>
all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None" )
intersection = list(set(train_unique)& set(test_unique))
len(intersection )<categorify>
all_data["Alley"] = all_data["Alley"].fillna("None" )
<categorify>
all_data["Fence"] = all_data["Fence"].fillna("None" )
%%time
for f in train.names:
if f not in ['MachineIdentifier', 'HasDetections']:
if train[:, f].ltypes[0] == dt.ltype.str:
print('factorizing %s' % f)
 col_f = pd.concat([train[:, f].to_pandas() , test[:, f].to_pandas() ], ignore_index=True)
 encoding = col_f.groupby(f ).size()
encoding = encoding/len(col_f)
 column = col_f[f].map(encoding ).values.flatten()
del col_f, encoding
gc.collect()
train[:, f] = dt.Frame(column[:8921483])
 test[:, f] = dt.Frame(column[8921483:])
 del column
gc.collect()<choose_model_class>
all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None" )
features = [f for f in train.names if f not in ['HasDetections']]
ftrl = Ftrl(nepochs=2, interactions=True)
<train_model>
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
%%time
print('Start Fitting on ', train.shape, ' @ ', datetime.now())
 ftrl.fit(train[:, features], train[:, 'HasDetections'])
 print('Fitted complete on ', train.shape, ' @ ', datetime.now())
 print('Current loss : %.6f'
% log_loss(np.array(train[:, 'HasDetections'])[:, 0],
np.array(ftrl.predict(train[:, features]))))<compute_train_metric>
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None' )
print('Current AUC : %.6f'
% roc_auc_score(np.array(train[:, 'HasDetections'])[:, 0],
np.array(ftrl.predict(train[:, features]))))<predict_on_test>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0 )
preds1 = np.array(ftrl.predict(test[:, features]))
preds1 = preds1.flatten()<train_on_grid>
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0 )
ftrl = Ftrl(nepochs=20, interactions=False)
 ftrl.fit(train[:, features], train[:, 'HasDetections'])
 preds2 = np.array(ftrl.predict(test[:, features]))
preds2 = preds2.flatten()<save_model>
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None' )
np.save('preds1', preds1)
 np.save('preds2', preds2 )<load_from_csv>
all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )
sample_submission = pd.read_csv('.. /input/sample_submission.csv' )<define_variables>
all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0] )
sample_submission['HasDetections'] = 0.6*preds1+0.4*preds2<save_to_csv>
all_data = all_data.drop(['Utilities'], axis=1 )
sample_submission.to_csv('datatable_ftrl_submission.csv', index=False )<define_variables>
all_data["Functional"] = all_data["Functional"].fillna("Typ" )
dtypes = {
'MachineIdentifier': 'category',
'ProductName': 'category',
'EngineVersion': 'category',
'AppVersion': 'category',
'AvSigVersion': 'category',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float16',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int8',
'Platform': 'category',
'Processor': 'category',
'OsVer': 'category',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'category',
'OsBuildLab': 'category',
'SkuEdition': 'category',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'category',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'category',
'Firewall': 'float16',
'UacLuaenable': 'float32',
'Census_MDC2FormFactor': 'category',
'Census_DeviceFamily': 'category',
'Census_OEMNameIdentifier': 'float16',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float16',
'Census_ProcessorClass': 'category',
'Census_PrimaryDiskTotalCapacity': 'float32',
'Census_PrimaryDiskTypeName': 'category',
'Census_SystemVolumeTotalCapacity': 'float32',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'category',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float16',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float16',
'Census_InternalPrimaryDisplayResolutionVertical': 'float16',
'Census_PowerPlatformRoleName': 'category',
'Census_InternalBatteryType': 'category',
'Census_InternalBatteryNumberOfCharges': 'float32',
'Census_OSVersion': 'category',
'Census_OSArchitecture': 'category',
'Census_OSBranch': 'category',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'category',
'Census_OSSkuName': 'category',
'Census_OSInstallTypeName': 'category',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'category',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'category',
'Census_ActivationChannel': 'category',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'category',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'int8'
}<load_from_csv>
all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0] )
train = pd.read_csv('.. /input/train.csv', iterator=True, chunksize=1_500_000, dtype=dtypes)
 test = pd.read_csv('.. /input/test.csv', iterator=True, chunksize=1_000_000, dtype=dtypes )<feature_engineering>
all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0] )
gf_defaults = {'col': [], 'ocol':[], 'dcol' : ['EngineVersion', 'AppVersion', 'AvSigVersion', 'OsBuildLab', 'Census_OSVersion']}
one_hot = {}
def get_features(df, gf_train=False):
global one_hot
global gf_defaults
for c in gf_defaults['dcol']:
for i in range(5):
df[c + str(i)] = df[c].map(lambda x: str(x ).split('.')[i] if len(str(x ).split('.')) >i else -1)
 col = [c for c in df.columns if c not in ['MachineIdentifier', 'HasDetections']]
if gf_train:
for c in col:
if df[c].dtype == 'O' or df[c].dtype.name == 'category':
gf_defaults['ocol'].append(c)
 else:
gf_defaults['col'].append(c)
 one_hot = {c: list(df[c].value_counts().index)for c in gf_defaults['ocol']}
for c in one_hot:
if len(one_hot[c])>1 and len(one_hot[c])< 20:
for val in one_hot[c]:
df[c+'_oh_' + str(val)] =(df[c].values == val ).astype(np.int)
 if gf_train:
gf_defaults['col'].append(c+'_oh_' + str(val))
return df[gf_defaults['col']+['MachineIdentifier', 'HasDetections']]<split>
all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0] )
col = gf_defaults['col']
model = []
params = {'objective':'binary', "boosting": "gbdt", 'learning_rate': 0.02, 'max_depth': -1,
"feature_fraction": 0.8, "bagging_freq": 1, "bagging_fraction": 0.8 , "bagging_seed": 11,
"metric": 'auc', "lambda_l1": 0.1, 'num_leaves': 60, 'min_data_in_leaf': 60, "verbosity": -1, "random_state": 3}
online_start = True
for df in train:
if online_start:
df = get_features(df, True)
 x1, x2, y1, y2 = model_selection.train_test_split(df[col], df['HasDetections'], test_size=0.2, random_state=25)
 model = lgb.train(params, lgb.Dataset(x1, y1), 2500, lgb.Dataset(x2, y2), verbose_eval=100, early_stopping_rounds=200)
 model.save_model('lgb.model')
 else:
df = get_features(df)
 x1, x2, y1, y2 = model_selection.train_test_split(df[col], df['HasDetections'], test_size=0.2, random_state=25)
 model = lgb.train(params, lgb.Dataset(x1, y1), 2500, lgb.Dataset(x2, y2), verbose_eval=100, early_stopping_rounds=200, init_model='lgb.model')
 model.save_model('lgb.model')
 online_start = False
print('training...' )<predict_on_test>
all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0] )
predictions = []
for df in test:
df['HasDetections'] = 0.0
df = get_features(df)
 df['HasDetections'] = model.predict(df[col], num_iteration=model.best_iteration + 50)
 predictions.append(df[['MachineIdentifier', 'HasDetections']].values)
 print('testing...' )<save_to_csv>
all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None" )
sub = np.concatenate(predictions)
 sub = pd.DataFrame(sub, columns = ['MachineIdentifier', 'HasDetections'])
 sub.to_csv('submission.csv', index=False )<define_variables>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head()
LABELS = ["HasDetections"]<load_from_csv>
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )
predict_list = []
predict_list.append(pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_5.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_6.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_7.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_8.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_9.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/new-blend/super_blend.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/new-blend/super_blend.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/new-blend/super_blend.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/new-blend/super_blend.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/new-blend/super_blend.csv")[LABELS].values)
<save_to_csv>
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))
print("Rank averaging on ", len(predict_list), " files")
 predictions = np.zeros_like(predict_list[0])
 for predict in predict_list:
for i in range(1):
predictions[:, i] = np.add(predictions[:, i], rankdata(predict[:, i])/predictions.shape[0])
 predictions /= len(predict_list)
 submission = pd.read_csv('.. /input/microsoft-malware-prediction/sample_submission.csv')
 submission[LABELS] = predictions
submission.to_csv('super_blend.csv', index=False )<define_variables>
all_data
LABELS = ["HasDetections"]<load_from_csv>
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
predict_list = []
predict_list.append(pd.read_csv(".. /input/lightgbm/submission_lgbm.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/malware-predictions-3500-trees/submission0.72968.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/detecting-malwares-with-ftrl-proximal/submission.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/outputs-for-microsoft/submission_ashish_kfold.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_5.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_6.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_7.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_8.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_9.csv")[LABELS].values )<save_to_csv>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam)
print("Rank averaging on ", len(predict_list), " files")
 predictions = np.zeros_like(predict_list[0])
 for predict in predict_list:
for i in range(1):
predictions[:, i] = np.add(predictions[:, i], rankdata(predict[:, i])/predictions.shape[0])
 predictions /= len(predict_list)
 submission = pd.read_csv('.. /input/microsoft-malware-prediction/sample_submission.csv')
 submission[LABELS] = predictions
submission.to_csv('super_blend.csv', index=False )<import_modules>
all_data = pd.get_dummies(all_data)
 print(all_data.shape )
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
import seaborn as sns
from math import sqrt
from sklearn import metrics<load_from_csv>
train = all_data[:ntrain]
test = all_data[ntrain:]
df_base0 = pd.read_csv(".. /input/lightgbm/submission_lgbm.csv",names=["MachineIdentifier","HasDetections0"], skiprows=[0],header=None)
 df_base1 = pd.read_csv(".. /input/malware-predictions-3500-trees/submission0.72968.csv",names=["MachineIdentifier","HasDetections1"], skiprows=[0],header=None)
 df_base2 = pd.read_csv(".. /input/detecting-malwares-with-ftrl-proximal/submission.csv",names=["MachineIdentifier","HasDetections2"], skiprows=[0],header=None)
 df_base3 = pd.read_csv(".. /input/outputs-for-microsoft/submission_ashish_kfold.csv",names=["MachineIdentifier","HasDetections3"], skiprows=[0],header=None)
 df_base4 = pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_5.csv",names=["MachineIdentifier","HasDetections4"], skiprows=[0],header=None)
 df_base5 = pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_6.csv",names=["MachineIdentifier","HasDetections5"], skiprows=[0],header=None)
 df_base6 = pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_7.csv",names=["MachineIdentifier","HasDetections6"], skiprows=[0],header=None)
 df_base7 = pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_8.csv",names=["MachineIdentifier","HasDetections7"], skiprows=[0],header=None)
 df_base8 = pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_9.csv",names=["MachineIdentifier","HasDetections8"], skiprows=[0],header=None )<merge>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
df_base = pd.merge(df_base0,df_base1,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base2,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base3,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base4,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base5,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base6,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base7,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base8,how='inner',on='MachineIdentifier' )<compute_test_metric>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
M = np.zeros([df_base.iloc[:,1:].shape[1],df_base.iloc[:,1:].shape[1]])
 for i in np.arange(M.shape[1]):
for j in np.arange(M.shape[1]):
M[i,j] = sqrt(metrics.mean_squared_error(df_base.iloc[:,i+1], df_base.iloc[:,j+1]))<save_to_csv>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
df_base['HasDetections'] = df_base.iloc[:,1:].mean(axis=1)
 df_base[['MachineIdentifier','HasDetections']].to_csv("Bestoutput.csv",index=False )<load_from_csv>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
df_base0 = pd.read_csv(".. /input/lightgbm/submission_lgbm.csv",names=["MachineIdentifier","HasDetections0"], skiprows=[0],header=None)
 df_base1 = pd.read_csv(".. /input/malware-predictions-3500-trees/submission0.72968.csv",names=["MachineIdentifier","HasDetections1"], skiprows=[0],header=None)
 df_base2 = pd.read_csv(".. /input/detecting-malwares-with-ftrl-proximal/submission.csv",names=["MachineIdentifier","HasDetections2"], skiprows=[0],header=None)
 df_base3 = pd.read_csv(".. /input/outputs-for-microsoft/submission_ashish_kfold.csv",names=["MachineIdentifier","HasDetections3"], skiprows=[0],header=None)
 df_base8 = pd.read_csv(".. /input/hung-the-nguyen/submission_lgbm_9.csv",names=["MachineIdentifier","HasDetections8"], skiprows=[0],header=None )<merge>
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5 )
df_base = pd.merge(df_base0,df_base1,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base2,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base3,how='inner',on='MachineIdentifier')
 df_base = pd.merge(df_base,df_base8,how='inner',on='MachineIdentifier' )<compute_test_metric>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )
M = np.zeros([df_base.iloc[:,1:].shape[1],df_base.iloc[:,1:].shape[1]])
 for i in np.arange(M.shape[1]):
for j in np.arange(M.shape[1]):
M[i,j] = sqrt(metrics.mean_squared_error(df_base.iloc[:,i+1], df_base.iloc[:,j+1]))<save_to_csv>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1 )
df_base['HasDetections'] = df_base.iloc[:,1:].mean(axis=1)
 df_base[['MachineIdentifier','HasDetections']].to_csv("Bestoutput1.csv",index=False )<load_from_csv>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )
double_blend = df_base['HasDetections'] * 0.5 + submission['HasDetections'] * 0.5
sample = pd.read_csv('.. /input/microsoft-malware-prediction/sample_submission.csv')
 sample.head()<save_to_csv>
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
sample['HasDetections'] = double_blend
sample.to_csv('Bouble_blend.csv', index=False )<set_options>
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
gc.enable()<define_variables>
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
dtypes = {
'MachineIdentifier': 'category',
'ProductName': 'category',
'EngineVersion': 'category',
'AppVersion': 'category',
'AvSigVersion': 'category',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float16',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int8',
'Platform': 'category',
'Processor': 'category',
'OsVer': 'category',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'category',
'OsBuildLab': 'category',
'SkuEdition': 'category',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'category',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'category',
'Firewall': 'float16',
'UacLuaenable': 'float32',
'Census_MDC2FormFactor': 'category',
'Census_DeviceFamily': 'category',
'Census_OEMNameIdentifier': 'float16',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float16',
'Census_ProcessorClass': 'category',
'Census_PrimaryDiskTotalCapacity': 'float32',
'Census_PrimaryDiskTypeName': 'category',
'Census_SystemVolumeTotalCapacity': 'float32',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'category',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float16',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float16',
'Census_InternalPrimaryDisplayResolutionVertical': 'float16',
'Census_PowerPlatformRoleName': 'category',
'Census_InternalBatteryType': 'category',
'Census_InternalBatteryNumberOfCharges': 'float32',
'Census_OSVersion': 'category',
'Census_OSArchitecture': 'category',
'Census_OSBranch': 'category',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'category',
'Census_OSSkuName': 'category',
'Census_OSInstallTypeName': 'category',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'category',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'category',
'Census_ActivationChannel': 'category',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'category',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'int8'
}<load_from_csv>
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
print('Download Train and Test Data.
 ')
 train = reduce_mem_usage(pd.read_csv('.. /input/train.csv', dtype=dtypes, low_memory=True))
train['MachineIdentifier'] = train.index.astype('uint32')
 test = reduce_mem_usage(pd.read_csv('.. /input/test.csv', dtype=dtypes, low_memory=True))
test['MachineIdentifier'] = test.index.astype('uint32')
 gc.collect()<categorify>
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
print('Transform all features to category.
 ')
 for usecol in train.columns.tolist() [1:-1]:
train[usecol] = train[usecol].astype('str')
 test[usecol] = test[usecol].astype('str')
 le = LabelEncoder().fit(
 np.unique(train[usecol].unique().tolist() +
test[usecol].unique().tolist()))
train[usecol] = le.transform(train[usecol])+1
test[usecol] = le.transform(test[usecol])+1
agg_tr =(train 
.groupby([usecol])
.aggregate({'MachineIdentifier':'count'})
.reset_index() 
.rename({'MachineIdentifier':'Train'}, axis=1))
agg_te =(test 
.groupby([usecol])
.aggregate({'MachineIdentifier':'count'})
.reset_index() 
.rename({'MachineIdentifier':'Test'}, axis=1))
agg = pd.merge(agg_tr, agg_te, on=usecol, how='outer' ).replace(np.nan, 0)
 agg = agg[(agg['Train'] > 1000)].reset_index(drop=True)
 agg['Total'] = agg['Train'] + agg['Test']
agg = agg[(agg['Train'] / agg['Total'] > 0.2)&(agg['Train'] / agg['Total'] < 0.8)]
agg[usecol+'Copy'] = agg[usecol]
train[usecol] =(pd.merge(train[[usecol]],
agg[[usecol, usecol+'Copy']],
on=usecol, how='left')[usecol+'Copy'] 
.replace(np.nan, 0 ).astype('int' ).astype('category'))
test[usecol] =(pd.merge(test[[usecol]],
agg[[usecol, usecol+'Copy']],
on=usecol, how='left')[usecol+'Copy'] 
.replace(np.nan, 0 ).astype('int' ).astype('category'))
del le, agg_tr, agg_te, agg, usecol
gc.collect()
y_train = np.array(train['HasDetections'])
 train_ids = train.index
test_ids = test.index
del train['HasDetections'], train['MachineIdentifier'], test['MachineIdentifier']
gc.collect()<categorify>
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
ohe = OneHotEncoder(categories='auto', sparse=True, dtype='uint8' ).fit(train)
 m = 100000
train = vstack([ohe.transform(train[i*m:(i+1)*m])for i in range(train.shape[0] // m + 1)])
 test = vstack([ohe.transform(test[i*m:(i+1)*m])for i in range(test.shape[0] // m + 1)])
 save_npz('train.npz', train, compressed=True)
 save_npz('test.npz', test, compressed=True)
 del ohe, train, test
gc.collect()<train_model>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 skf.get_n_splits(train_ids, y_train)
 lgb_test_result_1 = np.zeros(test_ids.shape[0])
 oof_1= np.zeros(train_ids.shape[0])
 counter = 0
print('
LightGBM
')
 for train_index, test_index in skf.split(train_ids, y_train):
print('Fold {}
'.format(counter + 1))
train = load_npz('train.npz')
 X_fit = vstack([train[train_index[i*m:(i+1)*m]] for i in range(train_index.shape[0] // m + 1)])
 X_val = vstack([train[test_index[i*m:(i+1)*m]] for i in range(test_index.shape[0] // m + 1)])
 X_fit, X_val = csr_matrix(X_fit, dtype='float32'), csr_matrix(X_val, dtype='float32')
 y_fit, y_val = y_train[train_index], y_train[test_index]
del train
gc.collect()
lgb_model = lgb.LGBMClassifier(max_depth=-1,
n_estimators=30000,
learning_rate=0.05,
num_leaves=2**12-1,
colsample_bytree=0.28,
objective='binary',
n_jobs=-1)
 lgb_model.fit(X_fit, y_fit, eval_metric='auc',
eval_set=[(X_val, y_val)],
verbose=100, early_stopping_rounds=100)
 oof_1[test_index] += lgb_model.predict_proba(X_val)[:,1]
del X_fit, X_val, y_fit, y_val, train_index, test_index
gc.collect()
test = load_npz('test.npz')
 test = csr_matrix(test, dtype='float32')
 lgb_test_result_1 += lgb_model.predict_proba(test)[:,1]
counter += 1
del test
gc.collect()<save_to_csv>
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['HasDetections'] =(lgb_test_result_1 / counter)
 submission.to_csv('lgb_submission.csv', index=False )<set_options>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
warnings.simplefilter(action='ignore', category=FutureWarning)
 init_notebook_mode(connected=True)
 pd.set_option('display.max_columns', 500 )<define_variables>
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))
dtypes = {
'MachineIdentifier': 'category',
'ProductName': 'category',
'EngineVersion': 'category',
'AppVersion': 'category',
'AvSigVersion': 'category',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float16',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int8',
'Platform': 'category',
'Processor': 'category',
'OsVer': 'category',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'category',
'OsBuildLab': 'category',
'SkuEdition': 'category',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'category',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'category',
'Firewall': 'float16',
'UacLuaenable': 'float32',
'Census_MDC2FormFactor': 'category',
'Census_DeviceFamily': 'category',
'Census_OEMNameIdentifier': 'float16',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float16',
'Census_ProcessorClass': 'category',
'Census_PrimaryDiskTotalCapacity': 'float32',
'Census_PrimaryDiskTypeName': 'category',
'Census_SystemVolumeTotalCapacity': 'float32',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'category',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float16',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float16',
'Census_InternalPrimaryDisplayResolutionVertical': 'float16',
'Census_PowerPlatformRoleName': 'category',
'Census_InternalBatteryType': 'category',
'Census_InternalBatteryNumberOfCharges': 'float32',
'Census_OSVersion': 'category',
'Census_OSArchitecture': 'category',
'Census_OSBranch': 'category',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'category',
'Census_OSSkuName': 'category',
'Census_OSInstallTypeName': 'category',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'category',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'category',
'Census_ActivationChannel': 'category',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'category',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'int8'
}<define_variables>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
numerical_columns = [c for c,v in dtypes.items() if v in numerics]
categorical_columns = [c for c,v in dtypes.items() if v not in numerics]<load_from_csv>
stacked_averaged_models.fit(train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))
nrows = 4000000
retained_columns = numerical_columns + categorical_columns
train = pd.read_csv('.. /input/train.csv',
nrows = nrows,
usecols = retained_columns,
dtype = dtypes)
 retained_columns += ['MachineIdentifier']
retained_columns.remove('HasDetections')
 test = pd.read_csv('.. /input/test.csv',
usecols = retained_columns,
dtype = dtypes )<define_variables>
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))
true_numerical_columns = [
'Census_ProcessorCoreCount',
'Census_PrimaryDiskTotalCapacity',
'Census_SystemVolumeTotalCapacity',
'Census_TotalPhysicalRAM',
'Census_InternalPrimaryDiagonalDisplaySizeInInches',
'Census_InternalPrimaryDisplayResolutionHorizontal',
'Census_InternalPrimaryDisplayResolutionVertical',
'Census_InternalBatteryNumberOfCharges'
]<count_unique_values>
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))
binary_variables = [c for c in train.columns if train[c].nunique() == 2]<define_variables>

 print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.70 +
xgb_train_pred*0.15 + lgb_train_pred*0.15))
categorical_columns = [c for c in train.columns
if(c not in true_numerical_columns)&(c not in binary_variables)]<categorify>
ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15
<define_variables><EOS> 
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('submission.csv',index=False )