Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<save_to_csv>
warnings.filterwarnings('ignore')
 %matplotlib inline
sample_submission = pd.read_csv('.. /input/microsoft-malware-prediction/sample_submission.csv')
 sample_submission['HasDetections'] = predictions_3
sample_submission.to_csv('submission_ashish_kfold.csv', index=False )<import_modules>
data_train = pd.read_csv('.. /input/train.csv')
 print(data_train.columns)
 data_test = pd.read_csv('.. /input/test.csv' )
import numpy as np
import pandas as pd
from gensim.models import Word2Vec
from sklearn.decomposition import PCA
from sklearn.feature_extraction.text import TfidfVectorizer
from gensim.test.utils import common_texts, get_tmpfile
from keras.preprocessing.text import Tokenizer
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn import metrics
from keras.layers import *
from keras.models import *
from keras import initializers, regularizers, constraints, optimizers, layers
from keras.initializers import *
from keras.optimizers import *
import keras.backend as K
from keras.callbacks import *
import tensorflow as tf
from keras.preprocessing.sequence import pad_sequences
from smart_open import smart_open
import datetime
from keras.utils import multi_gpu_model
import os
import time
import gc
import re
import random 
<load_from_csv>
elt_name = ['Alley','BsmtQual','BsmtCond','BsmtExposure','BsmtFinType1','BsmtFinType2','FireplaceQu'
,'GarageType','GarageFinish','GarageQual','GarageCond','PoolQC','Fence','MiscFeature']
data_train.update(data_train[elt_name].fillna('None'))
data_test.update(data_test[elt_name].fillna('None'))
def convertCats2sentence(x,cols):
ref = [[col.lower() + '=' +(str(val)).lower() for val in x[col]] for col in cols]
tmp = pd.DataFrame.from_records(ref)
 tmp = tmp.transpose()
X = tmp.values.tolist()
return X
chunksize = 100000
header = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv',nrows=1)
 cols = list(set(header.head(0)) - set(['Census_PrimaryDiskTotalCapacity',
'Census_SystemVolumeTotalCapacity','HasDetections']))
n = 8921484 - 1
s = 500000
skip = sorted(random.sample(range(1,n+1),n-s))
X = []
t0 = time.time()
steps = 1
for chunk in pd.read_csv('.. /input/microsoft-malware-prediction/train.csv', chunksize=chunksize,dtype='category',skiprows=skip):
print('step...'+str(steps))
X.extend(convertCats2sentence(chunk,cols))
steps = steps + 1
t1 = time.time()
print(t1-t0)
 print('train converted...')
 gc.collect()<train_model>
total = data_train.isnull().sum().sort_values(ascending=False)
 percent =(data_train.isnull().sum() /data_train.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total,percent],axis=1,keys=['Total','Percent'])
 missing_data.head(20)
embed_size = 300
maxlen = len(cols)
 max_features = None
window_size = 40
t0 = time.time()
model = Word2Vec(X,size=embed_size,window=window_size,min_count=1)
 t1 = time.time()
print(t1-t0)
 print('model fit...')
 tokenizer = Tokenizer(num_words=max_features, filters='')
 tokenizer.fit_on_texts(X)
<load_from_csv>
total_test = data_test.isnull().sum().sort_values(ascending=False)
 percent_test =(data_test.isnull().sum() /data_test.isnull().count() ).sort_values(ascending=False)
 missing_data_test = pd.concat([total_test,percent_test],axis=1,keys=['Total','Percent'])
 missing_data_test.head(20 )
n = 8921484 - 1
s = 1000000
skip = sorted(random.sample(range(1,n+1),n-s))
X = []
t0 = time.time()
steps = 1
gc.collect()
for chunk in pd.read_csv('.. /input/microsoft-malware-prediction/train.csv', chunksize=chunksize,dtype='category',skiprows=skip):
print('step...'+str(steps))
X.extend(convertCats2sentence(chunk,cols))
steps = steps + 1
t1 = time.time()
print(t1-t0)
 print('train converted...')
 X = tokenizer.texts_to_sequences(X)
 X = pad_sequences(X, maxlen=maxlen)
 word_index = tokenizer.word_index
max_features = len(word_index)+1
train = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv',usecols=['HasDetections'],skiprows=skip)
 Y = train['HasDetections']
del train
gc.collect()<compute_train_metric>
print(( missing_data[missing_data['Total'] > 1] ).index)
 data_train.update(data_train['Electrical'].fillna(data_train['Electrical'].dropna().mode().values.item()))
col_num=[f for f in data_train.columns if data_train.dtypes[f] != 'object']
data_train.update(data_train[col_num].fillna(0))
col_obj=[f for f in data_train.columns if data_train.dtypes[f] == 'object']
data_train.update(data_train[col_obj].fillna('None'))
data_train.isnull().sum().max()
def load_wv(model, word_index):
embedding_matrix = np.random.normal(model.wv[model.wv.vocab].mean() , model.wv[model.wv.vocab].std() , 
(max_features, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
try:
embedding_vector = model.wv[word]
if embedding_vector is not None: embedding_matrix[i] = embedding_vector
except:
print(word, ' not found')
 return embedding_matrix
embedding_matrix = load_wv(model, word_index )<load_from_csv>
col_num=[f for f in data_test.columns if data_test.dtypes[f] != 'object']
data_test.update(data_test[col_num].fillna(0))
col_obj=[f for f in data_test.columns if data_test.dtypes[f] == 'object']
data_test.update(data_test[col_obj].fillna('None'))
data_test.isnull().sum().max()
sub = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv', usecols=['MachineIdentifier'])
<statistical_test>
saleprice_scaled = StandardScaler().fit_transform(data_train['SalePrice'][:,np.newaxis]);
low_range = saleprice_scaled[saleprice_scaled[:,0].argsort() ][:10]
high_range= saleprice_scaled[saleprice_scaled[:,0].argsort() ][-10:]
print('outer range(low)of the distribution:')
 print(low_range)
 print('
outer range(high)of the distribution:')
 print(high_range )
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3,
kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule)
 def capsule() :
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size,
weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.2 )(x)
 x = Bidirectional(CuDNNLSTM(100, return_sequences=True,
kernel_initializer=glorot_normal(seed=12300),
recurrent_initializer=orthogonal(gain=1.0,
seed=10000)) )(x)
 x = Capsule(num_capsule=10, dim_capsule=5, routings=2, share_weights=True )(x)
 x = Flatten()(x)
 x = Dense(100, activation="relu", kernel_initializer=glorot_normal(seed=12300))(x)
 x = Dropout(0.12 )(x)
 x = BatchNormalization()(x)
 x = Dense(100, activation="relu", kernel_initializer=glorot_normal(seed=12300))(x)
 x = Dropout(0.2 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam() ,)
 return model
def auc(y_true, y_pred):
fpr, tpr, thresholds = metrics.roc_curve(y_true, y_pred, pos_label=1)
 return metrics.auc(fpr, tpr )<save_to_csv>
outlier_ids = data_train.sort_values(by = 'GrLivArea', ascending = False)[:2]['Id']
print(outlier_ids.index)
 data_train = data_train.drop(outlier_ids.index)
 print(data_train.sort_values(by = 'GrLivArea', ascending = False)[:2][['SalePrice','GrLivArea']] )
sub['HasDetections'] = y_test.reshape(( -1, 1))
tmp = pd.read_csv('.. /input/lgbyhat/lgb_submission.csv')
 tmp.columns = ['MachineIdentifier','HD']
sub = pd.merge(sub,tmp)
 sub['mean'] = sub.mean(axis=1)
 sub.drop(['HD','HasDetections'],axis=1,inplace=True)
 sub.columns = ['MachineIdentifier','HasDetections']
sub.to_csv("submission.csv", index=False )<load_from_csv>
data_train['SalePrice'] = np.log(data_train['SalePrice'] )
def load(x):
ignore = ['MachineIdentifier']
if x in ignore: return False
else: return True
df_train = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv',dtype='category',usecols=load)
 df_train['HasDetections'] = df_train['HasDetections'].astype('int8')
 if 5244810 in df_train.index:
df_train.loc[5244810,'AvSigVersion'] = '1.273.1144.0'
df_train['AvSigVersion'].cat.remove_categories('1.2&
df_test = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv',dtype='category',usecols=load)
<categorify>
data_train['GrLivArea'] = np.log(data_train['GrLivArea'])
 data_test['GrLivArea'] = np.log(data_test['GrLivArea'] )
def factor_data(df_train, df_test, col):
df_comb = pd.concat([df_train[col],df_test[col]],axis=0)
 df_comb,_ = df_comb.factorize(sort=True)
 df_comb += 1
df_comb = np.where(df_comb==0, df_comb.max() +1, df_comb)
 df_train[col] = df_comb[:len(df_train)]
df_test[col] = df_comb[len(df_train):]
del df_comb
def reduce_memory(df,col):
mx = df[col].max()
if mx<256:
df[col] = df[col].astype('uint8')
 elif mx<65536:
df[col] = df[col].astype('uint16')
 else:
df[col] = df[col].astype('uint32')
 def relax_data(df_train, df_test, col):
cv1 = pd.DataFrame(df_train[col].value_counts().reset_index().rename({col:'train'},axis=1))
cv2 = pd.DataFrame(df_test[col].value_counts().reset_index().rename({col:'test'},axis=1))
cv3 = pd.merge(cv1,cv2,on='index',how='outer')
 cv3['train'].fillna(0,inplace=True)
 cv3['test'].fillna(0,inplace=True)
 factor = len(df_test)/len(df_train)
 cv3['remove'] = False
cv3['remove'] = cv3['remove'] |(cv3['train'] < len(df_train)/9000)
 cv3['remove'] = cv3['remove'] |(factor*cv3['train'] < cv3['test']/4)
 cv3['remove'] = cv3['remove'] |(factor*cv3['train'] > 4*cv3['test'])
 cv3['new'] = cv3.apply(lambda x: x['index'] if x['remove']==False else 0,axis=1)
 cv3['new'],_ = cv3['new'].factorize(sort=True)
 cv3.set_index('index',inplace=True)
 cc = cv3['new'].to_dict()
df_train[col] = df_train[col].map(cc)
 reduce_memory(df_train,col)
 df_test[col] = df_test[col].map(cc)
 reduce_memory(df_test,col)
 def display_memory(df_train, df_test):
print(len(df_train),'rows of training data use',df_train.memory_usage(deep=True ).sum() //1e6,'Mb memory!')
 print(len(df_test),'rows of test data use',df_test.memory_usage(deep=True ).sum() //1e6,'Mb memory!')
 def categorize(df_train, df_test, cols):
for col in cols:
df_train[col] = df_train[col].astype('category')
 df_test[col] = df_test[col].astype('category' )<categorify>
data_train['HasBsmt'] = pd.Series(len(data_train['TotalBsmtSF']), index=data_train.index)
 data_train['HasBsmt'] = 0
data_train.loc[data_train['TotalBsmtSF']>0,'HasBsmt'] = 1
data_test['HasBsmt'] = pd.Series(len(data_test['TotalBsmtSF']), index=data_test.index)
 data_test['HasBsmt'] = 0
data_test.loc[data_test['TotalBsmtSF']>0,'HasBsmt'] = 1
del df_trainA, df_trainB, df_trainC, df_trainD
del df_train['DateAS'], df_test['DateAS']; x=gc.collect()
cols = [x for x in df_train.columns if x not in ['HasDetections','AvSigVersion2']]
print('Factorizing...')
 for col in cols: factor_data(df_train, df_test, col)
 print('Reducing memory...')
 for col in cols: reduce_memory(df_train, col)
 for col in cols: reduce_memory(df_test, col)
 categorize(df_train, df_test, cols)
 display_memory(df_train, df_test )<train_model>
data_train.loc[data_train['HasBsmt']==1,'TotalBsmtSF'] = np.log(data_train['TotalBsmtSF'])
 data_test.loc[data_test['HasBsmt']==1,'TotalBsmtSF'] = np.log(data_test['TotalBsmtSF'] )
df_trainA = df_train.sample(frac=0.5)
 df_trainB = df_train[ ~df_train.index.isin(df_trainA.index)]
model = lgb.LGBMClassifier(n_estimators=3000, colsample_bytree=0.2, objective='binary', num_leaves=16,
max_depth=-1, learning_rate=0.1)
 h=model.fit(df_trainA[cols], df_trainA['HasDetections'], eval_metric='auc',
eval_set=[(df_trainB[cols], df_trainB['HasDetections'])], verbose=250,
early_stopping_rounds=100 )<save_to_csv>
data_all = pd.concat([data_train.drop(['SalePrice'], axis=1), data_test] ).reset_index(drop=True)
 data_all = pd.get_dummies(data_all)
 data_all.drop(['Id'],axis=1,inplace=True)
del df_trainA, df_trainB; x=gc.collect()
idx = 0; chunk = 2000000
pred_val = np.zeros(len(df_test))
while idx < len(df_test):
idx2 = min(idx + chunk, len(df_test))
idx = range(idx, idx2)
 pred_val[idx] = model.predict_proba(df_test.iloc[idx][cols])[:,1]
idx = idx2
submit = pd.read_csv('.. /input/microsoft-malware-prediction/sample_submission.csv')
 submit['HasDetections'] = pred_val
submit.to_csv('ModelOne.csv', index=False )<train_model>
y = data_train['SalePrice'].reset_index(drop=True)
 x_train = data_all.iloc[:len(y), :]
x_test = data_all.iloc[len(y):, :]
train = data_all[:len(y)]
test = data_all[len(y):]
x_train.shape, x_test.shape, y.shape,train.shape,test.shape
df_trainC = df_train[ df_train['AvSigVersion2']<275 ]
df_trainD = df_train[ df_train['AvSigVersion2']>=275 ]
model = lgb.LGBMClassifier(n_estimators=3000, colsample_bytree=0.2, objective='binary', num_leaves=16,
max_depth=-1, learning_rate=0.1)
 h=model.fit(df_trainC[cols], df_trainC['HasDetections'], eval_metric='auc',
eval_set=[(df_trainD[cols], df_trainD['HasDetections'])], verbose=250,
early_stopping_rounds=100 )<train_model>
print('START ML', datetime.now() ,)
 kfolds = KFold(n_splits=5, shuffle=True, random_state=42)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, x=x_train, y=y):
rmse = np.sqrt(-cross_val_score(model, x, y, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse )
print('Converting data to Model Two...')
 df_trainC = df_trainC.copy()
df_trainD = df_trainD.copy()
for col in cols: relax_data(df_trainC, df_trainD, col)
 categorize(df_trainC, df_trainD, cols)
 model = lgb.LGBMClassifier(n_estimators=3000, colsample_bytree=0.2, objective='binary', num_leaves=16,
max_depth=-1, learning_rate=0.1)
 h=model.fit(df_trainC[cols], df_trainC['HasDetections'], eval_metric='auc',
eval_set=[(df_trainD[cols], df_trainD['HasDetections'])], verbose=250,
early_stopping_rounds=100 )<train_model>
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
print('Converting data to Model Two...')
 del df_trainC, df_trainD; x=gc.collect()
for col in cols: relax_data(df_train, df_test, col)
 categorize(df_train, df_test, cols)
 df_trainA = df_train.sample(frac=0.5)
 df_trainB = df_train[ ~df_train.index.isin(df_trainA.index)]
model = lgb.LGBMClassifier(n_estimators=3000, colsample_bytree=0.2, objective='binary', num_leaves=16,
max_depth=-1, learning_rate=0.1)
 h=model.fit(df_trainA[cols], df_trainA['HasDetections'], eval_metric='auc',
eval_set=[(df_trainB[cols], df_trainB['HasDetections'])], verbose=250,
early_stopping_rounds=100 )<save_to_csv>
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7, alphas=alphas2, random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, alphas=e_alphas, cv=kfolds, l1_ratio=e_l1ratio))
del df_trainA, df_trainB, df_train; x=gc.collect()
idx = 0; chunk = 2000000
pred_val = np.zeros(len(df_test))
while idx < len(df_test):
idx2 = min(idx + chunk, len(df_test))
idx = range(idx, idx2)
 pred_val[idx] = model.predict_proba(df_test.iloc[idx][cols])[:,1]
idx = idx2
submit = pd.read_csv('.. /input/microsoft-malware-prediction/sample_submission.csv')
 submit['HasDetections'] = pred_val
submit.to_csv('ModelTwo.csv', index=False )<load_from_csv>
gbr = GradientBoostingRegressor(n_estimators=2000, learning_rate=0.01, max_depth=30, max_features='sqrt', min_samples_leaf=15, min_samples_split=10, loss='huber', random_state =42 )
df_train = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv',dtype='category',usecols=load, nrows=10000)
 df_train['HasDetections'] = df_train['HasDetections'].astype('int8')
 if 5244810 in df_train.index:
df_train.loc[5244810,'AvSigVersion'] = '1.273.1144.0'
df_train['AvSigVersion'].cat.remove_categories('1.2&
df_test = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv',dtype='category',usecols=load, nrows=10000)
 cols = [x for x in df_train.columns if x not in ['HasDetections','AvSigVersion2']]
for col in cols: factor_data(df_train, df_test, col)
 for col in cols: reduce_memory(df_train, col)
 for col in cols: reduce_memory(df_test, col)
 categorize(df_train, df_test, cols)
 df_train['HasDetections'] = 0
df_test['HasDetections'] = 1
df_comb = pd.concat([df_train,df_test],axis=0)
 model = DecisionTreeClassifier(max_leaf_nodes=5)
 model.fit(df_comb[cols], df_comb['HasDetections'])
 pred_val = model.predict_proba(df_comb[cols])[:,1]
print('Model One: Adversarial Training AUC = ',round(roc_auc_score(df_comb['HasDetections'],pred_val),4))
print('Adversarial Model has max_leaf_nodes=5')
 tree_graph = tree.export_graphviz(model, out_file=None, max_depth = 10,
impurity = False, feature_names = cols, class_names = ['No', 'Yes'],
rounded = True, filled= True)
 graphviz.Source(tree_graph )<choose_model_class>
random_forest = RandomForestRegressor(n_estimators=2000,
max_depth=30,
min_samples_split=5,
min_samples_leaf=5,
max_features=None,
random_state=10,
oob_score=True
)
for col in cols: relax_data(df_train, df_test, col)
 categorize(df_train, df_test, cols)
 df_comb = pd.concat([df_train,df_test],axis=0)
 cols2 = cols.copy()
cols2.remove('SMode')
 model = DecisionTreeClassifier(max_leaf_nodes=5)
 model.fit(df_comb[cols2], df_comb['HasDetections'])
 pred_val = model.predict_proba(df_comb[cols2])[:,1]
print('Model Two: Adversarial Training AUC = ',round(roc_auc_score(df_comb['HasDetections'],pred_val),4))
print('Adversarial Model has max_leaf_nodes=5')
 tree_graph = tree.export_graphviz(model, out_file=None, max_depth = 10,
impurity = False, feature_names = cols2, class_names = ['No', 'Yes'],
rounded = True, filled= True)
 graphviz.Source(tree_graph )<load_from_csv>
xgboost = XGBRegressor(learning_rate=0.01,n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006 )
cols2 = {'AVProductStatesIdentifier':0.01, 'CountryIdentifier':0.4, 'LocaleEnglishNameIdentifier':0.3, 'SmartScreen':0.4,
'Census_OEMNameIdentifier':0.1,'Census_TotalPhysicalRAM':0.5,'Census_InternalPrimaryDiagonalDisplaySizeInInches':0.05,
'Census_OSInstallTypeName':0.75,'Census_OSInstallLanguageIdentifier':0.3,'Census_FirmwareManufacturerIdentifier':0.1,
'EngineVersion':1.0, 'AppVersion':0.7, 'OsBuildLab':0.2, 'Census_OEMModelIdentifier':0.15,
'Census_InternalBatteryNumberOfCharges':0.05}
df_train = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv',dtype='category',usecols=load)
 df_test = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv',dtype='category',usecols=load)
 df_test['DateAS'] = df_test['AvSigVersion'].map(datedictAS)
 df_testA = df_test[ df_test['DateAS']<datetime(2018,10,25)]
df_testB = df_test[ df_test['DateAS']>datetime(2018,10,25)]
for x in df_train.columns[:-2]:
s = 0.5
if x in cols2: s = cols2[x]
comparePlot(df_train,df_testA,x,scale=s, title='Public Test vs.Train', prefix='Public ')
 comparePlot(df_train,df_testB,x,scale=s, title='Private Test vs.Train', prefix='Private ' )<set_options>
stack_gen = StackingCVRegressor(regressors=(ridge, lasso, random_forest),
meta_regressor=gbr,
use_features_in_secondary=True )
%matplotlib inline
plt.style.use('ggplot')
 py.init_notebook_mode(connected=True)
 warnings.filterwarnings("ignore")
 logging.basicConfig(filename='log.txt',level=logging.DEBUG, format='%(asctime)s %(message)s')
 pd.set_option('max_colwidth', 500)
 pd.set_option('max_columns', 500)
 pd.set_option('max_rows', 100)
 print(os.listdir(".. /input")) 
<compute_test_metric>
print('TEST score on CV')
 names = ["Kernel Ridge", "Lasso", "ElasticNet", "xgboost", "gbr", "stack_gen"]
models= [ridge, lasso, elasticnet, xgboost, gbr, stack_gen]
for name, model in zip(names, models):
score = cv_rmse(model)
 print("{} score: {:.6f}({:.4f})".format(name,score.mean() ,score.std()))
@jit
def fast_auc(y_true, y_prob):
y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
nfalse = 0
auc = 0
n = len(y_true)
 for i in range(n):
y_i = y_true[i]
nfalse +=(1 - y_i)
 auc += y_i * nfalse
auc /=(nfalse *(n - nfalse))
return auc
def eval_auc(preds, dtrain):
labels = dtrain.get_label()
return 'auc', fast_auc(labels, preds), True<define_variables>
class MixModelWeight() :
def __init__(self,mod,weight):
self.mod = mod
self.weight = weight
def fit(self,X,y):
for model in self.mod:
model.fit(X,y)
 return self
def predict(self,X):
w = list()
pred = np.array([model.predict(X)for model in self.mod])
 for data in range(pred.shape[1]):
single = [pred[model,data]*weight for model,weight in zip(range(pred.shape[0]),self.weight)]
w.append(np.sum(single))
return w
mix_model = MixModelWeight(mod = [elasticnet,lasso,ridge,xgboost,stack_gen],weight=[0.1,0.1,0.1,0.1,0.6])
 print('START fit model')
 mix_model.fit(x_train,y)
 print('RMSLE score on train data:')
 score = rmsle(y, mix_model.predict(x_train))
print(score )
dtypes = {
'MachineIdentifier': 'object',
'ProductName': 'object',
'EngineVersion': 'object',
'AppVersion': 'object',
'AvSigVersion': 'object',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float32',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int16',
'Platform': 'object',
'Processor': 'object',
'OsVer': 'object',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'object',
'OsBuildLab': 'object',
'SkuEdition': 'object',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'object',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'object',
'Firewall': 'float16',
'UacLuaenable': 'float64',
'Census_MDC2FormFactor': 'object',
'Census_DeviceFamily': 'object',
'Census_OEMNameIdentifier': 'float32',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float32',
'Census_ProcessorClass': 'object',
'Census_PrimaryDiskTotalCapacity': 'float64',
'Census_PrimaryDiskTypeName': 'object',
'Census_SystemVolumeTotalCapacity': 'float64',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'object',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float32',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float32',
'Census_InternalPrimaryDisplayResolutionVertical': 'float32',
'Census_PowerPlatformRoleName': 'object',
'Census_InternalBatteryType': 'object',
'Census_InternalBatteryNumberOfCharges': 'float64',
'Census_OSVersion': 'object',
'Census_OSArchitecture': 'object',
'Census_OSBranch': 'object',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'object',
'Census_OSSkuName': 'object',
'Census_OSInstallTypeName': 'object',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'object',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'object',
'Census_ActivationChannel': 'object',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'object',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'float32',
}
def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage(deep=True ).sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage(deep=True ).sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df<define_variables>
print('Predict submission', datetime.now() ,)
 submission=pd.DataFrame({'Id':data_test['Id'], 'SalePrice':np.floor(np.exp(mix_model.predict(x_test)))})
 submission_stack=pd.DataFrame({'Id':data_test['Id'], 'SalePrice':np.floor(np.exp(stack_gen.predict(x_test)))})
 submission_mix=pd.DataFrame({'Id':data_test['Id'], 'SalePrice':(submission['SalePrice'])*0.8+(submission_stack['SalePrice']*0.2)})
 print(submission )
<load_from_csv><EOS> 
submission.to_csv("submission_v4_1.csv", index=False)
 submission_stack.to_csv("submission_v4_2.csv", index=False)
 submission_mix.to_csv("submission_v4_3.csv", index=False)
 print('Save submission', datetime.now() , )