Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<drop_column>
warnings.filterwarnings('ignore' )
def drop_sparse(train, test):
flist = [x for x in train.columns if not x in ['ID','target']]
for f in flist:
if len(np.unique(train[f])) <2:
train.drop(f, axis=1, inplace=True)
 test.drop(f, axis=1, inplace=True)
 return train, test<train_model>
train_df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test_df = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
X_train, X_test = drop_sparse(X_train, X_test)
 print('
Train shape: {}
Test shape: {}'.format(X_train.shape, X_test.shape))<data_type_conversions>
train_df.drop(['Id'],axis=1,inplace=True)
 test_df.drop(['Id'],axis=1,inplace=True )
def add_SumZeros(train, test, features):
flist = [x for x in train.columns if not x in ['ID','target']]
if 'SumZeros' in features:
train.insert(1, 'SumZeros',(train[flist] == 0 ).astype(int ).sum(axis=1))
test.insert(1, 'SumZeros',(test[flist] == 0 ).astype(int ).sum(axis=1))
flist = [x for x in train.columns if not x in ['ID','target']]
return train, test<compute_test_metric>
train_df.drop(train_df[(train_df['SalePrice'] < 300000)& 
(train_df['GrLivArea'] > 4000)].index,
inplace=True)
 train_df.drop(train_df[(train_df['TotalBsmtSF'] > 6000)| 
(train_df['1stFlrSF'] > 4000)].index,
inplace=True )
<data_type_conversions>
df = pd.concat([train_df.drop(['SalePrice'],axis=1),
test_df] ).reset_index(drop=True)
 df.shape
def add_SumValues(train, test, features):
flist = [x for x in train.columns if not x in ['ID','target']]
if 'SumValues' in features:
train.insert(1, 'SumValues',(train[flist] != 0 ).astype(int ).sum(axis=1))
test.insert(1, 'SumValues',(test[flist] != 0 ).astype(int ).sum(axis=1))
flist = [x for x in train.columns if not x in ['ID','target']]
return train, test<compute_test_metric>
msno.matrix(test_df )
<feature_engineering>
def missing_percentage(df):
total = df.isnull().sum().sort_values(ascending = False)[df.isnull().sum().sort_values(ascending = False)!= 0]
percent = round(df.isnull().sum().sort_values(ascending = False)/len(df)*100,2)[round(df.isnull().sum().sort_values(ascending = False)/len(df)*100,2)!= 0]
return pd.concat([total, percent], axis=1, keys=['Total','Percent'])
 missing_percentage(df )
def add_OtherAgg(train, test, features):
flist = [x for x in train.columns if not x in ['ID','target','SumZeros','SumValues']]
if 'OtherAgg' in features:
train['Mean'] = train.mean(axis=1)
 train['Median'] = train.median(axis=1)
 train['Mode'] = train.mode(axis=1)
 train['Max'] = train.max(axis=1)
 train['Var'] = train.var(axis=1)
 train['Std'] = train.std(axis=1)
 test['Mean'] = test.mean(axis=1)
 test['Median'] = test.median(axis=1)
 test['Mode'] = test.mode(axis=1)
 test['Max'] = test.max(axis=1)
 test['Var'] = test.var(axis=1)
 test['Std'] = test.std(axis=1)
 flist = [x for x in train.columns if not x in ['ID','target','SumZeros','SumValues']]
return train, test<find_best_params>
for i in ['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',
'PoolQC', 'MiscFeature', 'Alley', 'Fence', 'GarageType', 'GarageFinish',
'GarageQual', 'GarageCond', 'MasVnrType', 'FireplaceQu', 'MSSubClass']:
df[i] = df[i].fillna('None')
 for i in ['GarageYrBlt', 'GarageArea', 'GarageCars', 'BsmtFinSF1', 'BsmtFinSF2',
'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath', 'MasVnrArea']:
df[i] = df[i].fillna(0)
 for i in ['Exterior1st', 'Exterior2nd', 'KitchenQual', 'Electrical', 'MSZoning',
'SaleType', 'Functional']:
df[i] = df[i].fillna(df[i].mode() [0])
 df['LotFrontage'] = df.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
df.drop(['Utilities'],inplace=True,axis=1 )
def kmeans(X_Tr,Xte):
flist = [x for x in X_Tr.columns if not x in ['ID','target']]
flist_kmeans = []
for ncl in range(2,11):
cls = KMeans(n_clusters=ncl)
 cls.fit_predict(X_train[flist].values)
 X_Tr['kmeans_cluster_'+str(ncl)] = cls.predict(X_Tr[flist].values)
 Xte['kmeans_cluster_'+str(ncl)] = cls.predict(Xte[flist].values)
 flist_kmeans.append('kmeans_cluster_'+str(ncl))
print(flist_kmeans)
 return X_Tr,Xte<normalization>
missing_percentage(df )
def pca(X_Tr,Xte):
flist = [x for x in X_Tr.columns if not x in ['ID','target']]
n_components = 20
flist_pca = []
pca = PCA(n_components=n_components)
 x_train_projected = pca.fit_transform(StandardScaler(X_Tr[flist], axis=0))
x_test_projected = pca.transform(StandardScaler(X_test[flist], axis=0))
for npca in range(0, n_components):
X_Tr.insert(1, 'PCA_'+str(npca+1), x_train_projected[:, npca])
 Xte.insert(1, 'PCA_'+str(npca+1), x_test_projected[:, npca])
 flist_pca.append('PCA_'+str(npca+1))
print(flist_pca )<normalization>
df['MSSubClass'] = df['MSSubClass'].astype(str)
 df['OverallCond'] = df['OverallCond'].astype(str)
 df['YrSold'] = df['YrSold'].astype(str)
 df['MoSold'] = df['MoSold'].astype(str )
PERC_TRESHOLD = 0.98
N_COMP = 97
print("
Start decomposition process...")
 print("PCA")
 pca = PCA(n_components=N_COMP, random_state=17)
 pca_results_train = pca.fit_transform(X_train)
 pca_results_test = pca.transform(X_test)
 print(pca.explained_variance_ratio_)
 print("tSVD")
 tsvd = TruncatedSVD(n_components=N_COMP, random_state=17)
 tsvd_results_train = tsvd.fit_transform(X_train)
 tsvd_results_test = tsvd.transform(X_test)
 print("ICA")
 ica = FastICA(n_components=N_COMP, random_state=17)
 ica_results_train = ica.fit_transform(X_train)
 ica_results_test = ica.transform(X_test)
 print("GRP")
 grp = GaussianRandomProjection(n_components=N_COMP, eps=0.1, random_state=17)
 grp_results_train = grp.fit_transform(X_train)
 grp_results_test = grp.transform(X_test)
 print("SRP")
 srp = SparseRandomProjection(n_components=N_COMP, dense_output=True, random_state=17)
 srp_results_train = srp.fit_transform(X_train)
 srp_results_test = srp.transform(X_test)
 print("Append decomposition components to datasets...")
 for i in range(1, N_COMP + 1):
X_train['pca_' + str(i)] = pca_results_train[:, i - 1]
X_test['pca_' + str(i)] = pca_results_test[:, i - 1]
X_train['ica_' + str(i)] = ica_results_train[:, i - 1]
X_test['ica_' + str(i)] = ica_results_test[:, i - 1]
X_train['tsvd_' + str(i)] = tsvd_results_train[:, i - 1]
X_test['tsvd_' + str(i)] = tsvd_results_test[:, i - 1]
X_train['grp_' + str(i)] = grp_results_train[:, i - 1]
X_test['grp_' + str(i)] = grp_results_test[:, i - 1]
X_train['srp_' + str(i)] = srp_results_train[:, i - 1]
X_test['srp_' + str(i)] = srp_results_test[:, i - 1]
print('
Train shape: {}
Test shape: {}'.format(X_train.shape, X_test.shape)) 
<define_variables>
var = ['FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual',
'BsmtFinType1', 'BsmtFinType2', 'Functional', 'Fence',
'BsmtExposure', 'GarageFinish', 'LandSlope', 'LotShape',
'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass',
'OverallCond', 'YrSold', 'MoSold']
for i in var:
mdl = LabelEncoder().fit(list(df[i].values))
df[i] = mdl.transform(list(df[i].values))
df[var].head()
print(tsvd.explained_variance_ratio_)
 sum1 = 0
for i in range(len(tsvd.explained_variance_ratio_)) :
sum1 = sum1 + tsvd.explained_variance_ratio_[i]
print(sum1 )<groupby>
df['Total_SF_Main'] = df['TotalBsmtSF'] + df['1stFlrSF'] + df['2ndFlrSF']
sum = 0
for i in range(len(pca.explained_variance_ratio_)) :
sum = sum + pca.explained_variance_ratio_[i]
print(sum )<train_model>
numeric_var_skews = pd.DataFrame(df.dtypes[df.dtypes != 'object'].index,columns=['Numeric_Variables'])
 numeric_var_skews['Skew'] = numeric_var_skews['Numeric_Variables'].apply(lambda x: df[x].skew())
 numeric_var_skews.sort_values('Skew',ascending=False,inplace=True)
 numeric_var_skews.reset_index(inplace=True,drop=True)
 display(numeric_var_skews )
def run_lgb(train_X, train_y, val_X, val_y, test_X):
params = {
'task': 'train',
'boosting_type': 'dart',
'objective': 'regression',
'metric': 'rmse',
"learning_rate": 0.01,
"num_leaves": 200,
"feature_fraction": 0.50,
"bagging_fraction": 0.50,
'bagging_freq': 4,
"max_depth": -1,
"reg_alpha": 0.3,
"reg_lambda": 0.1,
"min_child_weight":10,
'zero_as_missing':True
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 evals_result = {}
model = lgb.train(params, lgtrain, 5000,
valid_sets=[lgval],
early_stopping_rounds=100,
verbose_eval=50,
evals_result=evals_result)
 pred_test_y = np.expm1(model.predict(test_X, num_iteration=model.best_iteration))
return pred_test_y, model, evals_result<train_model>
high_skew = numeric_var_skews[abs(numeric_var_skews['Skew'])> 0.75]
high_skew_vars = high_skew['Numeric_Variables']
for var in high_skew_vars:
df[var] = boxcox1p(df[var], 0.15,
)
def run_xgb(train_X, train_y, val_X, val_y, test_X):
params = {'objective': 'reg:linear', 'eval_metric': 'rmse', 'eta': 0.005, 'max_depth': 10, 'subsample': 0.7, 'colsample_bytree': 0.5, 'alpha':0, 'silent': True, 'random_state':5}
tr_data = xgb.DMatrix(train_X, train_y)
 va_data = xgb.DMatrix(val_X, val_y)
 watchlist = [(tr_data, 'train'),(va_data, 'valid')]
model_xgb = xgb.train(params, tr_data, 2000, watchlist, maximize=False, early_stopping_rounds = 30, verbose_eval=100)
 dtest = xgb.DMatrix(test_X)
 xgb_pred_y = np.expm1(model_xgb.predict(dtest, ntree_limit=model_xgb.best_ntree_limit))
return xgb_pred_y, model_xgb<split>
df_dummy = pd.get_dummies(df, 
)
 df_dummy.shape
dev_X, val_X, dev_y, val_y = train_test_split(X_train, y_train, test_size = 0.2, random_state = 42)
 pred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, X_test)
 print("LightGBM Training Completed...")
 print("Features Importance...")
 gain = model.feature_importance('gain')
 featureimp = pd.DataFrame({'feature':model.feature_name() ,
'split':model.feature_importance('split'),
'gain':100 * gain / gain.sum() } ).sort_values('gain', ascending=False)
 print(featureimp[:15])
 pred_test_xgb, model_xgb = run_xgb(dev_X, dev_y, val_X, val_y, X_test)
 print("XGB Training Completed..." )<save_to_csv>
def overfit_reducer(df):

 overfit = []
for i in df.columns:
counts = df[i].value_counts()
zeros = counts.iloc[0]
if zeros / len(df)* 100 > 99.94:
overfit.append(i)
 overfit = list(overfit)
 return overfit
overfitted_features = overfit_reducer(df_dummy[:train_df.shape[0]])
 df_dummy = df_dummy.drop(overfitted_features, axis=1 )
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub_lgb = pd.DataFrame()
sub_lgb["target"] = pred_test
sub_lgb["ID"] = sub["ID"]
sub_lgb.to_csv("sub_lgb.csv", index=False)
 sub_xgb = pd.DataFrame()
sub_xgb["target"] = pred_test_xgb
sub_xgb["ID"] = sub["ID"]
sub_lgb.to_csv("sub_xgb.csv", index=False)
 sub["target"] =(sub_lgb["target"] + sub_xgb["target"])/2
print(sub.head())
 sub.to_csv('sub_lgb_xgb.csv', index=False )<set_options>
train = df_dummy[:train_df.shape[0]]
Y_train = train_df['SalePrice'].values
ols = sm.OLS(endog = Y_train,
exog = train)
 fit = ols.fit()
test2 = fit.outlier_test() ['bonf(p)']
outliers = list(test2[test2<1e-2].index)
 print('There were {:.0f} outliers at indices:'.format(len(outliers)))
 print(outliers)
 train_df = train_df.drop(train_df.index[outliers])
 df_dummy = df_dummy.drop(df_dummy.index[outliers])
 df_dummy.shape
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
start = dt.datetime.now()<define_variables>
train = df_dummy[:train_df.shape[0]]
test = df_dummy[train_df.shape[0]:]
Y_train = train_df['SalePrice'].values
def rmsle_cv(model):
kf = KFold(5, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse = np.sqrt(-cross_val_score(model, train.values, Y_train,
scoring='neg_mean_squared_error', cv=kf))
return(rmse )
DP_DIR = '.. /input/shuffle-csvs/'
INPUT_DIR = '.. /input/quickdraw-doodle-recognition/'
BASE_SIZE = 256
NCSVS = 100
NCATS = 340
np.random.seed(seed=1987)
 tf.set_random_seed(seed=1987)
 def f2cat(filename: str)-> str:
return filename.split('.')[0]
def list_all_categories() :
files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))
return sorted([f2cat(f)for f in files], key=str.lower )<compute_test_metric>
models = pd.DataFrame([],columns=['model_name','model_object','score_mean','score_std'] )
def apk(actual, predicted, k=3):

 if len(predicted)> k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i, p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i + 1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=3):

 return np.mean([apk(a, p, k)for a, p in zip(actual, predicted)])
 def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
 def top_3_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3 )<define_variables>
knr = KNeighborsRegressor(9, weights='distance')
 score = rmsle_cv(knr)
 print('KNN Regression score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['knr',knr,score.mean() ,score.std() ]
STEPS = 800
EPOCHS = 30
size = 128
batchsize = 340<choose_model_class>
sgd = make_pipeline(RobustScaler() , SGDRegressor(alpha=1000000000000000,l1_ratio=1))
score = rmsle_cv(sgd)
 print('SGD score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['sgd',sgd,score.mean() ,score.std() ]
base_model = MobileNet(input_shape=(size, size,3), alpha=1., weights="imagenet", include_top = False)
 inp = base_model.input
x = base_model.output
x = GlobalAveragePooling2D()(x)
 x = Dense(NCATS, activation='softmax' )(x)
 model = Model(inp, x)
 base_model = Sequential(model.layers[:-2] )<choose_model_class>
rfr = RandomForestRegressor()
score = rmsle_cv(rfr)
 print('Random Forest score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['rfr',rfr,score.mean() ,score.std() ]
inp = Input(shape =(70,3))
x = BatchNormalization()(inp)
 x = Conv1D(256,(5,), activation = "relu" )(x)
 x = Dropout(0.2 )(x)
 x = Conv1D(256,(5,), activation = 'relu' )(x)
 x = Dropout(0.2 )(x)
 x = Conv1D(256,(3,), activation = 'relu' )(x)
 x = Dropout(0.2 )(x)
 x = Bidirectional(LSTM(128, return_sequences = True))(x)
 x = Dropout(0.2 )(x)
 x = Bidirectional(LSTM(128, return_sequences = False))(x)
 x = Dropout(0.2 )(x)
 x = Dense(512, activation = 'relu' )(x)
 x = Dense(NCATS, activation='softmax' )(x)
 stroke_read_model = Model(inp,x)
 stroke_read_model.compile(optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics = ['categorical_accuracy', top_3_accuracy])
 stroke_read_model = Sequential(stroke_read_model.layers[:-1])
 stroke_read_model.summary()<choose_model_class>
lnr = LinearRegression()
score = rmsle_cv(lnr)
 print('Linear Regression score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['lnr',lnr,score.mean() ,score.std() ]
inp = base_model.input
y = base_model.output
y = GlobalAveragePooling2D()(y)
 inp2 = Input(shape =(70, 3))
z = stroke_read_model(inp2)
 x = concatenate([y, z])
 x = Dropout(0.3 )(x)
 x = Dense(NCATS, activation='softmax' )(x)
 model = Model([inp, inp2], x)
 model.compile(optimizer=Adam(lr=0.001), loss='categorical_crossentropy',
metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy] )<concatenate>
ridg = make_pipeline(RobustScaler() , Ridge(alpha =.17,normalize=True, random_state=4))
score = rmsle_cv(ridg)
 print('Ridge score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['ridg',ridg,score.mean() ,score.std() ]
def _stack_it(raw_strokes):

 stroke_vec = literal_eval(raw_strokes)
 in_strokes = [(xi,yi,i)
 for i,(x,y)in enumerate(stroke_vec)
 for xi,yi in zip(x,y)]
c_strokes = np.stack(in_strokes)
 c_strokes[:,2] = [1]+np.diff(c_strokes[:,2] ).tolist()
c_strokes[:,2] += 1
return pad_sequences(c_strokes.swapaxes(0, 1),
maxlen=70,
padding='post' ).swapaxes(0, 1)
<define_variables>
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.02, gamma=0.00046))
score = rmsle_cv(svr)
 print('SVR score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['svr',svr,score.mean() ,score.std() ]
train_datagen = image_generator_xd(size=size, batchsize=batchsize, ks=range(NCSVS - 1))
val_datagen = image_generator_xd(size=size, batchsize=batchsize, ks=range(NCSVS - 1, NCSVS))<train_model>
lasso = make_pipeline(RobustScaler() , Lasso(alpha = 0.00042, max_iter=100000, random_state=1))
score = rmsle_cv(lasso)
 print('Lasso Score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['lasso',lasso,score.mean() ,score.std() ]
callbacks = [
ReduceLROnPlateau(monitor='val_categorical_accuracy', factor=0.5, patience=3,
min_delta=0.005, mode='max', cooldown=3, verbose=1),
ModelCheckpoint("mobilenet_lstm.model",monitor='val_top_3_accuracy',
mode = 'max', save_best_only=True, verbose=1)
 ]
hists = []
hist = model.fit_generator(
 train_datagen, steps_per_epoch=STEPS, epochs=EPOCHS, verbose=1,
validation_data=val_datagen, validation_steps = 100,
callbacks = callbacks 
)
 hists.append(hist )<load_from_csv>
e_net = make_pipeline(RobustScaler() , ElasticNet(alpha = 0.00045, l1_ratio=0.9, random_state=1))
score = rmsle_cv(e_net)
 print('Elastic Net score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['e_net',e_net,score.mean() ,score.std() ]
df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS)) , nrows=34000)
 for i in range(10):
valid_df = df.loc[i*3400:(i+1)*3400,:].copy()
x_valid, x2 = df_to_image_array_xd(valid_df, size)
 y_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)
 print(x_valid.shape, y_valid.shape)
 print('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 1024.**3))
valid_predictions = model.predict([x_valid, x2], batch_size=128, verbose=1)
 map3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions ).values)
 print('Map3: {:.3f}'.format(map3))<predict_on_test>
kr = make_pipeline(RobustScaler() , KernelRidge(alpha=0.04, kernel='polynomial', degree=1, coef0=2.5))
score = rmsle_cv(kr)
 print('Kernel Ridge score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['kr',kr,score.mean() ,score.std() ]
test = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))
for i in range(10):
end = min(( i+1)*11220, 112199)
 subtest= test.iloc[i*11220:end].copy().reset_index(drop=True)
 x_test = df_to_image_array_xd(subtest, size)
 test_predictions = model.predict(x_test, batch_size=128, verbose=1)
 top3 = preds2catids(test_predictions)
 cats = list_all_categories()
id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(cats)}
top3cats = top3.replace(id2cat)
 subtest['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
subtest.head()
if i ==0:
submission = subtest[['key_id', 'word']]
else:
submission = submission.append(subtest[['key_id', 'word']], ignore_index=True)
<save_to_csv>
dtr = make_pipeline(RobustScaler() , DecisionTreeRegressor(random_state=0, max_depth=20))
score = rmsle_cv(dtr)
 print('Decision Tree score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['dtr',dtr,score.mean() ,score.std() ]
submission.to_csv('lstm_mobilenet.csv', index=False)
 submission.head()
submission.shape<train_model>
gbr = GradientBoostingRegressor(n_estimators=3000,
learning_rate=0.05, max_depth=4, max_features='sqrt',
min_samples_leaf=1, min_samples_split=2, loss='huber',
random_state=5,)
 score = rmsle_cv(gbr)
 print('Gradient Boosting score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['gbr',gbr,score.mean() ,score.std() ]
end = dt.datetime.now()
print('Latest run {}.
 Total time {}s'.format(end,(end - start ).seconds))<import_modules>
lgbr = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720, max_bin = 55,
bagging_fraction = 0.8, bagging_freq = 5,
feature_fraction = 0.2319, feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score = rmsle_cv(lgbr)
 print('LightGBM score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['lgbr',lgbr,score.mean() ,score.std() ]
BASE_SIZE = 256
DP_DIR = '.. /input/shuffle-csvs/'
INPUT_DIR = '.. /input/quickdraw-doodle-recognition/'
NCSVS = 100
NCATS = 340
print(os.listdir(".. /input")) 
<define_variables>
xgbr = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3, min_child_weight=1.7817,
n_estimators=2200, reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=True, random_state =7, nthread = -1)
 score = rmsle_cv(xgbr)
 print('XGBoost score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
models.loc[len(models)] = ['xgbr',xgbr,score.mean() ,score.std() ]
startTime = time.time()<define_variables>
models.sort_values(by='score_mean',inplace=True)
 models.reset_index(inplace=True,drop=True)
 models
classes_path = os.listdir(INPUT_DIR + 'train_simplified/')
 classes_path = sorted(classes_path, key=lambda s: s.lower())
 class_dict = {x[:-4].replace(" ", "_"):i for i, x in enumerate(classes_path)}
labels = {x[:-4].replace(" ", "_")for i, x in enumerate(classes_path)}
n_labels = len(labels)
 print("Number of labels: {}".format(n_labels))
fileList = glob(INPUT_DIR + "train_simplified/*.csv")
 n_files = n_labels
n_records = 49707919
size = 80
print("Number of records: {}".format(n_records))<compute_test_metric>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
def top_3_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3 )<define_variables>
def subset(lst, count):
return list(set(combinations(lst, count)))
 model_list = list(models[models['score_mean']<0.11]['model_name'])
 combo = list()
for i in range(1,len(model_list)) :
combo = combo + subset(model_list, i)
 print('There are {:.0f} combinations.First 20 include:'.format(len(combo)))
 combo[:20]
STEPS = 1000
batchsize = 512
epochs = 15<load_from_csv>
simple_avg_final = AveragingModels(models =(lasso, gbr, lgbr, kr))
score = rmsle_cv(simple_avg_final)
 print('Simple Average score = {:.4f}(std = {:.4f})'.format(score.mean() , score.std()))
valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)) , nrows=30000)
 x_valid = df_to_image_array(valid_df, size)
 y_valid = tf.keras.utils.to_categorical(valid_df.y, num_classes=NCATS)
 print(x_valid.shape, y_valid.shape)
 print('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 1024.**3))<define_variables>
stacked = StackingCVRegressor(regressors=(lasso, gbr, lgbr, kr),
meta_regressor=lasso,
use_features_in_secondary=True)
 score = rmsle_cv(stacked)
 print('Stacked score = {:.8f}(std = {:.4f})'.format(score.mean() , score.std()))
train_datagen = image_generator(size=size, batchsize=batchsize, ks=range(NCSVS - 1))<choose_model_class>
class StackingCVRegressor_Scratch(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=42)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_])
 return self.meta_model_.predict(meta_features )
base_model = MobileNet(input_shape=(size, size, 1), include_top=False, weights=None, classes=n_labels)
 x = base_model.output
x = Flatten()(x)
 x = Dense(1024, activation='relu' )(x)
 predictions = Dense(n_labels, activation='softmax' )(x)
 model = Model(inputs=base_model.input, outputs=predictions )<choose_model_class>
stacked_scratch = StackingCVRegressor_Scratch(base_models =(lasso, gbr, lgbr, kr),
meta_model = lasso)
 score = rmsle_cv(stacked_scratch)
 print('Stacked(Scratch)score = {:.8f}(std = {:.4f})'.format(score.mean() , score.std()))
model.compile(optimizer=Adam(lr=1e-4), loss='categorical_crossentropy',
metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy] )<train_model>
stacked_final = StackingCVRegressor(regressors=(svr, ridg, xgbr),
meta_regressor=e_net,
use_features_in_secondary=True)
 score = rmsle_cv(stacked_final)
 print('stacked_final score = {:.8f}(std = {:.4f})'.format(score.mean() , score.std()))
callbacks = [
ReduceLROnPlateau(monitor='val_categorical_accuracy', factor=0.5, patience=5,
min_delta=0.005, mode='max', cooldown=3, verbose=1)
 ]
hist = model.fit_generator(
 train_datagen, steps_per_epoch=STEPS, epochs=epochs, verbose=1,
validation_data=(x_valid, y_valid),
callbacks = callbacks
)<load_from_csv>
model_1 = simple_avg_final
model_2 = stacked_final
mod_1_share =.5
mod_2_share =.5
model_1.fit(train.values, Y_train)
 model_1_test_predictions = np.expm1(model_1.predict(test.values))
model_2.fit(train.values, Y_train)
 model_2_test_predictions = np.expm1(model_2.predict(test.values))
test_predictions = mod_1_share * model_1_test_predictions + mod_2_share * model_2_test_predictions
<define_variables><EOS> 
test_id = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')[['Id']]
test_id['SalePrice'] = np.round(test_predictions,2)
 test_id.to_csv('out51_simple(lasso,gbr,lgbr,kr)_meta(e_net,svr,ridg,xgbr ).csv',index=False )