Kaggle competition: House Prices - Advanced Regression Techniques
def mis_impute(data):
for i in data.columns:
if data[i].dtype == "object":
data[i] = data[i].fillna("other")
 elif(data[i].dtype == "int64" or data[i].dtype == "float64"):
data[i] = data[i].fillna(data[i].mean())
 else:
pass
return data
market_train_df = mis_impute(market_train_df )<categorify>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 print("setup completed" )
def data_prep(market_train):
lbl = {k: v for v, k in enumerate(market_train['assetCode'].unique())}
market_train['assetCodeT'] = market_train['assetCode'].map(lbl)
 market_train = market_train.dropna(axis=0)
 return market_train
market_train_df = data_prep(market_train_df)
 print(market_train_df.shape )<define_variables>
train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1 , inplace = True)
 test.drop("Id", axis = 1 , inplace = True)
 print(f"The train and test data size before dropping Id feature is: {train.shape}, {test.shape}" )
up = market_train_df['returnsOpenNextMktres10'] >= 0
universe = market_train_df['universe'].values 
<define_variables>
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print(f"all_data size is : {all_data.shape}" )
d = market_train_df['time']
fcol = [c for c in market_train_df if c not in ['assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'time_x','provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'universe','sourceTimestamp']]
fcol<prepare_x_and_y>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head(30 )
X = market_train_df[fcol].values
X<sort_values>
all_data["PoolQC"] = all_data["PoolQC"].fillna("None")
 all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None")
 all_data["Alley"] = all_data["Alley"].fillna("None")
 all_data["Fence"] = all_data["Fence"].fillna("None")
 all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None" )
up = up.values
up<find_best_params>
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
r = market_train_df.returnsOpenNextMktres10.values
r<categorify>
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None' )
mins = np.min(X, axis=0)
 mins<define_variables>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0 )
maxs = np.max(X, axis=0)
 maxs<define_variables>
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0 )
rng = maxs - mins
rng<define_variables>
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None' )
X = 1 -(( maxs - X)/ rng)
 X<import_modules>
all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0)
 all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0] )
from xgboost import XGBClassifier
from sklearn import model_selection
from sklearn.metrics import mean_squared_error
import time<split>
all_data = all_data.drop(['Utilities'], axis=1 )
X_train, X_test, up_train, up_test, r_train, r_test,u_train,u_test,d_train,d_test = \
model_selection.train_test_split(X, up, r,universe,d, test_size=0.25, random_state=99 )<prepare_x_and_y>
all_data["Functional"] = all_data["Functional"].fillna("Typ" )
train_data = lgb.Dataset(X, label=up.astype(int))
test_data = lgb.Dataset(X_test, label=up_test.astype(int))<compute_test_metric>
all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0] )
x_1 = [0.19000424246380565, 2452, 212, 328, 202]
x_2 = [0.19016805202090095, 2583, 213, 312, 220]
print(up_train)
 def exp_loss(p,y):
y = y.get_label()
grad = -y*(1.0-1.0/(1.0+np.exp(-y*p)))
 hess = -(np.exp(y*p)*(y*p-1)-1)/(( np.exp(y*p)+1)**2)
 return grad,hess
params_1 = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'binary',
'learning_rate': x_1[0],
'num_leaves': x_1[1],
'min_data_in_leaf': x_1[2],
'num_iteration': 239,
'max_bin': x_1[4],
'verbose': 1
}
params_2 = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'binary',
'learning_rate': x_2[0],
'num_leaves': x_2[1],
'min_data_in_leaf': x_2[2],
'num_iteration': 172,
'max_bin': x_2[4],
'verbose': 1
}
gbm_1 = lgb.train(params_1,
train_data,
num_boost_round=133,
valid_sets=test_data,
early_stopping_rounds=5, 
)
 gbm_2 = lgb.train(params_2,
train_data,
num_boost_round=133,
valid_sets=test_data,
early_stopping_rounds=5,
)<predict_on_test>
all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0] )
confidence_test =(gbm_1.predict(X_test)+ gbm_2.predict(X_test)) /2
confidence_test<compute_test_metric>
all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0] )
confidence_test =(confidence_test-confidence_test.min())/(confidence_test.max() -confidence_test.min())
 confidence_test<prepare_output>
all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0] )
confidence_test = confidence_test*2-1
print(max(confidence_test),min(confidence_test))
confidence_test<create_dataframe>
all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None" )
r_test = r_test.clip(-1,1)
 x_t_i = confidence_test * r_test * u_test
data = {'day' : d_test, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_test = mean / std
print(score_test )<set_options>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head()
del X_train,X_test
gc.collect()<feature_engineering>
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )
days = env.get_prediction_days()
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
total_market_obs_df = []
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
if(n_days%50==0):
print(n_days,end=' ')
 t = time.time()
market_obs_df['time'] = market_obs_df['time'].dt.date
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close']
total_market_obs_df.append(market_obs_df)
 if len(total_market_obs_df)==1:
history_df = total_market_obs_df[0]
else:
history_df = pd.concat(total_market_obs_df[-(np.max(n_lag)+1):])
 print(history_df)
 new_df = generate_lag_features(history_df,n_lag=[3,7,14])
 market_obs_df = pd.merge(market_obs_df,new_df,how='left',on=['time','assetCode'])
 market_obs_df = mis_impute(market_obs_df)
 market_obs_df = data_prep(market_obs_df)
 X_live = market_obs_df[fcol].values
X_live = 1 -(( maxs - X_live)/ rng)
 prep_time += time.time() - t
t = time.time()
lp =(gbm_1.predict(X_live)+ gbm_2.predict(X_live)) /2
prediction_time += time.time() -t
t = time.time()
confidence = lp
confidence =(confidence-confidence.min())/(confidence.max() -confidence.min())
 confidence = confidence * 2 - 1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':confidence})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
env.write_submission_file()
sub = pd.read_csv("submission.csv" )<import_modules>
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))
import numpy as np
import lightgbm as lgb
import pandas as pd
from kaggle.competitions import twosigmanews
import matplotlib.pyplot as plt
import random
from datetime import datetime, date
from xgboost import XGBClassifier
from sklearn import model_selection
from sklearn.metrics import mean_squared_error
import time<import_modules>
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
env = twosigmanews.make_env()
print('Done!' )<prepare_x_and_y>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam)
( market_train_df, news_train_df)= env.get_training_data() 
<feature_engineering>
all_data = pd.get_dummies(all_data)
 print(all_data.shape )
market_train_df['time'] = market_train_df['time'].dt.date
market_train_df = market_train_df.loc[market_train_df['time']>=date(2010, 1, 1)]<merge>
train = all_data[:ntrain]
test = all_data[ntrain:]
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close']
n_lag = [3,7,14]
new_df = generate_lag_features(market_train_df,n_lag=n_lag)
 market_train_df = pd.merge(market_train_df,new_df,how='left',on=['time','assetCode'] )<categorify>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
def mis_impute(data):
for i in data.columns:
if data[i].dtype == "object":
data[i] = data[i].fillna("other")
 elif(data[i].dtype == "int64" or data[i].dtype == "float64"):
data[i] = data[i].fillna(data[i].mean())
 else:
pass
return data
market_train_df = mis_impute(market_train_df )<categorify>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
def data_prep(market_train):
lbl = {k: v for v, k in enumerate(market_train['assetCode'].unique())}
market_train['assetCodeT'] = market_train['assetCode'].map(lbl)
 market_train = market_train.dropna(axis=0)
 return market_train
market_train_df = data_prep(market_train_df)
 print(market_train_df.shape )<define_variables>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
up = market_train_df['returnsOpenNextMktres10'] >= 0
universe = market_train_df['universe'].values
d = market_train_df['time']
fcol = [c for c in market_train_df if c not in ['assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', 'universe','sourceTimestamp']]
X = market_train_df[fcol].values
up = up.values
r = market_train_df.returnsOpenNextMktres10.values
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng)
 assert X.shape[0] == up.shape[0] == r.shape[0]
te = market_train_df['time']>date(2015, 1, 1)
 tt = 0
for tt,i in enumerate(te.values):
if i:
idx = tt
print(i,tt)
 break
print(idx)
 X_train, X_test = X[:idx],X[idx:]
up_train, up_test = up[:idx],up[idx:]
r_train, r_test = r[:idx],r[idx:]
u_train,u_test = universe[:idx],universe[idx:]
d_train,d_test = d[:idx],d[idx:]
train_data = lgb.Dataset(X_train, label=up_train.astype(int))
test_data = lgb.Dataset(X_test, label=up_test.astype(int))<compute_train_metric>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
confidence_test =(gbm_1.predict(X_test)+ gbm_2.predict(X_test)) /2
confidence_test =(confidence_test-confidence_test.min())/(confidence_test.max() -confidence_test.min())
 confidence_test = confidence_test*2-1
print(max(confidence_test),min(confidence_test))
r_test = r_test.clip(-1,1)
 x_t_i = confidence_test * r_test * u_test
data = {'day' : d_test, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_test = mean / std
print(score_test)
<set_options>
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5 )
del X_train,X_test
gc.collect()<feature_engineering>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )
env = twosigmanews.make_env()
df = env._var07.set_index(["assetCode", "time"], verify_integrity=True)
 df["confidenceValue"] = 2 *(df.returnsOpenNextMktres10 > 0 ).astype(float)- 1
m = min(df[df.returnsOpenNextMktres10 > 0].returnsOpenNextMktres10.min() , df[df.returnsOpenNextMktres10 < 0].returnsOpenNextMktres10.max() * -1)
 for(
 market_obs_df,
news_obs_df,
predictions_template_df, 
)in env.get_prediction_days() :
predictions_df =(
 predictions_template_df.set_index("assetCode")
.join(
 market_obs_df.set_index(["assetCode", "time"])
.join(df.returnsOpenNextMktres10, how="left")
.returnsOpenNextMktres10.reset_index() 
.drop("time", axis=1)
.set_index("assetCode"),
how="inner",
lsuffix="_", 
)
.reset_index() 
)
 predictions_df.confidenceValue = 1 / predictions_df.returnsOpenNextMktres10 * m
predictions_df.drop("returnsOpenNextMktres10", axis=1)
 env.predict(predictions_df)
 env.write_submission_file()<set_options>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1 )
env = twosigmanews.make_env()
print('Done!' )<prepare_x_and_y>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )
( market_train_df, news_train_df)= env.get_training_data()<define_variables>
score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
days = env.get_prediction_days()<predict_on_test>
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
eps = 1.8745641033852102e-20
for(market_obs_df, news_obs_df, predictions_template_df)in days:
market_obs_df['returnsOpenPrevMktres10'] = market_obs_df['returnsOpenPrevMktres10'].fillna(eps)
 predictions_template_df['confidenceValue'] = 1/market_obs_df['returnsOpenPrevMktres10']*eps
env.predict(predictions_template_df)
 print('Done!' )<save_to_csv>
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
env.write_submission_file()<import_modules>
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
from sklearn.metrics import accuracy_score
from kaggle.competitions import twosigmanews
from keras import initializers
from keras import regularizers
from keras import constraints
from keras.utils import conv_utils
from keras.utils.data_utils import get_file
from keras.engine.topology import get_source_inputs
from keras.engine import InputSpec
from keras import backend as K
from keras.layers import LeakyReLU
from keras.layers import ZeroPadding2D
from keras.losses import binary_crossentropy
import keras.callbacks as callbacks
from keras.callbacks import Callback
from keras.applications.xception import Xception
from keras.layers import multiply
import keras
from keras import optimizers
from keras.legacy import interfaces
from keras.utils.generic_utils import get_custom_objects
from keras.engine.topology import Input
from keras.engine.training import Model
from keras.layers.convolutional import Conv2D, UpSampling2D, Conv2DTranspose
from keras.layers.core import Activation, SpatialDropout2D
from keras.layers.merge import concatenate
from keras.layers.normalization import BatchNormalization
from keras.layers.pooling import MaxPooling2D
from keras.layers import Input,Dropout,BatchNormalization,Activation,Add
from keras.regularizers import l2
from keras.layers.core import Dense, Lambda
from keras.layers.merge import concatenate, add
from keras.layers import GlobalAveragePooling2D, Reshape, Dense, multiply, Permute
from keras.optimizers import SGD<split>
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
env = twosigmanews.make_env() 
(market_train, _)= env.get_training_data()<define_variables>
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
cat_cols = ['assetCode']
num_cols = ['volume', 'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10', 'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10'] 
<split>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
train_indices, val_indices = train_test_split(market_train.index.values,test_size=0.25, random_state=23)
<categorify>
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
def encode(encoder, x):
len_encoder = len(encoder)
 try:
id = encoder[x]
except KeyError:
id = len_encoder
return id
encoders = [{} for cat in cat_cols]
for i, cat in enumerate(cat_cols):
print('encoding %s...' % cat, end=' ')
 encoders[i] = {l: id for id, l in enumerate(market_train.loc[train_indices, cat].astype(str ).unique())}
market_train[cat] = market_train[cat].astype(str ).apply(lambda x: encode(encoders[i], x))
print('Done')
 embed_sizes = [len(encoder)+ 1 for encoder in encoders] 
<normalization>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
market_train[num_cols] = market_train[num_cols].fillna(0)
 print('scaling numerical columns')
 scaler = StandardScaler()
scaler = StandardScaler()
market_train[num_cols] = scaler.fit_transform(market_train[num_cols])
<choose_model_class>
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))
categorical_inputs = []
for cat in cat_cols:
categorical_inputs.append(Input(shape=[1], name=cat))
categorical_embeddings = []
for i, cat in enumerate(cat_cols):
categorical_embeddings.append(Embedding(embed_sizes[i], 10 )(categorical_inputs[i]))
categorical_logits = Flatten()(categorical_embeddings[0])
 categorical_logits = Dense(32,activation='relu' )(categorical_logits)
 numerical_inputs = Input(shape=(11,), name='num')
 numerical_logits = numerical_inputs
numerical_logits = BatchNormalization()(numerical_logits)
 numerical_logits = Dense(128,activation='relu' )(numerical_logits)
 numerical_logits = Dense(64,activation='relu' )(numerical_logits)
 logits = Concatenate()([numerical_logits,categorical_logits])
 logits = Dense(64,activation='relu' )(logits)
 out = Dense(1, activation='sigmoid' )(logits)
 model = Model(inputs = categorical_inputs + [numerical_inputs], outputs=out)
 model.compile(optimizer='adam',loss=binary_crossentropy )<prepare_x_and_y>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def get_input(market_train, indices):
X_num = market_train.loc[indices, num_cols].values
X = {'num':X_num}
for cat in cat_cols:
X[cat] = market_train.loc[indices, cat_cols].values
y =(market_train.loc[indices,'returnsOpenNextMktres10'] >= 0 ).values
r = market_train.loc[indices,'returnsOpenNextMktres10'].values
u = market_train.loc[indices, 'universe']
d = market_train.loc[indices, 'time'].dt.date
return X,y,r,u,d
X_train,y_train,r_train,u_train,d_train = get_input(market_train, train_indices)
 X_valid,y_valid,r_valid,u_valid,d_valid = get_input(market_train, val_indices )<train_model>
stacked_averaged_models.fit(train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))
class SWA(keras.callbacks.Callback):
def __init__(self, filepath, swa_epoch):
super(SWA, self ).__init__()
self.filepath = filepath
self.swa_epoch = swa_epoch
def on_train_begin(self, logs=None):
self.nb_epoch = self.params['epochs']
print('Stochastic weight averaging selected for last {} epochs.' 
.format(self.nb_epoch - self.swa_epoch))
def on_epoch_end(self, epoch, logs=None):
if epoch == self.swa_epoch:
self.swa_weights = self.model.get_weights()
elif epoch > self.swa_epoch:
for i in range(len(self.swa_weights)) :
self.swa_weights[i] =(self.swa_weights[i] * 
(epoch - self.swa_epoch)+ self.model.get_weights() [i])/(( epoch - self.swa_epoch)+ 1)
 else:
pass
def on_train_end(self, logs=None):
self.model.set_weights(self.swa_weights)
 print('Final model parameters set to stochastic weight average.')
 self.model.save_weights(self.filepath)
 print('Final stochastic averaged weights saved to file.' )<choose_model_class>
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))
class SnapshotCallbackBuilder:
def __init__(self, nb_epochs, nb_snapshots, init_lr=0.1):
self.T = nb_epochs
self.M = nb_snapshots
self.alpha_zero = init_lr
def get_callbacks(self, model_prefix='Model'):
callback_list = [
callbacks.ModelCheckpoint("model.hdf5",monitor='val_my_iou_metric',
mode = 'max', save_best_only=True, verbose=1),
swa,
callbacks.LearningRateScheduler(schedule=self._cosine_anneal_schedule)
 ]
return callback_list
def _cosine_anneal_schedule(self, t):
cos_inner = np.pi *(t %(self.T // self.M))
cos_inner /= self.T // self.M
cos_out = np.cos(cos_inner)+ 1
return float(self.alpha_zero / 2 * cos_out )<train_model>
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))
check_point = ModelCheckpoint('model.hdf5',verbose=True, save_best_only=True)
 early_stop = EarlyStopping(patience=5,verbose=True)
 model.fit(X_train,y_train.astype(int),
validation_data=(X_valid,y_valid.astype(int)) ,
epochs=5,
verbose=True,
callbacks=[early_stop,check_point] )<compute_test_metric>

 print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.70 +
xgb_train_pred*0.15 + lgb_train_pred*0.15))
r_valid = r_valid.clip(-1,1)
 x_t_i = confidence_valid * r_valid * u_valid
data = {'day' : d_valid, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print(score_valid )<define_variables>
ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15
<categorify><EOS> 
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('submission.csv',index=False )