Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_from_csv>
warnings.filterwarnings("ignore")
train_data = pd.read_csv(".. /input/train.csv" )<prepare_x_and_y>
def import_data(data):
df = pd.read_csv('.. /input/{}.csv'.format(data))
df = df.set_index('Id')
 return df
train = import_data('train')
 test = import_data('test' )
X = train_data[label_names[1:]]
y = train_data[label_names[0]]<split>
print('Train dataset:
Rows: {}
Columns: {}'.format(train.shape[0], train.shape[1]))
train.head()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2 )<categorify>
train = train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<450000)].index )
img_rows, img_cols = 28, 28
num_classes =10
X_train = X_train.values.reshape([-1,28,28,1])/255
X_test = X_test.values.reshape([-1,28,28,1])/255
y_train = y_train.values
y_test = y_test.values
y_train = keras.utils.to_categorical(y_train, num_classes)
 y_test = keras.utils.to_categorical(y_test, num_classes )<choose_model_class>
commonNa = [
'MSZoning','Electrical','KitchenQual',
'Exterior1st','Exterior2nd','SaleType',
'LotFrontage','Functional'
]
toZero = [
'MasVnrArea','GarageYrBlt','BsmtHalfBath',
'BsmtFullBath','GarageArea','GarageCars',
'BsmtFinSF1','BsmtFinSF2','BsmtUnfSF',
'TotalBsmtSF'
]
toNoData = [
'PoolQC','MiscFeature','Alley',
'Fence','FireplaceQu','GarageType',
'GarageFinish','GarageQual',
'GarageCond','BsmtQual','BsmtCond',
'BsmtExposure','BsmtFinType1','BsmtFinType2',
'MasVnrType'
]
def fillNa_fe(df):
df['Functional'] = df['Functional'].fillna('Typ')
 for i in commonNa:
df[i] = df[i].fillna(df[i].mode() [0])
 for i in toNoData:
df[i] = df[i].fillna('None')
 for i in toZero:
df[i] = df[i].fillna(0)
 df.drop(
 ['Utilities'],
axis=1,
inplace=True 
)
 df['totalSF'] = df['TotalBsmtSF'] \
+ df['1stFlrSF'] \
+ df['2ndFlrSF'] \
+ df['GrLivArea']
return df
train, test = fillNa_fe(train), fillNa_fe(test )
model = Sequential() 
<choose_model_class>
for df in [train,test]:
print(df[[col for col in df.columns if df[col].isna().sum() > 0]].isna().sum().sort_values(ascending = False ).any() )
model.add(tf.keras.layers.Conv2D(32,(3, 3), padding='same',
input_shape=(img_rows,img_cols,1)))
 model.add(tf.keras.layers.Activation('relu'))
model.add(tf.keras.layers.Conv2D(32,(3, 3)))
 model.add(tf.keras.layers.Activation('relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
 model.add(tf.keras.layers.Dropout(0.25))
model.add(tf.keras.layers.Conv2D(64,(3, 3), padding='same'))
model.add(tf.keras.layers.Activation('relu'))
model.add(tf.keras.layers.Conv2D(64,(3, 3)))
 model.add(tf.keras.layers.Activation('relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
 model.add(tf.keras.layers.Dropout(0.25))
model.add(tf.keras.layers.Flatten())
 model.add(tf.keras.layers.Dense(512))
model.add(tf.keras.layers.Activation('relu'))
model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.Dense(num_classes))
model.add(tf.keras.layers.Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=tf.train.AdamOptimizer() ,
metrics=['accuracy'])
<train_model>
batch_size =128
epochs = 100
history = model.fit(X_train, y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(X_test, y_test))<predict_on_test>
encoder = LabelEncoder()
sc = StandardScaler()
def encode(df):
cat_df = [col for col in df.columns if df[col].dtype not in ['int','float']]
for col in cat_df:
df[col] = encoder.fit_transform(df[col])
 df_ = sc.fit_transform(df)
 df = pd.DataFrame(data=df_, columns = df.columns)
 return df
test,train=encode(test),encode(train )
test_data = pd.read_csv(".. /input/test.csv")
 y_pred = model.predict(test_data.values.reshape([-1,28,28,1])/255,batch_size=batch_size)
 y_pred = np.argmax(y_pred,axis=1)
<save_to_csv>
X = train.copy()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.35, random_state=16 )
save = pd.DataFrame()
save["ImageId"] = list(range(1,len(y_pred)+1))
save["Label"] = y_pred
save.to_csv("submit.csv", index=False)
<load_from_csv>
params = {
'colsample_bytree': [0.4],
'gamma': [0.0],
'learning_rate': [0.01],
'max_depth': [4],
'min_child_weight': [2],
'n_estimators': [4000],
'seed': [36],
'subsample': [0.5],
'verbosity':[0]
}
gbm = XGBRegressor()
reg_cv = GridSearchCV(gbm, params, verbose = 0)
 reg_cv.fit(X_train,y_train)
 reg_cv.best_params_
XGBoost = XGBRegressor(**reg_cv.best_params_)
 XGBoost.fit(X_train,y_train,verbose = False)
 xgboost_predictions = XGBoost.predict(X_test)
 print('XGBoost \tRMSE: \t{}$'.format(int(np.sqrt(metrics.mean_squared_error(y_test, xgboost_predictions)))) )
train_df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 print(train_df.shape )<prepare_x_and_y>
reg_cv.best_params_
X_train = train_df.iloc[:,1:].values
y_train = train_df.iloc[:,0].values<categorify>
parameters = {
'kernel':('linear','poly'),
'C':[120],
'gamma': [.0003],
'epsilon':[0.1]
}
svr = svm.SVR()
clf = GridSearchCV(svr, parameters)
 clf.fit(X_train,y_train)
 clf.best_params_
supv = XGBRegressor(**clf.best_params_)
 supv.fit(X_train,y_train,verbose = False)
 supv_predictions = supv.predict(X_test)
 print(int(np.sqrt(metrics.mean_squared_error(y_test,supv_predictions))))
N = X_train.shape[0]
X_train = X_train.reshape(N,28,28,1)
 X_train = X_train.astype('float32')
 X_train /= 255.
 y_train = tf.one_hot(y_train, 10)
 X_train.shape, y_train.shape<choose_model_class>
clf.best_params_
def make_model(inputs):
x = tf.keras.layers.Conv2D(filters=32,kernel_size=(3,3))(inputs)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.ReLU()(x)
 x = tf.keras.layers.MaxPool2D()(x)
 x = tf.keras.layers.Conv2D(filters=64,kernel_size=(3,3))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.ReLU()(x)
 x = tf.keras.layers.MaxPool2D()(x)
 x = tf.keras.layers.Conv2D(filters=1024,kernel_size=(5,5))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.ReLU()(x)
 x = tf.keras.layers.Conv2D(filters=256,kernel_size=(1,1))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.ReLU()(x)
 x = tf.keras.layers.Conv2D(filters=10,kernel_size=(1,1))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Softmax()(x)
 x = tf.keras.layers.Reshape(( 10,))(x)
 return x<choose_model_class>
lasso.fit(X_train,y_train)
 lasso_predictions = lasso.predict(X_test)
 print('Lasso \t \tRMSE: \t{}$'.format(int(np.sqrt(metrics.mean_squared_error(y_test, lasso_predictions)))) )
tf.keras.backend.clear_session()
inputs = tf.keras.Input(shape=(28,28,1))
outputs = make_model(inputs)
 model = tf.keras.Model(
 inputs=inputs,
outputs=outputs,
name="simple" 
)
 model.compile(optimizer='adam', loss='categorical_crossentropy' )<train_model>
%%time
parameters = {
'objective':['regression'],
'num_leaves':[4],
'learning_rate':[0.01],
'n_estimators':[9000],
'max_bin':[500],
'bagging_fraction':[.5,.3],
'bagging_freq':[5],
'bagging_seed':[7],
'feature_fraction':[.2],
'feature_fraction_seed':[7],
'verbose':[-1]
}
light = LGBMRegressor()
clf = GridSearchCV(light, parameters)
 clf.fit(X_train, y_train)
 clf.best_params_
lightgbm = LGBMRegressor(**clf.best_params_)
 lightgbm.fit(X_train,y_train)
 lightgbm_predictions = lightgbm.predict(X_test )
history = model.fit(X_train,y_train, validation_split=0.2,epochs=10 )<load_from_csv>
clf.best_params_
X_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' ).values
N_test = X_test.shape[0]
X_test = X_test.reshape(N_test,28,28,1)
 X_test = X_test.astype('float32')
 X_test /= 255.
 X_test.shape<predict_on_test>
%%time
alphas_ela = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
l1 = [0.70, 0.80, 0.85, 0.90, 0.95, 0.99]
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=5000, cv=kfolds, alphas=alphas_ela, l1_ratio=l1)
 elasticnet.fit(X_train,y_train)
 elasticnet_predictions = np.expm1(elasticnet.predict(X_test))
print('Elasticnet \tRMSE: \t{}$'.format(int(np.sqrt(metrics.mean_squared_error(y_test,elasticnet_predictions)))) )
pred = model.predict(X_test)
 pred.shape<save_to_csv>
parameters = {
'n_estimators':[7000],
'learning_rate':[0.01],
'max_depth':[4],
'max_features':['sqrt'],
'min_samples_leaf':[10],
'min_samples_split':[10],
'loss':['huber'],
'random_state' :[16]
}
gbr = GradientBoostingRegressor()
clf = GridSearchCV(gbr, parameters)
 clf.fit(X_train, y_train)
 clf.best_params_
gbr = GradientBoostingRegressor(**clf.best_params_)
 gbr.fit(X_train,y_train)
 gbr_predictions = np.expm1(gbr.predict(X_test))
submissions=pd.DataFrame({"ImageId": list(range(1,len(pred)+1)) ,
"Label": np.argmax(pred, axis=1)})
 submissions.to_csv("my_submissions.csv", index=False, header=True )<load_from_csv>
clf.best_params_
df = pd.read_csv('/kaggle/input/spooky-author-identification/train.zip' )<drop_column>
alphas_alt = [
14.5, 14.6, 14.7,
14.8, 14.9, 15.0,
15.1, 15.2, 15.3,
15.4, 15.5, 15.6
]
ridge = RidgeCV(
 alphas=alphas_alt,
cv=kfolds 
)
 ridge.fit(X_train,y_train)
 ridge_predictions = np.expm1(ridge.predict(X_test))
df.drop(columns='id', inplace=True )<feature_engineering>
%%time
stack_gen = StackingCVRegressor(
 regressors=(
 lightgbm,
supv,
XGBoost,
lightgbm,
ridge,
gbr,
elasticnet,
lasso 
),
meta_regressor=XGBoost,
use_features_in_secondary=True 
)
 stack_gen_model = stack_gen.fit(np.array(X_test), np.array(y_test))
stack_predictions = stack_gen_model.predict(np.array(X_test))
df['text'] = df['text'].str.lower()<count_values>
def blend(data):
return(
(0.10 * supv.predict(data)) + 
(0.20 * XGBoost.predict(data)) + 
(0.15 * lightgbm.predict(data)) + 
(0.15 * gbr.predict(data)) + 
(0.10 * ridge.predict(data)) + 
(0.10 * elasticnet.predict(data)) + 
(0.10 * lasso.predict(data)) + 
(0.10 * stack_gen_model.predict(np.array(data)))
 )
df.author.value_counts()<define_variables>
def calc(model): return int(np.sqrt(metrics.mean_squared_error(y_test, model)))
 print('XGBoost \tRMSE: \t{}$'.format(calc(xgboost_predictions)))
 print('SVR \t\tRMSE: \t{}$'.format(calc(supv_predictions)))
 print('lightGBM \tRMSE: \t{}$'.format(calc(lightgbm_predictions)))
 print('GBR \t\tRMSE: \t{}$'.format(calc(gbr_predictions)))
 print('Ridge \t \tRMSE: \t{}$'.format(calc(ridge_predictions)))
 print('Elasticnet \tRMSE: \t{}$'.format(calc(elasticnet_predictions)))
 print('Lasso \t \tRMSE: \t{}$'.format(calc(lasso_predictions)))
 print('Stack_gen \tRMSE: \t{}$'.format(calc(stack_predictions)))
 print('Blended \tRMSE: \t{}$'.format(int(np.sqrt(metrics.mean_squared_error(y_test, blend(X_test))))))
authors_dict = {'EAP' : 0,
'MWS' : 1,
'HPL' : 2}<categorify>
res = pd.DataFrame(
 blended_predictions,
index=import_data('test' ).reset_index() ['Id'] 
)
 res.rename(columns={0:'SalePrice','index':'Id'},inplace=True)
 res.reset_index(inplace=True)
 res.head()
df['author'] = df['author'].map(authors_dict )<import_modules>
res.to_csv('subm.csv',index = 0 )
<split><EOS> 
for kol in preds.columns:
res = preds[kol].to_frame()
res.set_index(import_data('test' ).reset_index() ['Id'])
 res.rename(columns={kol:'SalePrice','index':'Id'},inplace=True)
 res.to_csv('{}.csv'.format(kol))