Kaggle competition: House Prices - Advanced Regression Techniques
data = data.reshape(data.shape[0],28,28,1)
 data_test = data_test.reshape(data_test.shape[0],28,28,1)
 data = data/255
data_test = data_test/255<categorify>
warnings.filterwarnings('ignore')
 % matplotlib inline
print("Before conding")
 print(label[:10])
 labels = np_utils.to_categorical(label,10)
 print("Encoded Data")
 print(labels[:10] )<choose_model_class>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
def model_generator(dropout=[0.25],denses=[512,10],activation="relu"):
model = Sequential()
model.add(Conv2D(filters=32,kernel_size=3,padding='same', activation='relu', input_shape=(28, 28,1)))
 model.add(Conv2D(filters=32, kernel_size=3, border_mode='same', activation='relu'))
model.add(MaxPool2D(pool_size=3))
model.add(Dropout(0.20))
model.add(Conv2D(filters=64,kernel_size=3,padding='same', activation='relu'))
model.add(Conv2D(filters=64, kernel_size=3, border_mode='same', activation='relu'))
model.add(MaxPool2D(pool_size=2))
model.add(Dropout(0.25))
model.add(Conv2D(filters=128, kernel_size=3, border_mode='same', activation='relu'))
model.add(MaxPool2D(pool_size=2))
model.add(Dropout(0.20))
model.add(Flatten())
 model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(10))
model.add(Activation('softmax'))
model.summary()
return model<choose_model_class>
target = np.log(train.SalePrice)
 print("Skewness : %f" % target.skew())
 print("Kurtosis : %f" % target.kurt() )
def model_generator2(dropout=[0.25],denses=[512,10],activation="relu"):
model = Sequential()
model.add(Conv2D(filters=16,kernel_size=2,padding='same', activation='relu', input_shape=(28, 28,1)))
 model.add(MaxPool2D(pool_size=2))
model.add(Dropout(0.20))
model.add(Conv2D(filters=32, kernel_size=2, border_mode='same', activation='relu'))
model.add(MaxPool2D(pool_size=2))
model.add(Dropout(0.20))
model.add(Conv2D(filters=64,kernel_size=2,padding='same', activation='relu'))
model.add(MaxPool2D(pool_size=2))
model.add(Dropout(0.15))
model.add(Flatten())
 model.add(Dense(512, name='aux_output'))
model.add(Activation('relu'))
model.add(Dropout(0.4))
model.add(Dense(10, name='aux_output2'))
model.add(Activation('softmax'))
model.summary()
return model<train_model>
total = train.isnull().sum().sort_values(ascending = False)
 percent =(train.isnull().sum() /train.isnull().count() ).sort_values(ascending = False)
 missing_data = pd.concat([total,percent], axis = 1, keys = ['Total', 'Percent'])
 missing_data.head(20 )
def model_fit(model,batch_size=64,epochs=10):
optimizer = Adam(lr=0.0001)
 model.compile(loss="categorical_crossentropy",optimizer=optimizer,metrics=['accuracy'])
 checkpointer = ModelCheckpoint(filepath='mnist.model.best', verbose=1, monitor='val_loss', save_best_only=True)
 training = model.fit(data, labels,batch_size=batch_size, epochs=epochs,validation_split=0.25, callbacks=[checkpointer],verbose=1, shuffle=True)
 return training<train_model>
cols =('MSZoning', 'Street', 'Alley', 'LotShape', 'LandContour', 'Utilities',
'LotConfig', 'LandSlope', 'Neighborhood', 'Condition1', 'Condition2',
'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st',
'Exterior2nd', 'MasVnrType', 'ExterQual', 'ExterCond', 'Foundation',
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2',
'Heating', 'HeatingQC', 'CentralAir', 'Electrical', 'KitchenQual',
'Functional', 'FireplaceQu', 'GarageType', 'GarageFinish', 'GarageQual',
'GarageCond', 'PavedDrive', 'PoolQC', 'Fence', 'MiscFeature',
'SaleType', 'SaleCondition')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(train[c].values))
train[c] = lbl.transform(list(train[c].values))
lbl.fit(list(test[c].values))
test[c] = lbl.transform(list(test[c].values))
model1 = model_generator(dropout=[0.25],denses=[128,10],activation="relu")
 training = model_fit(model1,batch_size=128,epochs=100 )<predict_on_test>
train=train.drop(columns=['Street', 'Utilities', 'Condition2', 'RoofMatl', 'Heating', 'PoolQC'])
 test=test.drop(columns=['Street', 'Utilities', 'Condition2', 'RoofMatl', 'Heating', 'PoolQC'] )
def scoring(model):
model.load_weights('mnist.model.best')
 score = model.evaluate(data[:2000], labels[:2000], verbose=0)
 accuracy = 100*score[1]
print('Test accuracy: %.4f%%' % accuracy)
 label_test = model.predict_classes(data_test)
 print("Sample of the prdiction",label_test[:10])
 return label_test<define_variables>
train = train.dropna(thresh=0.70*len(train), axis=1)
 test = test.dropna(thresh=0.70*len(test), axis=1 )
label_test = scoring(model1 )<save_to_csv>
train = train.fillna(train.mean())
 test = test.fillna(test.mean() )
np.savetxt("submission.csv", np.dstack(( np.arange(1, label_test.size+1),label_test)) [0],"%d,%d",header="ImageId,Label",comments="" )<set_options>
%matplotlib inline
np.random.seed()
sns.set(style='white', context='notebook', palette='deep')
 print("Done" )<load_from_csv>
dataset = pd.read_csv(".. /input/train.csv")
 competition_dataset = pd.read_csv(".. /input/test.csv")
 dataset.describe 
<drop_column>
del dataset
print("Done" )<categorify>
y = np.log(train.SalePrice)
 X = train.drop(['SalePrice', 'Id'], axis=1)
label = to_categorical(label, num_classes = 10)
 feature = feature / 255.0
competition_dataset = competition_dataset / 255.0
print("Done" )<split>
def identify_zero_importance_features(X, y, iterations = 2):

 feature_importances = np.zeros(X.shape[1])
 model = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.05,
n_estimators=1250,
max_bin=75,
bagging_fraction=0.8,
bagging_freq=9,
feature_fraction=0.45,
feature_fraction_seed=9,
bagging_seed=12,
min_data_in_leaf=3,
min_sum_hessian_in_leaf=2)
 for i in range(iterations):
train_features, valid_features, train_y, valid_y = train_test_split(X, y,
test_size = 0.25,
random_state = i)
 model.fit(train_features, train_y, early_stopping_rounds=100,
eval_set = [(valid_features, valid_y)])
 feature_importances += model.feature_importances_ / iterations
feature_importances = pd.DataFrame({'feature': list(X.columns),
'importance': feature_importances} ).sort_values('importance',
ascending = False)
 zero_features = list(feature_importances[feature_importances['importance'] == 0.0]['feature'])
 print('
There are %d features with 0.0 importance' % len(zero_features))
return zero_features, feature_importances
zero_features, feature_importances = identify_zero_importance_features(X, y, iterations = 2)
 print('zero_features:',zero_features)
 print('feature_importances : ', feature_importances )
feature_train, feature_val, label_train, label_val = train_test_split(feature, label, test_size = 0.1, stratify=label )<choose_model_class>
to_drop = feature_importances[feature_importances['importance'] <= pp]['feature']
X = X.drop(columns = to_drop )
model_1 = Sequential()
model_1.add(Dense(200, activation = "relu", input_shape =(784,)))
 model_1.add(Dense(100, activation = "relu"))
model_1.add(Dense(60, activation = "relu"))
model_1.add(Dense(30, activation = "relu"))
model_1.add(Dense(10, activation = "softmax"))
optimizer = optimizers.SGD(lr=0.03, clipnorm=5.)
model_1.compile(optimizer= optimizer , loss = "categorical_crossentropy", metrics=["accuracy"])
 print(model_1.summary() )<train_model>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1 )
history = model_1.fit(feature_train, label_train, batch_size = 100, epochs = 8,
validation_data =(feature_val, label_val), verbose = 1)
<split>
regr = linear_model.LinearRegression()
regr.fit(X_train, y_train)
 y_pred = regr.predict(X_test)
 print('The accuracy of the Linear Regression is',r2_score(y_test,y_pred))
print('RMSE is:
', mean_squared_error(y_test, y_pred))
feature_train, feature_val, label_train, label_val = train_test_split(feature, label, test_size = 0.1, stratify=label )<choose_model_class>
xg_reg = xgb.XGBRegressor(learning_rate =0.01, n_estimators=5580,
max_depth=3,min_child_weight=0 ,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective= 'reg:linear',nthread=4,
scale_pos_weight=1,seed=27,
reg_alpha=0.00006)
 xg_reg.fit(X_train,y_train)
 preds = xg_reg.predict(X_test)
 print('The accuracy of the xgboost is',r2_score(y_test,preds))
print('RMSE is:
', mean_squared_error(y_test,preds))
model_2 = Sequential()
model_2.add(Conv2D(filters = 4, kernel_size =(5,5), strides = 1, padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model_2.add(Conv2D(filters = 8, kernel_size =(4,4), strides = 2, padding = 'Same',
activation ='relu'))
model_2.add(Conv2D(filters = 12, kernel_size =(4,4), strides = 2, padding = 'Same',
activation ='relu'))
model_2.add(Flatten())
 model_2.add(Dense(200, activation = "relu"))
model_2.add(Dense(10, activation = "softmax"))
optimizer = optimizers.SGD(lr=0.03, clipnorm=5.)
model_2.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"])
 print(model_2.summary())
<train_model>
gbr_model = GradientBoostingRegressor(n_estimators=4780, learning_rate=0.01,
max_depth=10, max_features='sqrt',
min_samples_leaf=1, min_samples_split=250,
loss='huber', random_state =6 ).fit(X_train,y_train)
 gbr_preds = gbr_model.predict(X_test)
 print('The accuracy of the Gradient boost is',r2_score(y_test,gbr_preds))
print('RMSE is:
', mean_squared_error(y_test,gbr_preds))
history = model_2.fit(feature_train, label_train, batch_size = 100, epochs = 16,
validation_data =(feature_val, label_val), verbose = 1)
<init_hyperparams>
lgbm_model = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.05,
n_estimators=1250,
max_bin=75,
bagging_fraction=0.8,
bagging_freq=9,
feature_fraction=0.45,
feature_fraction_seed=9,
bagging_seed=12,
min_data_in_leaf=3,
min_sum_hessian_in_leaf=2 ).fit(X_train, y_train)
 lgbm_preds = lgbm_model.predict(X_test)
 print('The accuracy of the lgbm Regressor is',r2_score(y_test,lgbm_preds))
print('RMSE is:
', mean_squared_error(y_test,lgbm_preds))
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(feature_train)
 print("Done" )<choose_model_class>
r_alphas = [0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30]
ridge_model = make_pipeline(RobustScaler() , RidgeCV(alphas = r_alphas, cv =3)).fit(X_train, y_train)
 ridge_preds = ridge_model.predict(X_test)
 print('The accuracy of the ridge Regressor is',r2_score(y_test,ridge_preds))
print('RMSE is:
', mean_squared_error(y_test,ridge_preds))
model_3 = Sequential()
model_3.add(Conv2D(filters = 6, kernel_size =(6,6), strides = 1, padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model_3.add(Conv2D(filters = 12, kernel_size =(5,5), strides = 2, padding = 'Same',
activation ='relu'))
model_3.add(Conv2D(filters = 24, kernel_size =(4,4), strides = 2, padding = 'Same',
activation ='relu'))
model_3.add(Flatten())
 model_3.add(Dense(200, activation = "relu"))
model_3.add(Dropout(0.40))
model_3.add(Dense(10, activation = "softmax"))
model_3.compile(optimizer = 'adam' , loss = "categorical_crossentropy", metrics=["accuracy"])
 print(model_3.summary() )<train_model>
alpha_lasso = np.logspace(-3, -1, 30)
 lasso_model = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e6,
alphas = alpha_lasso,
random_state = 1)).fit(X_train, y_train)
 lasso_preds = lasso_model.predict(X_test)
 print('The accuracy of the lasso Regressor is',r2_score(y_test,lasso_preds))
print('RMSE is:
', mean_squared_error(y_test,lasso_preds))
history = model_3.fit(datagen.flow(feature_train,label_train, batch_size=100),
epochs = 8, validation_data =(feature_val, label_val),
verbose = 2 )<choose_model_class>
e_alphas = np.logspace(-3 -2, 30)
 e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
elastic_model= make_pipeline(RobustScaler() ,
ElasticNetCV(max_iter=1e6, alphas=e_alphas,
l1_ratio=e_l1ratio)).fit(X_train, y_train)
 elastic_preds = elastic_model.predict(X_test)
 print('The accuracy of the Elastic Net CV is',r2_score(y_test,elastic_preds))
print('RMSE is:
', mean_squared_error(y_test,elastic_preds))
model_4 = Sequential()
model_4.add(Conv2D(filters = 32, kernel_size =(5,5), strides = 1, padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model_4.add(BatchNormalization())
 model_4.add(Conv2D(filters = 32, kernel_size =(5,5), strides = 1, padding = 'Same',
activation ='relu'))
model_4.add(BatchNormalization())
 model_4.add(Dropout(0.4))
model_4.add(Conv2D(filters = 64, kernel_size =(3,3), strides = 2, padding = 'Same',
activation ='relu'))
model_4.add(BatchNormalization())
 model_4.add(Conv2D(filters = 64, kernel_size =(3,3), strides = 2, padding = 'Same',
activation ='relu'))
model_4.add(BatchNormalization())
 model_4.add(Dropout(0.4))
model_4.add(Flatten())
 model_4.add(Dense(256, activation = "relu"))
model_4.add(Dropout(0.4))
model_4.add(Dense(10, activation = "softmax"))
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
 model_4.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"])
 learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 print(model_4.summary() )<train_model>
ridge = make_pipeline(RobustScaler() ,
RidgeCV(alphas = r_alphas))
lasso = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e6,
alphas = alpha_lasso,
random_state = 1)).fit(X_train,y_train)
 elasticnet = make_pipeline(RobustScaler() ,
ElasticNetCV(max_iter=1e6, alphas=e_alphas,
l1_ratio=e_l1ratio)).fit(X_train,y_train)
 lgbm_model = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.05,
n_estimators=1250,
max_bin=75,
bagging_fraction=0.8,
bagging_freq=9,
feature_fraction=0.45,
feature_fraction_seed=9,
bagging_seed=12,
min_data_in_leaf=3,
min_sum_hessian_in_leaf=2 ).fit(X_train,y_train)
 gbr_model = GradientBoostingRegressor(n_estimators=4780, learning_rate=0.01,
max_depth=10, max_features='sqrt',
min_samples_leaf=1, min_samples_split=250,
loss='huber', random_state =6 ).fit(X_train,y_train)
 stack_gen = StackingCVRegressor(regressors=(ridge,
lasso, elasticnet, gbr_model,
lgbm_model),
meta_regressor=gbr_model,
use_features_in_secondary=True)
 stackX = np.array(X_train)
 stacky = np.array(y_train)
 stack_gen_model = stack_gen.fit(stackX, stacky )
history = model_4.fit(datagen.flow(feature_train,label_train, batch_size=100),
epochs = 35, validation_data =(feature_val, label_val),
verbose = 2, callbacks=[learning_rate_reduction] )<predict_on_test>
em_preds = elastic_model.predict(X_test)
 lasso_preds = lasso_model.predict(X_test)
 ridge_preds = ridge_model.predict(X_test)
 stack_gen_preds = stack_gen_model.predict(X_test)
 lgbm_preds = lgbm_model.predict(X_test)
 gbr_preds = gbr_model.predict(X_test )
results = model_4.predict(competition_dataset)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )<save_to_csv>
print('RMSE is:
', mean_squared_error(y_test,stack_gen_preds))
print('The accuracy of the stack is',r2_score(y_test,stack_gen_preds))
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("submission_MNIST.csv",index=False)
<prepare_x_and_y>
stack_preds_1 =(( 0.1*em_preds)+(0.1*lasso_preds)+(0.1*ridge_preds)+(0.1 * gbr_preds)
 +(0.1*lgbm_preds)+(0.5*stack_gen_preds))
print('The accuracy of the stack Regressor is',r2_score(y_test,stack_preds_1))
print('RMSE is:
', mean_squared_error(y_test,stack_preds_1))
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_data = pd.read_csv('/kaggle/input/titanic/test.csv')
 train = train_data.copy()
test = test_data.copy()
train.shape
y_train = train['Survived']
Id = pd.DataFrame(test['PassengerId'])
 train.drop(['PassengerId'], axis = 1, inplace=True)
 test.drop(['PassengerId'], axis = 1, inplace=True)
 train.drop(['Survived'], axis = 1, inplace=True)
 train.drop(['Ticket'], axis = 1, inplace=True)
 test.drop(['Ticket'], axis = 1, inplace=True )<sort_values>
feats = test.drop(['Id'], axis=1)
 feats = feats[X_train.columns]
train.isnull().sum().sort_values(ascending=False)[0:20] 
<categorify>
lasso_preds = lasso_model.predict(feats)
 em_preds = elastic_model.predict(feats)
 ridge_preds = ridge_model.predict(feats)
 stack_gen_preds = stack_gen_model.predict(feats)
 lgbm_preds = lgbm_model.predict(feats)
 gbr_preds = gbr_model.predict(feats )
for i in list(train.columns):
dtype = train[i].dtype
values = 0
if(dtype == float or dtype == int):
method = 'mean'
else:
method = 'mode'
if(train[i].notnull().sum() / 891 <=.5):
train.drop(i, axis = 1, inplace=True)
 elif method == 'mean':
train[i]=train[i].fillna(train[i].mean())
 else:
train[i]=train[i].fillna(train[i].mode() [0])
 for i in list(test.columns):
dtype = test[i].dtype
values = 0
if(dtype == float or dtype == int):
method = 'mean'
else:
method = 'mode'
if(test[i].notnull().sum() / 418 <=.5):
test.drop(i, axis = 1, inplace=True)
 elif method == 'mean':
test[i]=test[i].fillna(test[i].mean())
 else:
test[i]=test[i].fillna(test[i].mode() [0])
<categorify>
stack_preds =(( 0.1*em_preds)+(0.1*lasso_preds)+(0.1*ridge_preds)+(0.1 * gbr_preds)
 +(0.1*lgbm_preds)+(0.5*stack_gen_preds))
train_test_data = [train, test]
for dataset in train_test_data:
dataset['Title'] = dataset['Name'].str.extract('([A-Za-z]+)\.', expand=False)
 title_mapping = {"Mr": 0, "Miss": 1, "Mrs": 1,
"Master": 0, "Dr": 1, "Rev": 0, "Col": 0, "Major": 0, "Mlle": 1,"Countess": 1,
"Ms": 1, "Lady": 1, "Jonkheer": 1, "Don": 0, "Dona" : 1, "Mme": 0,"Capt": 0,"Sir": 0 }
for dataset in train_test_data:
dataset['Title'] = dataset['Title'].map(title_mapping)
<categorify>
final_predictions = np.exp(stack_preds)
 print("Original predictions are:
", stack_preds[:5], "
")
 print("Final predictions are:
", final_predictions[:5] )
<count_values><EOS> 
submission = pd.DataFrame()
submission['Id'] = test.Id
submission['SalePrice'] = final_predictions
submission.to_csv('submission1.csv', index=False )