Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<train_model>
%load_ext autoreload
%autoreload 2
%matplotlib inline
training_history = model.fit(
 x_train,
y_train,
epochs=epochs,
verbose=1,
validation_data=(x_val, y_val),
callbacks=callback_list
)<train_model>
sns.set(style="darkgrid")
 warnings.filterwarnings('ignore' )
image_generator = ImageDataGenerator(
 rotation_range=15,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.1 
)
 image_generator.fit(x_train )<choose_model_class>
PATH = ".. /input/house-prices-advanced-regression-techniques/"
model_augmented = keras.Sequential()
model_augmented.add(Conv2D(32, kernel_size=(3, 3), strides=(1, 1),
activation='relu',
input_shape=(size, size, 1)))
 model_augmented.add(Conv2D(32,(3, 3), activation='relu', strides=(2, 2)))
 model_augmented.add(BatchNormalization())
 model_augmented.add(Dropout(0.3))
model_augmented.add(Conv2D(64,(3, 3), activation='relu'))
model_augmented.add(Conv2D(64,(3, 3), activation='relu', strides=(2, 2)))
 model_augmented.add(BatchNormalization())
 model_augmented.add(Dropout(0.3))
model_augmented.add(Conv2D(128,(3, 3), activation='relu'))
model_augmented.add(BatchNormalization())
 model_augmented.add(Flatten())
 model_augmented.add(Dense(256, activation='relu'))
model_augmented.add(Dropout(0.25))
model_augmented.add(Dense(128, activation='relu'))
model_augmented.add(Dense(num_classes, activation='softmax'))
model_augmented.compile(loss=keras.losses.categorical_crossentropy,
optimizer=RMSprop(0.001),
metrics=['accuracy'])
 aug_result = model_augmented.fit_generator(
 image_generator.flow(x_train, y_train, batch_size=batch_size),
epochs=epochs,
steps_per_epoch=len(x_train)// batch_size,
verbose=1,
validation_data=(x_val, y_val),
callbacks=callback_list 
)
 model_augmented.save('mnist_model.h5' )<predict_on_test>
df_train=pd.read_csv(f'{PATH}train.csv')
 df_test=pd.read_csv(f'{PATH}test.csv' )
pred = model.predict(x_test)
 pred_aug = model_augmented.predict(x_test )<save_to_csv>
df_train.SalePrice = np.log1p(df_train.SalePrice )
def convert_prediction_result(model_result):
result = []
for i in range(len(model_result)) :
result += [np.argmax(model_result[i])]
return result
def write_submission(_submission_path, result_arr):
f_out = open(_submission_path, 'w')
 f_out.write("ImageId,Label
")
 for i in range(len(result_arr)) :
f_out.write("{},{}
".format(i+1, result_arr[i]))
f_out.close()
write_submission('submission_base.csv', convert_prediction_result(pred))
write_submission('submission_aug.csv', convert_prediction_result(pred))<set_options>
print('Number of Training Examples = {}'.format(df_train.shape[0]))
print('Number of Test Examples = {}
'.format(df_test.shape[0]))
print('Training X Shape = {}'.format(df_train.shape))
print('Training y Shape = {}
'.format(df_train['SalePrice'].shape[0]))
print('Test X Shape = {}'.format(df_test.shape))
print('Test y Shape = {}
'.format(df_test.shape[0]))
np.random.seed(92)
<define_variables>
ntrain = df_train.shape[0]
ntest = df_test.shape[0]
train_ID = df_train['Id']
test_ID = df_test['Id']
train_data = '/kaggle/input/digit-recognizer/train.csv'
test_data = '/kaggle/input/digit-recognizer/test.csv' 
<load_from_csv>
def concat_df(train_data, test_data):
return pd.concat([train_data, test_data], sort=True ).reset_index(drop=True)
 df_all = concat_df(df_train, df_test)
 df_train.name = 'Training Set'
df_test.name = 'Test Set'
df_all.name = 'All Set'
dfs = [df_train, df_test]
df_all.shape
train_df = pd.read_csv(train_data)
 print(train_df.shape)
 train_df.head() 
<load_from_csv>
y_train_full = df_train.SalePrice.values
df_all.drop(['SalePrice'], axis=1, inplace=True)
 df_all.drop('Id',axis=1,inplace=True )
test_df = pd.read_csv(test_data)
 print(test_df.shape)
 test_df.head()<prepare_x_and_y>
def mark_missing(df):
for col in df.columns:
if df_all[col].isnull().sum() >0:
df_all[col+'_missed']=df_all[col].isnull()
if 'label' in train_df.columns:
y_train = train_df['label'].values.astype('int32')
 train_df = train_df.drop('label', axis = 1)
 else:
pass
x_train = train_df.values.astype('float32')
 x_test = test_df.values.astype('float32' )<define_variables>
mark_missing(df_all )
train_max = np.max(x_train)
 train_min = np.min(x_train)
 test_max = np.max(x_test)
 test_min = np.min(x_test)
<feature_engineering>
def display_missing(df):
for col in df.columns:
print(col, df[col].isnull().sum())
 print('
')
 for df in dfs:
print(format(df.name))
display_missing(df)
 def display_only_missing(df):
all_data_na =(df.isnull().sum() / len(df)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 print(missing_data )
x_train = x_train/255.0
x_test = x_test/255.0<define_variables>
display_only_missing(df_all )
norm_train_max = np.max(x_train)
 norm_train_min = np.min(x_train)
 norm_test_max = np.max(x_test)
 norm_test_min = np.min(x_test )<categorify>
for col in('Alley','MasVnrType','BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2','FireplaceQu','GarageType', 'GarageFinish', 'GarageQual',
'GarageCond','PoolQC','Fence','MiscFeature'):
df_all[col] = df_all[col].fillna('None' )
y_train= to_categorical(y_train)
 num_classes = y_train.shape[1]
num_classes 
<choose_model_class>
display_only_missing(df_all )
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(16,(5,5),activation='relu', input_shape=(28,28,1)) ,
tf.keras.layers.Conv2D(16,(5,5), activation= 'relu'),
tf.keras.layers.Conv2D(16,(5,5), activation= 'relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Conv2D(32,(3,3), activation= 'relu'),
tf.keras.layers.Conv2D(32,(3,3), activation= 'relu'),
tf.keras.layers.Conv2D(32,(3,3), activation = 'relu'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.25),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.MaxPooling2D(2,2),
tf.keras.layers.Dropout(0.50),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(256, activation = 'relu'),
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Dense(10, activation='softmax')
 ])
 print('input shape :', model.input_shape)
 print('output shape :', model.output_shape )<choose_model_class>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars','BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath','MasVnrArea'):
df_all[col] = df_all[col].fillna(0 )
<choose_model_class>
display_only_missing(df_all )
<choose_model_class>
for col in('Utilities','Functional','SaleType','KitchenQual','Exterior2nd','Exterior1st','Electrical'):
df_all[col].fillna(df_all[col].mode() [0], inplace=True)
<choose_model_class>
display_only_missing(df_all )
model.compile(loss = 'categorical_crossentropy', optimizer= RMSprop(lr=0.003), metrics = ['acc'] )<choose_model_class>
df_all.MSZoning.isnull().sum()
train_generator = image.ImageDataGenerator()<split>
df_all["MSZoning"] = df_all["MSZoning"].fillna("None" )
X = x_train
Y = y_train
X_train, X_val, Y_train , Y_val = train_test_split(x_train,y_train, test_size= 0.05, random_state = 92)
 print(X_train.shape)
 batches = train_generator.flow(X_train, Y_train, batch_size=32)
 val_batches = train_generator.flow(X_val, Y_val, batch_size=32 )<train_model>
display_only_missing(df_all )
history = model.fit_generator(
 generator=batches,
steps_per_epoch=batches.n,
epochs=20,
validation_data=val_batches,
validation_steps=val_batches.n,
)<predict_on_test>
df_all['LotFrontage'].isnull().sum()
predictions = model.predict_classes(x_test, verbose=0)
<save_to_csv>
df_all["LotFrontage"] = df_all.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
submissions=pd.DataFrame({"ImageId": list(range(1,len(predictions)+1)) ,
"Label": predictions})
 submissions.to_csv("DR.csv", index=False, header=True )<set_options>
df_all['LotFrontage'].isnull().sum()
%matplotlib inline
warnings.filterwarnings('ignore')
<load_from_csv>
display_only_missing(df_all )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<load_from_csv>
def quick_get_dumm(df):
X_train_full=df.iloc[:ntrain]
X_train, X_valid, y_train, y_valid = train_test_split(pd.get_dummies(X_train_full), y_train_full, random_state=42)
 return X_train, X_valid, y_train, y_valid
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<count_values>
X_train, X_valid, y_train, y_valid = quick_get_dumm(df_all )
train.Embarked.value_counts() /train.shape[0]
train["Embarked"] = train["Embarked"].fillna("S")
 test["Embarked"] = test["Embarked"].fillna("S" )<define_variables>
def rmse(x,y): return math.sqrt(((x-y)**2 ).mean())
 def print_score(m,X_train=X_train, X_valid=X_valid, y_train=y_train, y_valid=y_valid):
res = [rmse(m.predict(X_train), y_train), rmse(m.predict(X_valid), y_valid),
m.score(X_train, y_train), m.score(X_valid, y_valid)]
if hasattr(m, 'oob_score_'): res.append(m.oob_score_)
 print(res )
group_dataset = [train,test]
for df in group_dataset:
nan_age_index = list(df["Age"][df["Age"].isnull() ].index)
 for i in nan_age_index :
age_med = df["Age"].median()
age_pred = df["Age"][(( df['SibSp'] == df.iloc[i]["SibSp"])& 
(df['Parch'] == df.iloc[i]["Parch"])& 
(df['Pclass'] == df.iloc[i]["Pclass"])) ].median()
if not np.isnan(age_pred):
df['Age'].iloc[i] = age_pred
else :
df['Age'].iloc[i] = age_med
f, ax = plt.subplots(figsize =(10,5))
ax = sns.kdeplot(train.Age,shade=True)
 ax.axvline(train.Age.mean() ,color ='r',linestyle = '--')
 ax.legend(['Age','Age mean'],fontsize=12)
 ax.set_xlabel('Age',fontsize=12,color='black')
 ax.set_title('Age Distribution',color='black',fontsize=14)
 y_axis = ax.axes.get_yaxis().set_visible(False)
 sns.despine(left=True )<drop_column>
m_rf = RandomForestRegressor(n_estimators=160, min_samples_leaf=1, max_features=0.5, n_jobs=-1, oob_score=True, random_state=42)
 m_rf.fit(X_train, y_train)
 print_score(m_rf )
col_drop = ['Cabin','Name','Ticket']
group_dataset = [train,test]
for df in group_dataset:
df.drop(columns=col_drop,axis=1,inplace=True )<categorify>
def elastic_score(X,y):
elastic = ElasticNet(random_state=1)
 param = {'l1_ratio' : [0],
'alpha' : [0.017]}
elastic = GridSearchCV(elastic, param, cv=5, scoring='neg_mean_squared_error')
 elastic.fit(X,y)
 print('Elastic:', np.sqrt(elastic.best_score_*-1))
return elastic
col_dummies = ['Embarked','Sex','family_size','Title','Pclass']
train = pd.get_dummies(train,columns=col_dummies,drop_first=True)
 test = pd.get_dummies(test,columns=col_dummies,drop_first=True)
 train.dropna(inplace=True)
 train.Survived =train.Survived.astype('int' )<normalization>
elastic_score(X_train, y_train )
train.drop('PassengerId',axis=1,inplace=True)
 X = train.drop('Survived',axis= 1)
 y = train['Survived']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
 sc = StandardScaler()
X_train = sc.fit_transform(X_train)
 X_test = sc.transform(X_test )<import_modules>
m_xgb = XGBRegressor(n_estimators=160, learning_rate=0.05, random_state=42)
 m_xgb.fit(X_train, y_train)
 print_score(m_xgb )
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix, classification_report,accuracy_score
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier,AdaBoostClassifier,GradientBoostingClassifier,ExtraTreesClassifier
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold, learning_curve
from sklearn.neural_network import MLPClassifier
import xgboost as xgb<compute_train_metric>
def return_original_na(df, feature):
df[feature].loc[df.index[df[feature+'_missed'] == True].tolist() ]=np.nan
return df[feature]
def model_select(classifier,kfold_n):
cv_result = []
cv_means = []
kfold = StratifiedKFold(n_splits= kfold_n)
 cv_result.append(cross_val_score(classifier, X_train, y = y_train, scoring = "accuracy", cv = kfold, n_jobs=4))
cv_means.append(np.mean(cv_result))
return cv_means
model_type = [LogisticRegression() ,SVC() ,KNeighborsClassifier() ,GaussianNB() ,RandomForestClassifier() ,
AdaBoostClassifier() ,GradientBoostingClassifier() ,DecisionTreeClassifier() ,ExtraTreesClassifier() ,
MLPClassifier() ,LinearDiscriminantAnalysis() ]
model_score = [model_select(i,5)for i in model_type]
classifier = ['Logistic Regression','SVC','KNeighbors','Naive Bayes','Random Forest',
'AdaBoost','Gradient Boosting','Decision Tree','Extra Trees','Multiple Layer Perceptron',
'Linear Discriminant']
gbm = xgb.XGBClassifier(n_estimators=1000, learning_rate=0.05,silent=False,n_jobs=4)
 gbm.fit(X_train, y_train,early_stopping_rounds=5,eval_set=[(X_test, y_test)], verbose=False)
 y_pred = gbm.predict(X_test)
 acc_score_1 = accuracy_score(y_pred,y_test)
 ml_model = pd.DataFrame(model_score,classifier ).reset_index()
ml_model.columns=['Model','acc_score']
ml_model.loc[11] = ['xgboost',acc_score_1]
ml_model.sort_values('acc_score',ascending = False,inplace=True)
 ml_model.reset_index(drop=True,inplace = True)
 ml_model
f, ax = plt.subplots(figsize=(10,8))
sns.barplot('acc_score','Model',data=ml_model, ax=ax,palette='RdBu_r',edgecolor=".2")
 for i in ax.patches:
ax.text(i.get_width() +.01, i.get_y() +.55, \
str(round(( i.get_width()), 2)) , fontsize=12, color='black')
 kwargs= {'length':3, 'width':1, 'colors':'black','labelsize':'large'}
ax.tick_params(**kwargs)
 x_axis = ax.axes.get_xaxis().set_visible(False)
 ax.set_title('Model & Accuracy Score',fontsize=16)
 sns.despine(bottom=True)
 plt.show()<choose_model_class>
df_all['LotFrontage']=return_original_na(df_all, 'LotFrontage' )
def keras_model(kernel_init,act_func,optimizer):
classifier = Sequential()
classifier.add(Dense(9,input_shape=(X_train.shape[1],),kernel_initializer=kernel_init,activation=act_func))
classifier.add(Dense(9,kernel_initializer=kernel_init,activation=act_func))
classifier.add(Dense(9,kernel_initializer=kernel_init,activation=act_func))
classifier.add(Dense(9,kernel_initializer=kernel_init,activation=act_func))
classifier.add(Dense(9,kernel_initializer=kernel_init,activation=act_func))
classifier.add(Dense(1,kernel_initializer=kernel_init,activation='sigmoid'))
classifier.compile(optimizer=optimizer,loss = 'binary_crossentropy',metrics=['accuracy'])
 return classifier<train_model>
df_all['LotFrontage'].isnull().sum()
clf = keras_model('lecun_normal','selu','adam')
 clf.fit(X_train,y_train,batch_size=10,epochs=150,validation_data=(X_test,y_test))<predict_on_test>
display_only_missing(df_all )
y_pred_2 = clf.predict_classes(X_test,batch_size=32)
 y_pred =(y_pred_2 > 0.5)
 def y_pred_val(input):
y_pred_1 = []
for i in list(input):
if i == True:
y_pred_1.append(1)
 else:
y_pred_1.append(0)
 return y_pred_1
accuracy_score(y_test,y_pred_2 )<predict_on_test>
def filling_na_with_predictions(df, feature):

 flag_object=0
if df[feature].isnull().sum() >0:
na_rows_idxs=df[df[feature].isnull() ].index
df=pd.concat([ pd.Series(df[feature]), pd.get_dummies(df.drop([feature], axis=1)) ], axis=1)
 Feature_Train_X=df.drop(df[df[feature].isnull() ].index ).drop([feature], axis=1)
 Feature_Train_y=df[feature].drop(df[df[feature].isnull() ].index ).values
Feature_Predict_X=df[df[feature].isnull() ].drop([feature], axis=1)
 if is_string_dtype(df[feature])or is_categorical_dtype(df[feature]):
flag_object=1
le = LabelEncoder()
le.fit(Feature_Train_y)
 Feature_Train_y=le.transform(Feature_Train_y)
 elastic = ElasticNet(random_state=1)
 param = {'l1_ratio' : [0],
'alpha' : [0.017]}
elastic = GridSearchCV(elastic, param, cv=5, scoring='neg_mean_squared_error')
 elastic.fit(Feature_Train_X,Feature_Train_y)
 fillna_values=elastic.predict(Feature_Predict_X)
 if flag_object==1:
fillna_values=le.inverse_transform(np.around(fillna_values ).astype(int))
df[feature]=df[feature].fillna(pd.Series(index=na_rows_idxs,data=fillna_values))
return df[feature]
else:
print('There were no NA values' )
test_1 = test.drop('PassengerId',axis=1)
 test_1 = sc.transform(test_1)
 y_test_pred = clf.predict_classes(test_1,batch_size=32)
 y_test_pred =(y_test_pred > 0.5)
 y_test_pred_2 = y_pred_val(y_test_pred )<save_to_csv>
df_tmp_train=df_all.iloc[:ntrain]
df_tmp_train['SalePrice']=y_train_full
df_tmp_test=df_all.iloc[ntrain:]
submit = pd.DataFrame({"PassengerId": test["PassengerId"],"Survived": y_test_pred_2})
 submit.to_csv(".. /working/submit.csv", index=False )<set_options>
df_tmp_train['LotFrontage']=filling_na_with_predictions(df_tmp_train, "LotFrontage")
 df_tmp_test['LotFrontage']=filling_na_with_predictions(df_tmp_test, "LotFrontage" )
%matplotlib inline
keras_version = keras.__version__
tf_version = K.tensorflow_backend.tf.VERSION
print("keras version:", keras_version)
 print(K.backend() , "version:", tf_version )<load_from_csv>
df_tmp_train.drop(['SalePrice'], axis=1, inplace=True)
 df_all = concat_df(df_tmp_train, df_tmp_test )
rawdata = np.loadtxt('.. /input/train.csv', dtype=int, delimiter=',', skiprows=1 )<train_model>
df_all['LotFrontage'].isnull().sum()
y_oh = to_categorical(y, num_classes)
 X_scaled = X / 127.5 - 1
X_scaled = np.expand_dims(X_scaled, -1)
 num_val = int(y.shape[0] * 0.1)
 validation_mask = np.zeros(y.shape[0], np.bool)
 np.random.seed(1)
 for c in range(num_classes):
idxs = np.random.choice(np.flatnonzero(y == c), num_val // 10, replace=False)
 validation_mask[idxs] = 1
np.random.seed(None)
 X_train = X_scaled[~validation_mask]
X_val = X_scaled[validation_mask]
print("Train/val pixel shapes:", X_train.shape, X_val.shape)
 y_train = y_oh[~validation_mask]
y_val = y_oh[validation_mask]
print("Train/val label shapes:", y_train.shape, y_val.shape)
 print("Validation Set Class Distribution:", np.bincount(y[validation_mask]))<choose_model_class>
def evaluate(df):
X_train, X_valid, y_train, y_valid = quick_get_dumm(df)
 elastic_score(X_train, y_train)
 m_xgb.fit(X_train, y_train)
 print('XGBoost')
 print_score(m_xgb,X_train, X_valid, y_train, y_valid)
 m_rf.fit(X_train, y_train)
 print('Random Forest')
 print_score(m_rf,X_train, X_valid, y_train, y_valid )
def conv2D_bn_relu(x, filters, kernel_size, strides, padding='valid', kernel_initializer='glorot_uniform', name=None):

 x = layers.Conv2D(filters=filters,
kernel_size=kernel_size,
strides=strides,
padding=padding,
kernel_initializer=kernel_initializer,
name=name,
use_bias=False )(x)
 x = layers.BatchNormalization(scale=False )(x)
 return layers.Activation('relu' )(x)
 def inception_module_A(x, filters=None, kernel_initializer='glorot_uniform'):

 if filters is None:
filters = int(x.shape[-1])
 branch_filters = filters // 4
b1 = conv2D_bn_relu(x,
filters=(branch_filters // 3)* 2,
kernel_size=1,
strides=1,
kernel_initializer=kernel_initializer)
 b1 = conv2D_bn_relu(b1,
filters=branch_filters,
kernel_size=3,
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b2 = conv2D_bn_relu(x,
filters=(branch_filters // 3)* 2,
kernel_size=1,
strides=1,
kernel_initializer=kernel_initializer)
 b2 = conv2D_bn_relu(b2,
filters=branch_filters,
kernel_size=3,
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b2 = conv2D_bn_relu(b2,
filters=branch_filters,
kernel_size=3,
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b3 = conv2D_bn_relu(x,
filters=branch_filters,
kernel_size=1,
strides=1,
kernel_initializer=kernel_initializer)
 pool = layers.AveragePooling2D(pool_size=(3, 3), strides=1, padding='same' )(x)
 pool = conv2D_bn_relu(pool,
filters=branch_filters,
kernel_size=1,
strides=1,
kernel_initializer=kernel_initializer)
 return layers.concatenate([b1, b2, b3, pool])
 def inception_module_C(x, filters=None, kernel_initializer='glorot_uniform'):

 if filters is None:
filters = int(x.shape[-1])
 branch_filters = filters // 6
b1 = conv2D_bn_relu(x,
filters=(branch_filters // 2)* 3,
kernel_size=1,
strides=1,
kernel_initializer=kernel_initializer)
 b1a = conv2D_bn_relu(b1,
filters=branch_filters,
kernel_size=(1, 3),
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b1b = conv2D_bn_relu(b1,
filters=branch_filters,
kernel_size=(3, 1),
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b2 = conv2D_bn_relu(x,
filters=(branch_filters // 2)* 3,
kernel_size=1,
strides=1,
kernel_initializer=kernel_initializer)
 b2 = conv2D_bn_relu(b2,
filters=(branch_filters // 4)* 7,
kernel_size=(1, 3),
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b2 = conv2D_bn_relu(b2,
filters=branch_filters * 2,
kernel_size=(3, 1),
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b2a = conv2D_bn_relu(b2,
filters=branch_filters,
kernel_size=(1, 3),
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b2b = conv2D_bn_relu(b2,
branch_filters,
kernel_size=(3, 1),
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b3 = conv2D_bn_relu(x,
filters=branch_filters,
kernel_size=1,
strides=1,
kernel_initializer=kernel_initializer)
 pool = layers.AveragePooling2D(pool_size=(3, 3), strides=1, padding='same' )(x)
 pool = conv2D_bn_relu(pool,
filters=branch_filters,
kernel_size=1,
strides=1,
kernel_initializer=kernel_initializer)
 return layers.concatenate([b1a, b1b, b2a, b2b, b3, pool] )<define_search_model>
evaluate(df_all )
K.clear_session()
stem_width = 64
inputs = layers.Input(shape=X_scaled.shape[1:])
 x = conv2D_bn_relu(inputs,
filters=stem_width,
kernel_size=5,
strides=2,
padding='valid',
name='conv_1')
 x = inception_module_A(x, filters=int(1.5*stem_width))
x = layers.SpatialDropout2D(0.2 )(x)
 x = inception_module_A(x, filters=int(1.5*stem_width))
x = layers.MaxPooling2D(pool_size=(3, 3), strides=2, padding='same' )(x)
 x = layers.SpatialDropout2D(0.2 )(x)
 x = inception_module_C(x, filters=int(2.25*stem_width))
x = layers.GlobalAveragePooling2D()(x)
 x = layers.Dropout(0.3 )(x)
 x = layers.Dense(num_classes, name='logits' )(x)
 x = layers.Activation('softmax', name='softmax' )(x)
 model = Model(inputs=inputs, outputs=x)
 model.summary()<train_model>
df_all['MSZoning']=return_original_na(df_all, 'MSZoning' )
epsilon = 0.001
y_train_smooth = y_train *(1 - epsilon)+ epsilon / 10
print(y_train_smooth )<categorify>
display_only_missing(df_all )
def elastic_transform(image, alpha_range, sigma, random_state=None):

 if random_state is None:
random_state = np.random.RandomState(None)
 if np.isscalar(alpha_range):
alpha = alpha_range
else:
alpha = np.random.uniform(low=alpha_range[0], high=alpha_range[1])
 shape = image.shape
dx = gaussian_filter(( random_state.rand(*shape)* 2 - 1), sigma)* alpha
dy = gaussian_filter(( random_state.rand(*shape)* 2 - 1), sigma)* alpha
x, y, z = np.meshgrid(np.arange(shape[0]), np.arange(shape[1]), np.arange(shape[2]), indexing='ij')
 indices = np.reshape(x+dx,(-1, 1)) , np.reshape(y+dy,(-1, 1)) , np.reshape(z,(-1, 1))
return map_coordinates(image, indices, order=1, mode='reflect' ).reshape(shape )<choose_model_class>
df_all[df_all['MSZoning'].isnull() ].index
class CosineAnneal(keras.callbacks.Callback):

 def __init__(self, max_lr, min_lr, T, T_mul=1, decay_rate=1.0):
self.max_lr = max_lr
self.min_lr = min_lr
self.decay_rate = decay_rate
self.T = T
self.T_cur = 0
self.T_mul = T_mul
self.step = 0
def on_batch_begin(self, batch, logs=None):
if self.T <= self.T_cur:
self.max_lr *= self.decay_rate
self.min_lr *= self.decay_rate
self.T *= self.T_mul
self.T_cur = 0
self.step = 0
lr = self.min_lr + 0.5 *(self.max_lr - self.min_lr)*(1 + np.cos(self.T_cur * np.pi / self.T))
K.set_value(self.model.optimizer.lr, lr)
 self.step += 1
self.T_cur = self.step / self.params['steps']
def on_epoch_end(self, epoch, logs=None):
logs = logs or {}
logs['lr'] = K.get_value(self.model.optimizer.lr )<choose_model_class>
df_all['MSZoning']=filling_na_with_predictions(df_all, 'MSZoning' )
model.compile(loss='categorical_crossentropy',
optimizer=optimizers.Adamax(lr=0.01, beta_1=0.49, beta_2=0.999),
metrics=['accuracy'] )<init_hyperparams>
df_all['MSZoning'].loc[df_all.index[df_all['MSZoning'+'_missed'] == True].tolist() ]
batch_size = 32
epochs = 100
time_id = time.strftime("%Y-%m-%d_%H-%M-%S")
 best_model_filename = 'mnist-inception-best-' + time_id + '.hdf5'
pre_train_weights = model.get_layer('conv_1' ).get_weights() [0]
pre_train_weights = pre_train_weights.transpose(3, 2, 0, 1)
 annealer = CosineAnneal(max_lr=0.014, min_lr=0.003, T=5, T_mul=1, decay_rate=0.99)
 chkpt = keras.callbacks.ModelCheckpoint(best_model_filename, monitor='val_acc',
save_best_only=True, verbose=False)
 datagen = ImageDataGenerator(
 width_shift_range=2,
height_shift_range=2,
preprocessing_function=lambda x: elastic_transform(x, alpha_range=[8, 10], sigma=3)
)
 history = model.fit_generator(
 datagen.flow(X_train, y_train_smooth, batch_size=batch_size, shuffle=True),
epochs=epochs,
steps_per_epoch=(len(y_train)- 1)// batch_size + 1,
validation_data=(X_val, y_val),
callbacks=[annealer, chkpt]
)<train_model>
evaluate(df_all )
model = keras.models.load_model(best_model_filename)
 evaluate_model(model, X_val, y_val, log=history.history, pre_train_weights=pre_train_weights )<load_from_csv>
for col in('Utilities','Functional','SaleType','KitchenQual','Exterior2nd','Exterior1st','Electrical'):
print('Filling with most common:
',df_all[col].loc[df_all.index[df_all[col+'_missed'] == True].tolist() ])
 df_all[col]=return_original_na(df_all, col)
 df_all[col]=filling_na_with_predictions(df_all, col)
 print('Filling with predictions:
',df_all[col].loc[df_all.index[df_all[col+'_missed'] == True].tolist() ] )
test_data = np.loadtxt('.. /input/test.csv', dtype=int, delimiter=',', skiprows=1 )<predict_on_test>
evaluate(df_all )
test_scores = model.predict(test_images)
 test_predictions = np.argmax(test_scores, axis=1 )<save_to_csv>
df_train_save=df_all.iloc[:ntrain]
df_train_save['SalePrice']=y_train_full
df_test_save=df_all.iloc[ntrain:]
header = 'ImageId,Label'
submission = np.stack(( range(1, 28001), test_predictions), axis=1)
 np.savetxt('submission-' + time_id + '.csv', submission, fmt='%i', delimiter=',', header=header, comments='' )<load_from_csv>
df_train_save.to_csv('train_clean.csv', index=False)
 df_test_save.to_csv('test_clean.csv', index=False )
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 train['Type'] = 'train'
test['Type'] = 'test'
data = train.append(test)
 data['Title'] = data['Name']
for name_string in data['Name']:
data['Title'] = data['Name'].str.extract('([A-Za-z]+)\.', expand=True)
 mapping = {'Mlle': 'Miss', 'Ms': 'Miss', 'Mme': 'Mrs', 'Major': 'Other',
'Col': 'Other', 'Dr' : 'Other', 'Rev' : 'Other', 'Capt': 'Other',
'Jonkheer': 'Royal', 'Sir': 'Royal', 'Lady': 'Royal',
'Don': 'Royal', 'Countess': 'Royal', 'Dona': 'Royal'}
data.replace({'Title': mapping}, inplace=True)
 titles = ['Miss', 'Mr', 'Mrs', 'Royal', 'Other', 'Master']
for title in titles:
age_to_impute = data.groupby('Title')['Age'].median() [titles.index(title)]
data.loc[(data['Age'].isnull())&(data['Title'] == title), 'Age'] = age_to_impute
data['Family_Size'] = data['Parch'] + data['SibSp'] + 1
data.loc[:,'FsizeD'] = 'Alone'
data.loc[(data['Family_Size'] > 1),'FsizeD'] = 'Small'
data.loc[(data['Family_Size'] > 4),'FsizeD'] = 'Big'
fa = data[data["Pclass"] == 3]
data['Fare'].fillna(fa['Fare'].median() , inplace = True)
 data.loc[:,'Child'] = 1
data.loc[(data['Age'] >= 18),'Child'] =0
data['Last_Name'] = data['Name'].apply(lambda x: str.split(x, ",")[0])
 DEFAULT_SURVIVAL_VALUE = 0.5
data['Family_Survival'] = DEFAULT_SURVIVAL_VALUE
for grp, grp_df in data[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin == 0.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 0
for _, grp_df in data.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Survival'] == 0)|(row['Family_Survival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin == 0.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 0
data = data.drop(columns = ['Age','Cabin','Embarked','Name','Last_Name',
'Parch', 'SibSp','Ticket', 'Family_Size'])
 target_col = ["Survived"]
id_dataset = ["Type"]
cat_cols = data.nunique() [data.nunique() < 12].keys().tolist()
cat_cols = [x for x in cat_cols ]
num_cols = [x for x in data.columns if x not in cat_cols + target_col + id_dataset]
bin_cols = data.nunique() [data.nunique() == 2].keys().tolist()
multi_cols = [i for i in cat_cols if i not in bin_cols]
le = LabelEncoder()
for i in bin_cols :
data[i] = le.fit_transform(data[i])
 data = pd.get_dummies(data = data,columns = multi_cols)
 std = StandardScaler()
scaled = std.fit_transform(data[num_cols])
 scaled = pd.DataFrame(scaled,columns = num_cols)
 df_data_og = data.copy()
data = data.drop(columns = num_cols,axis = 1)
 data = data.merge(scaled,left_index = True,right_index = True,how = "left")
 data = data.drop(columns = ['PassengerId'],axis = 1)
 cols = data.columns.tolist()
cols.insert(0, cols.pop(cols.index('Survived')))
 data = data.reindex(columns= cols)
 train = data[data['Type'] == 1].drop(columns = ['Type'])
 test = data[data['Type'] == 0].drop(columns = ['Type'])
 X_train = train.iloc[:, 1:20].as_matrix()
y_train = train.iloc[:,0].as_matrix()<choose_model_class>
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
skewness = df_all.select_dtypes(include=numerics ).apply(lambda x: skew(x))
skew_index = skewness[abs(skewness)>= 0.85].index
skewness[skew_index].sort_values(ascending=False )
def make_model() :
model = models.Sequential()
model.add(Dense(13, input_dim = 18, activation = 'relu'))
model.add(Dropout(0.2))
model.add(BatchNormalization())
 model.add(Dense(8, activation = 'relu'))
model.add(BatchNormalization())
 model.add(Dense(1, activation = 'sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])
 return model 
<categorify>

 lam = 0.15
for column in skew_index:
df_all[column] = boxcox1p(df_all[column], lam)
def mixup_data(x, y, alpha=1.0):
if alpha > 0:
lam = np.random.beta(alpha, alpha)
 else:
lam = 1
sample_size = x.shape[0]
index_array = np.arange(sample_size)
 np.random.shuffle(index_array)
 mixed_x = lam * x +(1 - lam)* x[index_array]
mixed_y =(lam * y)+(( 1 - lam)* y[index_array])
 return mixed_x, mixed_y
def make_batches(size, batch_size):
nb_batch = int(np.ceil(size/float(batch_size)))
 return [(i*batch_size, min(size,(i+1)*batch_size)) for i in range(0, nb_batch)]
def batch_generator(X,y,batch_size=128,shuffle=True,mixup=False):
sample_size = X.shape[0]
index_array = np.arange(sample_size)
 while 1:
if shuffle:
np.random.shuffle(index_array)
 batches = make_batches(sample_size, batch_size)
 for batch_index,(batch_start, batch_end)in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
X_batch = X[batch_ids]
y_batch = y[batch_ids]
if mixup:
X_batch,y_batch = mixup_data(X_batch,y_batch,alpha=1.0)
 yield X_batch,y_batch<init_hyperparams>
evaluate(df_all )
def step_decay(epoch):
initial_lrate = 0.01
drop = 0.5
epochs_drop = 10.0
lrate = initial_lrate * np.power(drop, np.floor(( 1+epoch)/epochs_drop))
return lrate
lrate = LearningRateScheduler(step_decay )<train_model>
df_all=pd.get_dummies(df_all )
class LossHistory(keras.callbacks.Callback):
def on_train_begin(self, logs={}):
self.losses = []
self.lr = []
def on_epoch_end(self, batch, logs={}):
self.losses.append(logs.get('loss'))
self.lr.append(step_decay(len(self.losses)) )<compute_test_metric>

 X_test=df_all.iloc[ntrain:]
X_train_full=df_all.iloc[:ntrain]
X_train, X_valid, y_train, y_valid = train_test_split(pd.get_dummies(X_train_full), y_train_full )
def auc(y_true, y_pred):
try:
return tf.py_func(metrics.roc_auc_score,(y_true, y_pred), tf.double)
 except:
return 0.5<split>
all_features = df_all.keys()
df_all = df_all.drop(df_all.loc[:,(df_all==0 ).sum() >=(df_all.shape[0]*0.984)],axis=1)
 df_all = df_all.drop(df_all.loc[:,(df_all==1 ).sum() >=(df_all.shape[0]*0.984)],axis=1)
 remain_features = df_all.keys()
remov_features = [st for st in all_features if st not in remain_features]
print(len(remov_features), 'features were removed:', remov_features )
batch_size = 32
loss_history = LossHistory()
lrate = LearningRateScheduler(step_decay)
 annealer = LearningRateScheduler(lambda x: 1e-2 * 0.95 ** x)
 callbacks_list = [EarlyStopping(monitor='val_loss', patience=10,mode='min'),loss_history, annealer]
sss = StratifiedShuffleSplit(n_splits=5)
 hold_models = []
hold_models_no_mixup = []
for train_index, test_index in sss.split(X_train, y_train):
X_train_hold, X_val_hold = X_train[train_index], X_train[test_index]
y_train_hold, y_val_hold = y_train[train_index], y_train[test_index]
tr_gen = batch_generator(X_train_hold,y_train_hold,batch_size=batch_size,shuffle=True,mixup=True)
 model = make_model()
model.fit_generator(
 tr_gen,
steps_per_epoch=np.ceil(float(len(X_train_hold)) / float(batch_size)) ,
nb_epoch=1000,
verbose=1,
callbacks=callbacks_list,
validation_data=(X_val_hold,y_val_hold),
max_q_size=10, 
)
 hold_models.append(model)
 tr_gen_no_mixup = batch_generator(X_train_hold,y_train_hold,batch_size=batch_size,shuffle=True,mixup=False)
 model_no_mixup = make_model()
model_no_mixup.fit_generator(
 tr_gen,
steps_per_epoch=np.ceil(float(len(X_train_hold)) / float(batch_size)) ,
nb_epoch=1000,
verbose=1,
callbacks=callbacks_list,
validation_data=(X_val_hold,y_val_hold),
max_q_size=10, 
)
 hold_models_no_mixup.append(model_no_mixup)
 del X_train_hold, X_val_hold, y_train_hold, y_val_hold
gc.collect()<find_best_params>
evaluate(df_all )
%matplotlib inline
def append_last_value(target_list, length):
if len(target_list)>= length:
return target_list
for i in range(length - len(target_list)) :
target_list.append(target_list[-1])
 return target_list
model_index = 1
trained_epochs = max(len(hold_models[model_index].history.history['loss']), len(hold_models_no_mixup[model_index].history.history['loss']))
loss_list = append_last_value(hold_models[model_index].history.history['loss'], trained_epochs)
 val_acc_list = append_last_value(hold_models[model_index].history.history['val_accuracy'], trained_epochs)
 loss_list_no_mixup = append_last_value(hold_models_no_mixup[model_index].history.history['loss'], trained_epochs)
 val_acc_list_no_mixup = append_last_value(hold_models_no_mixup[model_index].history.history['val_accuracy'], trained_epochs)
 plt.plot(range(1, trained_epochs+1), loss_list, label="loss")
 plt.plot(range(1, trained_epochs+1), val_acc_list, label="val_acc")
 plt.plot(range(1, trained_epochs+1), loss_list_no_mixup, label="loss_no_mixup")
 plt.plot(range(1, trained_epochs+1), val_acc_list_no_mixup, label="val_accuracy_no_mixup")
 plt.xlabel('Epochs')
 plt.ylabel('Accuracy')
 plt.legend()
plt.show()<predict_on_test>
scaler = preprocessing.RobustScaler()
df_all = pd.DataFrame(scaler.fit_transform(df_all))
def get_pred(model_list, test_df):
preds = [model.predict(test_df.drop('Survived', axis=1)) for model in model_list]
model_count = len(preds)
 ensemble_pred = 0
for i in range(model_count):
ensemble_pred = ensemble_pred + preds[i]
ensemble_pred = ensemble_pred / model_count
pred_ = ensemble_pred[:,0]
return pred_
pred = get_pred(hold_models, test)
 pred_no_mixup = get_pred(hold_models_no_mixup, test )<define_variables>
evaluate(df_all )
train_survived = train['Survived'].sum()
train_len = train.shape[0]
print('train dataset : ' + str(train_survived)+ '/' + str(train_len)+ ' : ' + str(train_survived / train_len))
thresholds = [0.4, 0.45, 0.5, 0.55, 0.6, 0.7, 0.75, 0.8]
for threshold in thresholds:
pred_survived = pred[pred > threshold].shape[0]
pred_len = pred.shape[0]
print('threshold : ' + str(threshold)+ ' -> ' + str(pred_survived)+ '/' + str(pred_len)+ ' : ' + str(pred_survived / pred_len))<save_to_csv>

 X_test=df_all.iloc[ntrain:]
X_train_full=df_all.iloc[:ntrain]
X_train, X_valid, y_train, y_valid = train_test_split(pd.get_dummies(X_train_full), y_train_full )
submit_threshold = 0.4
submit_df = pd.read_csv('.. /input/titanic/test.csv')
 submit_df['Survived'] = np.where(pred > submit_threshold, 1, 0)
 submit_df = submit_df[['PassengerId', 'Survived']]
submit_df.to_csv('submission.csv', index=False )<init_hyperparams>
print('ElasticNet')
 def elastic_score(X,y):
elastic = ElasticNet(random_state=1)
 param = {'l1_ratio' : [0],
'alpha' : [0.017]}
elastic = GridSearchCV(elastic, param, cv=5, scoring='neg_mean_squared_error')
 elastic.fit(X,y)
 print('ElasticNet:', np.sqrt(elastic.best_score_*-1))
return elastic
elastic_score(X_train, y_train)
 print('XGBoost')
 m_xgb = XGBRegressor(n_estimators=160, learning_rate=0.05)
 m_xgb.fit(X_train, y_train)
 print_score(m_xgb,X_train, X_valid, y_train, y_valid)
 print('Random Forest')
 m_rf = RandomForestRegressor(n_estimators=160, min_samples_leaf=1, max_features=0.5, n_jobs=-1, oob_score=True)
 m_rf.fit(X_train, y_train)
 print_score(m_rf,X_train, X_valid, y_train, y_valid)
class SnapshotEnsemble(Callback):
__snapshot_name_fmt = "snapshot_%d.hdf5"
def __init__(self, n_models, n_epochs_per_model, lr_max, verbose=1):

 self.n_epochs_per_model = n_epochs_per_model
self.n_models = n_models
self.n_epochs_total = self.n_models * self.n_epochs_per_model
self.lr_max = lr_max
self.verbose = verbose
self.lrs = []
def cosine_annealing(self, epoch):
cos_inner =(math.pi *(epoch % self.n_epochs_per_model)) / self.n_epochs_per_model
return self.lr_max / 2 *(math.cos(cos_inner)+ 1)
 def on_epoch_begin(self, epoch, logs={}):
lr = self.cosine_annealing(epoch)
 backend.set_value(self.model.optimizer.lr, lr)
 self.lrs.append(lr)
 def on_epoch_end(self, epoch, logs={}):
if(epoch + 1)% self.n_epochs_per_model == 0:
filename = self.__snapshot_name_fmt %(( epoch + 1)// self.n_epochs_per_model)
 self.model.save(filename)
 if self.verbose:
print('Epoch %d: snapshot saved to %s' %(epoch, filename))
def load_ensemble(self):
models = []
for i in range(self.n_models):
models.append(load_model(self.__snapshot_name_fmt %(i + 1)))
 return models 
<load_from_csv>
def cv_train() :
elastic = ElasticNet(random_state=1)
 param = {'l1_ratio' : [0],
'alpha' : [0.017]}
elastic = GridSearchCV(elastic, param, cv=5, scoring='neg_mean_squared_error')
 elastic.fit(X_train_full, y_train_full)
 print('Elastic:', np.sqrt(elastic.best_score_*-1))
return elastic
elastic = cv_train()
path = '/kaggle/input/digit-recognizer/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 target = train['label']
train.drop(columns=['label'], inplace=True)
 im_size = 28
train = train.to_numpy().reshape(( -1, im_size, im_size, 1))
test = test.to_numpy().reshape(( -1, im_size, im_size, 1))
train = train / 255
test = test / 255
train.shape, test.shape<split>
y_pred=np.expm1(elastic.predict(X_test)) ; y_pred
target = to_categorical(target)
 x_train, x_test, y_train, y_test = train_test_split(train, target, test_size=0.2, random_state=289)
 x_train.shape, x_test.shape<choose_model_class>
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = y_pred
sub.to_csv('submission_31Aug19.csv',index=False )