Kaggle competition: Santander Customer Satisfaction
<SOS> metric: AUC Kaggle data source: santander-customer-satisfaction<prepare_x_and_y>
%matplotlib inline
def transform(image, inv_mat, image_shape):
h, w, c = image_shape
cx, cy = w//2, h//2
new_xs = tf.repeat(tf.range(-cx, cx, 1), h)
 new_ys = tf.tile(tf.range(-cy, cy, 1), [w])
 new_zs = tf.ones([h*w], dtype=tf.int32)
 old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))
old_coords_x, old_coords_y = tf.round(old_coords[0, :] + w//2), tf.round(old_coords[1, :] + h//2)
 clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)
 clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)
 clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)
 old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))
old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))
new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))
new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))
old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)
 new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)
 rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))
rotated_image_channel = list()
for i in range(c):
vals = rotated_image_values[:,i]
sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])
 rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))
return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])
 def random_rotate(image, angle, image_shape):
def get_rotation_mat_inv(angle):
angle = math.pi * angle / 180
cos_val = tf.math.cos(angle)
 sin_val = tf.math.sin(angle)
 one = tf.constant([1], tf.float32)
 zero = tf.constant([0], tf.float32)
 rot_mat_inv = tf.concat([cos_val, sin_val, zero,
-sin_val, cos_val, zero,
zero, zero, one], axis=0)
 rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])
 return rot_mat_inv
angle = float(angle)* tf.random.normal([1],dtype='float32')
 rot_mat_inv = get_rotation_mat_inv(angle)
 return transform(image, rot_mat_inv, image_shape)
 def GridMask(image_height, image_width, d1, d2, rotate_angle=1, ratio=0.5):
h, w = image_height, image_width
hh = int(np.ceil(np.sqrt(h*h+w*w)))
 hh = hh+1 if hh%2==1 else hh
d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)
 l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)
 st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)
 x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)
 for i in range(0, hh//d+1):
s1 = i * d + st_h
s2 = i * d + st_w
y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)
 x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)
 x_clip_mask = tf.logical_or(x_ranges <0 , x_ranges > hh-1)
 y_clip_mask = tf.logical_or(y_ranges <0 , y_ranges > hh-1)
 clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)
 x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))
y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))
hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)) , tf.int32)])
 x_ranges = tf.repeat(x_ranges, hh)
 y_ranges = tf.repeat(y_ranges, hh)
 y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))
x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))
y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64), tf.zeros_like(y_ranges), [hh, hh])
 y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)
 x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])
 x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)
 mask = tf.expand_dims(tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)
 mask = random_rotate(mask, rotate_angle, [hh, hh, 1])
 mask = tf.image.crop_to_bounding_box(mask,(hh-h)//2,(hh-w)//2, image_height, image_width)
 return mask
def apply_grid_mask(image, image_shape):
mask = GridMask(image_shape[0],
image_shape[1],
AugParams['d1'],
AugParams['d2'],
AugParams['rotate'],
AugParams['ratio'])
 if image_shape[-1] == 3:
mask = tf.concat([mask, mask, mask], axis=-1)
 return image * tf.cast(mask, tf.float32 )<categorify>
train_data = pd.read_csv('.. /input/santander-customer-satisfaction/train.csv')
 test_data = pd.read_csv('.. /input/santander-customer-satisfaction/test.csv' )
def random_blockout(img, sl=0.1, sh=0.2, rl=0.4):
h, w, c = IMAGE_SIZE[0], IMAGE_SIZE[1], 3
origin_area = tf.cast(h*w, tf.float32)
 e_size_l = tf.cast(tf.round(tf.sqrt(origin_area * sl * rl)) , tf.int32)
 e_size_h = tf.cast(tf.round(tf.sqrt(origin_area * sh / rl)) , tf.int32)
 e_height_h = tf.minimum(e_size_h, h)
 e_width_h = tf.minimum(e_size_h, w)
 erase_height = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_height_h, dtype=tf.int32)
 erase_width = tf.random.uniform(shape=[], minval=e_size_l, maxval=e_width_h, dtype=tf.int32)
 erase_area = tf.zeros(shape=[erase_height, erase_width, c])
 erase_area = tf.cast(erase_area, tf.uint8)
 pad_h = h - erase_height
pad_top = tf.random.uniform(shape=[], minval=0, maxval=pad_h, dtype=tf.int32)
 pad_bottom = pad_h - pad_top
pad_w = w - erase_width
pad_left = tf.random.uniform(shape=[], minval=0, maxval=pad_w, dtype=tf.int32)
 pad_right = pad_w - pad_left
erase_mask = tf.pad([erase_area], [[0,0],[pad_top, pad_bottom], [pad_left, pad_right], [0,0]], constant_values=1)
 erase_mask = tf.squeeze(erase_mask, axis=0)
 erased_img = tf.multiply(tf.cast(img,tf.float32), tf.cast(erase_mask, tf.float32))
return tf.cast(erased_img, img.dtype )<normalization>
train_data.isnull().sum()
def augmentation(image, label):
prob = tf.random.uniform(shape=[], minval=0.0, maxval=1.0)
 if prob <= 0.25: image = apply_grid_mask(image,(*IMAGE_SIZE,3))
elif prob <= 0.50: image = rot_shift_zoom_shear(image)
 elif prob <= 0.75: image = random_blockout(image)
 else: pass
return tf.cast(image, tf.float32), label<define_variables>
import plotly.express as px
from matplotlib import rcParams
import plotly.figure_factory as ff
from plotly import tools
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
train_input =(
 train_dataset 
.repeat() 
.map(augmentation, num_parallel_calls=AUTO)
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 val_input =(
 val_dataset 
.batch(BATCH_SIZE)
.cache() 
)
 test_input =(
 test_dataset.batch(BATCH_SIZE)
.prefetch(AUTO)
 )<choose_model_class>
train_data.drop(labels = constant_features, axis = 1, inplace=True)
 test_data.drop(labels = constant_features, axis = 1, inplace = True )
params = {
"include_top":False,
"input_shape":[*IMAGE_SIZE, 3],
"classes":104,
"pooling":"avg"}
def create_dnet() :
model = keras.applications.DenseNet201(**params)
 compiler = re.compile(r"conv5_block")
 for layer in model.layers:
if not compiler.search(layer.name):
layer.trainable = False
return model
def create_efn() :
model = efn.EfficientNetB6(**params)
 compiler = re.compile(r"block7")
 for layer in model.layers:
if not compiler.search(layer.name):
layer.trainable = False
return model<choose_model_class>
approx_constants = []
for feature in train_data.columns:
approx_value =(train_data[feature].value_counts() / np.float(
 len(train_data)) ).sort_values(ascending=False ).values[0]
if approx_value > 0.999:
approx_constants.append(feature)
 len(approx_constants)
 train_data[approx_constants]
loss = "sparse_categorical_crossentropy"
metric = "sparse_categorical_accuracy"
with strategy.scope() :
efficient = create_efn()
dense_net = create_dnet()
input_image = keras.Input(shape=[*IMAGE_SIZE, 3])
 efficient = efficient(input_image)
 dense_net = dense_net(input_image)
 dropout_1 = keras.layers.Dropout(0.3 )(efficient)
 dropout_2 = keras.layers.Dropout(0.3 )(dense_net)
 concat = keras.layers.concatenate([dropout_1, dropout_2])
 dense_2 = keras.layers.Dense(104, activation="softmax" )(concat)
 merged_model = keras.Model(input_image, dense_2)
 merged_model.compile(optimizer=keras.optimizers.Adam(lr_start),
loss=loss,
metrics=[metric])
 full_model = merged_model<choose_model_class>
train_data_ac=train_data.copy()
test_data_ac=test_data.copy()
early_stop = keras.callbacks.EarlyStopping(monitor=metric, patience=5)
 check_point = keras.callbacks.ModelCheckpoint(
 filepath="best_model.h5",
monitor="val_sparse_categorical_accuracy",
save_best_only=True)
 learning_schedule = keras.callbacks.LearningRateScheduler(scheduler)
 callbacks_list = [early_stop, check_point, learning_schedule]<train_model>
train_data_ac.drop(labels = approx_constants, axis = 1, inplace=True)
 test_data_ac.drop(labels = approx_constants, axis = 1, inplace = True )
<concatenate>
duplicate_features=[]
for i in range(0, len(train_data.columns)) :
col_1 = train_data.columns[i]
for col_2 in train_data.columns[i + 1:]:
if train_data[col_1].equals(train_data[col_2]):
duplicate_features.append(col_2)
 len(duplicate_features)
 train_data[duplicate_features]
full_input =(
 train_dataset.concatenate(val_dataset)
.repeat() 
.map(augmentation, num_parallel_calls=AUTO)
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )<train_model>
train_data.drop(labels = duplicate_features, axis = 1, inplace=True)
 test_data.drop(labels = duplicate_features, axis = 1, inplace = True )
check_point.monitor = metric
history = full_model.fit(full_input,
epochs=EPOCHS,
steps_per_epoch=STEPS_PER_EPOCHS_FULL,
callbacks=callbacks_list,
)<load_pretrained>
for feature in duplicate_features:
if feature in train_data_ac and feature in test_data_ac:
train_data_ac.drop(labels = feature, axis = 1, inplace=True)
 test_data_ac.drop(labels = feature, axis = 1, inplace = True )
best_model = keras.models.load_model("./best_model.h5" )<load_pretrained>
def features_wo_low_variance(data):
threshold_n=0.98
sel = VarianceThreshold(threshold=(threshold_n*(1 - threshold_n)))
 sel_var=sel.fit_transform(data)
 return data.columns[sel.get_support(indices=True)]
best_weights = best_model.get_weights()
merged_model.set_weights(best_weights )<categorify>
( train_data.var() < 0.02 ).value_counts()
def aug_gridmask(image):
image = apply_grid_mask(image,(*IMAGE_SIZE, 3))
return tf.cast(image, tf.float32)
 def aug_rot(image):
image = rot_shift_zoom_shear(image)
 return tf.cast(image, tf.float32)
 def aug_blockout(image):
image = random_blockout(image)
 return tf.cast(image, tf.float32)
 def aug_none(image):
return image
aug_list = [aug_gridmask, aug_rot, aug_blockout, aug_none]<predict_on_test>
train_data_hv = train_data[features_wo_low_variance(train_data)].copy()
test_data_hv = test_data[features_wo_low_variance(train_data)[:-1]].copy()
def predict_tta(model, test_image, aug_list=aug_list):
probs = []
for index, aug in enumerate(aug_list):
copy_image =(
 test_image.unbatch() 
.map(aug)
.batch(BATCH_SIZE))
probs.append(model.predict(copy_image))
return np.mean(probs, axis=0 )<predict_on_test>
train_data_ac_hv = train_data_ac[features_wo_low_variance(train_data_ac)].copy()
test_data_ac_hv = test_data_ac[features_wo_low_variance(train_data_ac)[:-1]].copy()
test_image = test_input.map(lambda image, _id: image, num_parallel_calls=AUTO)
 test_id = test_input.map(lambda image, _id: _id ).unbatch().batch(TEST_IMG_NUM)
 image_id = next(iter(test_id)).numpy().astype("U")
 probabilities = predict_tta(merged_model, test_image)
 predictions = np.argmax(probabilities, axis=-1 )<save_to_csv>
def find_correlated_features(data):
corr_matrix = data.corr().abs()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > 0.95)]
print(len(to_drop))
return to_drop
submission = pd.DataFrame({"id": image_id, "label": predictions})
 submission.to_csv("submission.csv", index=False )<import_modules>
train_data_hv_co= train_data_hv.drop(find_correlated_features(train_data_hv), axis=1)
 test_data_hv_co= test_data_hv.drop(find_correlated_features(train_data_hv), axis=1 )
import numpy as np
import pandas as pd
from fastai.vision import *
from fastai.tabular import *
import matplotlib.pyplot as plt
import re<define_variables>
train_data_ac_hv_co= train_data_ac_hv.drop(find_correlated_features(train_data_ac_hv), axis=1)
 test_data_ac_hv_co= test_data_ac_hv.drop(find_correlated_features(train_data_ac_hv), axis=1 )
path=Path('/kaggle/working/' )<load_from_csv>
print(train_data_ac_hv_co <(Q1 - 1.5 * IQR)) or(train_data_ac_hv_co >(Q3 + 1.5 * IQR))
train=pd.read_csv('/kaggle/input/train.csv')
 train['index'] = train.index
train.head()<feature_engineering>
train_data_ol=train_data_ac_hv_co.copy()
train_data_out = train_data_ol[(( train_data_ol >=(Q1 - 1.5 * IQR)) &(train_data_ol <=(Q3 + 1.5 * IQR)) ).all(axis=1)]
tfms=get_transforms(rand_pad(padding=5,size=28,mode='zeros'))<normalization>
train_data_out["TARGET"].value_counts()
data =(src.transform(tfms,size=128)
.databunch(num_workers=5,bs=48)
.normalize())
<define_variables>
train_data.var3 = train_data.var3.replace(-999999,2 )
data.show_batch(rows=3,figsize=(10,7))<choose_model_class>
test_data.var3 = test_data.var3.replace(-999999,2 )
learner=cnn_learner(data,models.resnet101,metrics=[FBeta() ,accuracy,error_rate] )<define_search_space>
test_data_hv_co.var3 = test_data_hv_co.var3.replace(-999999,2)
 train_data_hv_co.var3 = train_data_hv_co.var3.replace(-999999,2 )
lr=9e-3<train_model>
test_data_ac.var3 = test_data_ac.var3.replace(-999999,2)
 train_data_ac.var3 = train_data_ac.var3.replace(-999999,2 )
learner.fit_one_cycle(5,slice(lr))<save_model>
test_data_ac_hv.var3 = test_data_ac_hv.var3.replace(-999999,2)
 train_data_ac_hv.var3 = train_data_ac_hv.var3.replace(-999999,2 )
learner.save('Resnet1' )<train_model>
test_data_ac_hv_co.var3 = test_data_ac_hv_co.var3.replace(-999999,2)
 train_data_ac_hv_co.var3 = train_data_ac_hv_co.var3.replace(-999999,2 )
learner.fit_one_cycle(4,slice(5e-6,lr/50))<save_model>
train_data_scaled2= train_data_hv_co.drop(["ID","TARGET"],axis=1)
 train_data_scaled1= train_data_ac_hv_co.drop(["ID","TARGET"],axis=1)
 train_data_scaled= train_data.drop(["ID","TARGET"],axis=1 )
learner.save('Resnetfinal' )<load_from_csv>
for data in [train_data_scaled,train_data_scaled1,train_data_scaled2]:
scaler = StandardScaler()
find_pca_components(scaler.fit_transform(data))
train=pd.read_csv('/kaggle/input/train.csv')
 data=TabularDataBunch.from_df(path,train,dep_var='label',valid_idx=range(4000,6000))<choose_model_class>
def pca_analysis(n_co,data):
pca = PCA(n_components=n_co)
 data_transformed = pca.fit_transform(data)
 print("PCA Analysis for %s pca components"%(n_co))
print("Eigen vector for each principal component : ",pca.components_)
 print("Amount of variance by each PCA : ", pca.explained_variance_)
 print("Percentage of variance by each PCA : ", pca.explained_variance_ratio_)
 print("number of features in training data : ", pca.n_features_)
 print("number of samples in training data: ", pca.n_samples_)
 print("noise variance of the data : ",pca.noise_variance_)
 return pd.DataFrame(data_transformed )
tablearner=tabular_learner(data,layers=[200,100],ps=[0.001,0.01],emb_drop=0.004,metrics=accuracy )<find_best_params>
data_transformed=[]
for no, data in [(48,train_data_scaled),(44,train_data_scaled1),(60,train_data_scaled2)]:
scaler = StandardScaler()
data_transformed.append(pca_analysis(no,scaler.fit_transform(data)) )
tablearner.lr_find()
tablearner.recorder.plot(suggestion=True )<define_search_space>
def upsampling_dataset(data):
data_majority=data[data.TARGET==0]
data_minority=data[data.TARGET==1]
data_minority_upsampled=resample(data_minority,replace=True,n_samples=73012)
 data_upsampled=pd.concat([data_minority_upsampled,data_majority])
 data_upsampled.info()
print(data_upsampled['TARGET'].value_counts())
 return data_upsampled
lr=2.2e-2<train_model>
datasets = [train_data_ac,train_data_ac_hv_co,train_data_hv_co]
tablearner.fit_one_cycle(10,slice(lr),wd=0.1 )<find_best_params>
test_datasets = [test_data_ac,test_data_ac_hv_co,test_data_hv_co]
tablearner.lr_find()
tablearner.recorder.plot(suggestion=True )<train_model>
upsampled_data = []
for data in datasets:
upsampled_data.append(upsampling_dataset(data))
tablearner.fit_one_cycle(4,4e-6,wd=0.1 )<load_from_csv>
if 'ID' not in data_transformed[2]:
data_transformed[2].insert(1,'ID',train_data['ID'])
 x_pca = data_transformed[2]
y_pca = train_data["TARGET"]
df_test = pd.read_csv('.. /input/test.csv')
 df_test['label'] = 0
df_test['index'] = df_test.index
df_test.head()<load_pretrained>
x_upsample = upsampled_data[2].drop("TARGET",axis=1)
 y_upsample = upsampled_data[2]["TARGET"]
learner.load('Resnetfinal' )<create_dataframe>
x = datasets[2].drop("TARGET",axis=1)
 y = datasets[2]["TARGET"]
learner.data.add_test(PixelList.from_df(df_test,path='.',cols='index'))<predict_on_test>
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.30, random_state=44)
 X_pca_train, X_pca_test, y_pca_train, y_pca_test = train_test_split(x_pca, y_pca, test_size=0.30, random_state=44)
 X_upsample_train, X_upsample_test, y_upsample_train, y_upsample_test = train_test_split(x_upsample, y_upsample, test_size=0.30, random_state=44 )
pred_test = learner.get_preds(ds_type=DatasetType.Test)
 test_result = torch.argmax(pred_test[0],dim=1)
 result = test_result.numpy()<save_to_csv>
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_selection import SelectFromModel
final = pd.Series(result,name='Label')
 submission = pd.concat([pd.Series(range(1,28001),name='ImageId'),final],axis=1)
 submission.to_csv('submit.csv',index=False )<load_from_csv>
sel_ = SelectFromModel(RandomForestClassifier(n_estimators=200))
sel_.fit(X_train.fillna(0), y_train )
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )<count_missing_values>
selected_features = X_train.columns[(sel_.get_support())]
len(selected_features )
def check_missing(df):
missing_cols = df.columns[df.isna().any() ].tolist()
total_missing = df[missing_cols].isnull().sum().sort_values(ascending=False)
 percentage =(df[missing_cols].isnull().sum())/(len(df[missing_cols]))
percentage = percentage.sort_values(ascending=False)
 print(pd.concat([total_missing, percentage], axis=1, keys= ['<count_missing_values>
train_data[selected_features]
check_missing(train )<count_missing_values>
from sklearn import model_selection
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier, BaggingClassifier,GradientBoostingClassifier
from xgboost import XGBClassifier
from sklearn.svm import SVC
check_missing(test )<categorify>
models = []
models.append(( 'LR', LogisticRegression(class_weight='balanced')))
 models.append(( 'Bagging Classifier',BaggingClassifier()))
models.append(( 'KNN', KNeighborsClassifier(weights='distance')))
 models.append(( 'RandomForest', RandomForestClassifier(class_weight='balanced')))
 models.append(( 'DecisionTree', DecisionTreeClassifier(class_weight='balanced')))
 models.append(( 'GradientBoosting', GradientBoostingClassifier()))
models.append(( 'xgb', XGBClassifier(missing=np.nan, max_depth=6,
n_estimators=350, learning_rate=0.025, nthread=4, subsample=0.95,
colsample_bytree=0.85, seed=4242)) )
train_cleaned = train.copy()
test_cleaned = test.copy()
train_cleaned['PoolQC'].fillna('None', inplace = True)
 test_cleaned['PoolQC'].fillna('None', inplace = True)
 train_cleaned['MiscFeature'].fillna('None', inplace = True)
 test_cleaned['MiscFeature'].fillna('None', inplace = True)
 train_cleaned['Alley'].fillna('None', inplace = True)
 test_cleaned['Alley'].fillna('None', inplace = True)
 train_cleaned['Fence'].fillna('None', inplace = True)
 test_cleaned['Fence'].fillna('None', inplace = True)
 train_cleaned['FireplaceQu'].fillna('None', inplace = True)
 test_cleaned['FireplaceQu'].fillna('None', inplace = True)
 train_cleaned['GarageCond'].fillna('None', inplace = True)
 test_cleaned['GarageCond'].fillna('None', inplace = True)
 train_cleaned['GarageQual'].fillna('None', inplace = True)
 test_cleaned['GarageQual'].fillna('None', inplace = True)
 train_cleaned['GarageFinish'].fillna('None', inplace = True)
 test_cleaned['GarageFinish'].fillna('None', inplace = True)
 train_cleaned['GarageType'].fillna('None', inplace = True)
 test_cleaned['GarageType'].fillna('None', inplace = True)
 train_cleaned['GarageYrBlt'].fillna(0, inplace = True)
 test_cleaned['GarageYrBlt'].fillna(0, inplace = True)
 test_cleaned['GarageArea'].fillna(0, inplace = True)
 test_cleaned['GarageCars'].fillna(0, inplace = True)
 train_cleaned['BsmtFinType2'].fillna('None', inplace = True)
 test_cleaned['BsmtFinType2'].fillna('None', inplace = True)
 train_cleaned['BsmtExposure'].fillna('None', inplace = True)
 test_cleaned['BsmtExposure'].fillna('None', inplace = True)
 train_cleaned['BsmtFinType1'].fillna('None', inplace = True)
 test_cleaned['BsmtFinType1'].fillna('None', inplace = True)
 train_cleaned['BsmtCond'].fillna('None', inplace = True)
 test_cleaned['BsmtCond'].fillna('None', inplace = True)
 train_cleaned['BsmtQual'].fillna('None', inplace = True)
 test_cleaned['BsmtQual'].fillna('None', inplace = True)
 test_cleaned['BsmtHalfBath'].fillna(0, inplace = True)
 test_cleaned['BsmtFullBath'].fillna(0, inplace = True)
 test_cleaned['TotalBsmtSF'].fillna(0, inplace = True)
 test_cleaned['BsmtUnfSF'].fillna(0, inplace = True)
 test_cleaned['BsmtFinSF2'].fillna(0, inplace = True)
 test_cleaned['BsmtFinSF1'].fillna(0, inplace = True)
 train_cleaned['MasVnrType'].fillna('None', inplace = True)
 test_cleaned['MasVnrType'].fillna('None', inplace = True)
 train_cleaned['MasVnrArea'].fillna(0, inplace = True)
 test_cleaned['MasVnrArea'].fillna(0, inplace = True)
 train_cleaned['Electrical'].fillna('SBrkr', inplace=True)
 test_cleaned['KitchenQual'].fillna('TA', inplace=True)
 test_cleaned['Exterior1st'].fillna('VinylSd', inplace=True)
 test_cleaned['Exterior2nd'].fillna('VinylSd', inplace=True)
 test_cleaned['MSZoning'].fillna('RL', inplace=True)
 test_cleaned['Functional'].fillna('Typ', inplace=True)
 test_cleaned['Utilities'].fillna('AllPub', inplace=True)
 test_cleaned['SaleType'].fillna('WD', inplace=True)
 print(train_cleaned.groupby('Neighborhood' ).LotFrontage.median())
 train_cleaned['LotFrontage'] = train_cleaned.groupby('Neighborhood' ).LotFrontage.transform(lambda x: x.fillna(x.median()))
test_cleaned['LotFrontage'] = test_cleaned.groupby('Neighborhood' ).LotFrontage.transform(lambda x: x.fillna(x.median()))
print("_"*35)
 print(check_missing(train_cleaned))
print(check_missing(test_cleaned))<drop_column>
from sklearn import metrics
from sklearn.metrics import classification_report,confusion_matrix
from sklearn.model_selection import cross_val_score
from sklearn.metrics import f1_score,roc_curve,roc_auc_score,precision_score,recall_score,accuracy_score
train_ID = train_cleaned['Id']
test_ID = test_cleaned['Id']
train_cleaned = train_cleaned.drop(['Id'], axis=1)
 test_cleaned = test_cleaned.drop(['Id'], axis=1 )<data_type_conversions>
def model_analysis(title,x_train,y_train,x_test,y_test):
df_scores=pd.DataFrame()
pred_dict={}
for name,model in models:
model.fit(x_train,y_train)
 y_pred = model.predict(x_test)
 pred_dict[name] = y_pred
confusion = confusion_matrix(y_test,y_pred)
 TP = confusion[1, 1]
TN = confusion[0, 0]
FP = confusion[0, 1]
FN = confusion[1, 0]
accuracy = metrics.accuracy_score(y_test, y_pred)
 error = 1-accuracy
sensitivity = TP / float(FN + TP)
 specificity = TN /(TN + FP)
 False_positive_rate = 1-specificity
precision = TP / float(TP + FP)
 bal_acc = metrics.balanced_accuracy_score(y_test, y_pred)
 Null_accuracy = max(y_test.mean() ,(1 - y_test.mean()))
f1 = metrics.f1_score(y_test,y_pred)
 auc_score = metrics.roc_auc_score(y_test,y_pred)
 clf_score = pd.DataFrame(
 {name: [accuracy, bal_acc, Null_accuracy,precision,sensitivity,f1,error,specificity,auc_score]},
index=['Accuracy', 'Balanced accuracy','Null_accuracy','precision','recall','f1 score','error','specificity','auc_score'] 
)
 df_scores = pd.concat([df_scores, clf_score], axis=1 ).round(decimals=3)
 print("Roc_curve for all models")
 plot_roc_curve(y_test,pred_dict)
 print(title,end='
')
 print(df_scores.to_markdown() ,end='
')
 model_comparison_plot(df_scores )
train_cleaned['MSSubClass']=train_cleaned['MSSubClass'].apply(str)
 test_cleaned['MSSubClass']=test_cleaned['MSSubClass'].apply(str)
 train_cleaned['MoSold']=train_cleaned['MoSold'].astype(str)
 test_cleaned['MoSold']=test_cleaned['MoSold'].astype(str )<categorify>
model_analysis("Model with normal data",X_train,y_train,X_test,y_test )
train_cleaned = train_cleaned.replace({
'LotShape': {'IR3': 1, 'IR2': 2, 'IR1': 3, 'Reg': 4},
'LandSlope': {'Sev': 1, 'Mod': 2, 'Gtl': 3},
'ExterQual': {'Po' : 1, 'Fa' : 2, 'TA' : 3, 'Gd' : 4, 'Ex' : 5},
'BsmtQual' : {'None':0, 'Po' : 1, 'Fa' : 2, 'TA' : 3, 'Gd' : 4, 'Ex' : 5},
'BsmtCond' : {'None':0, 'Po' : 1, 'Fa' : 2, 'TA' : 3, 'Gd' : 4, 'Ex' : 5},
'BsmtExposure': {'None':0, 'No' : 1, 'Mn' : 2, 'Av' : 3, 'Gd' : 4},
'BsmtFinType1': {'None':0, 'Unf': 1, 'LwQ': 2, 'Rec': 3, 'BLQ': 4, 'ALQ': 5, 'GLQ': 6},
'BsmtFinType2': {'None':0, 'Unf': 1, 'LwQ': 2, 'Rec': 3, 'BLQ': 4, 'ALQ': 5, 'GLQ': 6},
'HeatingQC' : {'Po' : 1, 'Fa' : 2, 'TA' : 3, 'Gd' : 4, 'Ex' : 5},
'KitchenQual' : {'Po' : 1, 'Fa' : 2, 'TA' : 3, 'Gd' : 4, 'Ex' : 5},
'Functional': {'Sal': 1, 'Sev': 2, 'Maj2': 3, 'Maj1': 4, 'Mod': 5, 'Min2': 6, 'Min1': 7, 'Typ': 8},
'FireplaceQu': {'None':0, 'Po' : 1, 'Fa' : 2, 'TA' : 3, 'Gd' : 4, 'Ex' : 5},
'GarageFinish': {'None':0, 'Unf': 1, 'RFn': 2, 'Fin': 3},
'GarageQual': {'None':0, 'Po' : 1, 'Fa' : 2, 'TA' : 3, 'Gd' : 4, 'Ex' : 5},
'BsmtCond' : {'None':0, 'Po' : 1, 'Fa' : 2, 'TA' : 3, 'Gd' : 4, 'Ex' : 5},
'PavedDrive' : {'N' : 0, 'P' : 1, 'Y' : 2},
'PoolQC' : {'None':0, 'Fa' : 1, 'TA' : 2, 'Gd' : 3, 'Ex' : 4},
})
 test_cleaned = test_cleaned.replace({
'LotShape': {'IR3': 1, 'IR2': 2, 'IR1': 3, 'Reg': 4},
'LandSlope': {'Sev': 1, 'Mod': 2, 'Gtl': 3},
'ExterQual': {'Po' : 1, 'Fa' : 2, 'TA' : 3, 'Gd' : 4, 'Ex' : 5},
'BsmtQual' : {'None':0, 'Po' : 1, 'Fa' : 2, 'TA' : 3, 'Gd' : 4, 'Ex' : 5},
'BsmtCond' : {'None':0, 'Po' : 1, 'Fa' : 2, 'TA' : 3, 'Gd' : 4, 'Ex' : 5},
'BsmtExposure': {'None':0, 'No' : 1, 'Mn' : 2, 'Av' : 3, 'Gd' : 4},
'BsmtFinType1': {'None':0, 'Unf': 1, 'LwQ': 2, 'Rec': 3, 'BLQ': 4, 'ALQ': 5, 'GLQ': 6},
'BsmtFinType2': {'None':0, 'Unf': 1, 'LwQ': 2, 'Rec': 3, 'BLQ': 4, 'ALQ': 5, 'GLQ': 6},
'HeatingQC' : {'Po' : 1, 'Fa' : 2, 'TA' : 3, 'Gd' : 4, 'Ex' : 5},
'KitchenQual' : {'Po' : 1, 'Fa' : 2, 'TA' : 3, 'Gd' : 4, 'Ex' : 5},
'Functional': {'Sal': 1, 'Sev': 2, 'Maj2': 3, 'Maj1': 4, 'Mod': 5, 'Min2': 6, 'Min1': 7, 'Typ': 8},
'FireplaceQu': {'None':0, 'Po' : 1, 'Fa' : 2, 'TA' : 3, 'Gd' : 4, 'Ex' : 5},
'GarageFinish': {'None':0, 'Unf': 1, 'RFn': 2, 'Fin': 3},
'GarageQual': {'None':0, 'Po' : 1, 'Fa' : 2, 'TA' : 3, 'Gd' : 4},
'BsmtCond' : {'None':0, 'Po' : 1, 'Fa' : 2, 'TA' : 3, 'Gd' : 4, 'Ex' : 5},
'PavedDrive' : {'N' : 0, 'P' : 1, 'Y' : 2},
'PoolQC' : {'None':0, 'Fa' : 1, 'TA' : 2, 'Gd' : 3, 'Ex' : 4},
} )<feature_engineering>
model_analysis("Model with upsampled data",X_upsample_train,y_upsample_train,X_upsample_test,y_upsample_test )
train_cleaned["TotalSF"] = train_cleaned["GrLivArea"] + train_cleaned["TotalBsmtSF"]
test_cleaned["TotalSF"] = test_cleaned["GrLivArea"] + test_cleaned["TotalBsmtSF"]
train_cleaned["FlrsSF"] = train_cleaned["1stFlrSF"] + train_cleaned["2ndFlrSF"]
test_cleaned["FlrsSF"] = test_cleaned["1stFlrSF"] + test_cleaned["2ndFlrSF"]
train_cleaned['QualScore'] =(train_cleaned['TotalSF'] / max(train_cleaned['TotalSF'])) * 10 + train_cleaned['OverallQual']
test_cleaned['QualScore'] =(test_cleaned['TotalSF'] / max(train_cleaned['TotalSF'])) * 10 + test_cleaned['OverallQual']
train_cleaned['BedBath'] = train_cleaned['BedroomAbvGr'] + train_cleaned['FullBath'] + 0.5*train_cleaned['HalfBath']
test_cleaned['BedBath'] = test_cleaned['BedroomAbvGr'] + test_cleaned['FullBath'] + 0.5*test_cleaned['HalfBath'] 
<sort_values>
model_analysis("Model with pca data",X_pca_train,y_pca_train,X_pca_test,y_pca_test )
corr = train_cleaned.corr()
corr.sort_values(["SalePrice"], ascending = False, inplace = True)
 print(corr.SalePrice )<count_unique_values>
final_model = RandomForestClassifier(class_weight='balanced',random_state=42 )
s =(train_cleaned.dtypes == 'object')
 object_cols = list(s[s].index)
 object_cols
object_nunique = list(map(lambda col: train_cleaned[col].nunique() , object_cols))
d = dict(zip(object_cols, object_nunique))
sorted(d.items() , key=lambda x: x[1] )<categorify>
final_model.fit(X_upsample_train,y_upsample_train )
cols_to_onehot = [col for col in object_cols if train_cleaned[col].nunique() <10]
cols_to_label = [col for col in object_cols if train_cleaned[col].nunique() >=10]
print(f"The following features will be one-hot encoded:
{cols_to_onehot}")
 print("-"*120)
 print(f"The following features will be label-encoded:
{cols_to_label}" )<categorify>
probs = final_model.predict_proba(test_data_hv_co )
alldata = pd.concat([train_cleaned.iloc[:,:-1], test_cleaned])
 label_encoder = LabelEncoder()
for col in cols_to_label:
label_encoder.fit(alldata[col])
 train_cleaned[col] = label_encoder.transform(train_cleaned[col])
 test_cleaned[col] = label_encoder.transform(test_cleaned[col])
 OH_encode = OneHotEncoder(handle_unknown='ignore', sparse= False)
 OH_encode.fit(alldata[cols_to_onehot])
 OH_train = pd.DataFrame(OH_encode.transform(train_cleaned[cols_to_onehot]))
OH_test = pd.DataFrame(OH_encode.transform(test_cleaned[cols_to_onehot]))
OH_train.index = train_cleaned.index
OH_test.index = test_cleaned.index
train_encoded = pd.concat([train_cleaned, OH_train], axis=1)
 test_encoded = pd.concat([test_cleaned, OH_test], axis=1)
 train_encoded = train_encoded.drop(cols_to_onehot, axis=1)
 test_encoded = test_encoded.drop(cols_to_onehot, axis=1 )<train_model>
submission = pd.DataFrame({"ID":test_data_hv_co.ID, "TARGET": probs[:,1]} )
<choose_model_class>
submission.to_csv("santander_solution.csv" )
my_model = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.0473, n_estimators=576,
max_bin = 95, bagging_fraction = 0.8195,
bagging_freq = 5, feature_fraction = 0.1742,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =9, min_sum_hessian_in_leaf = 11.58)
<compute_test_metric>
xgb_classifier = XGBClassifier(missing=np.nan, max_depth=6,
n_estimators=350, learning_rate=0.025, nthread=4, subsample=0.95,
colsample_bytree=0.85, seed=4242)
 xgb_mdl = CalibratedClassifierCV(xgb_classifier, method='isotonic', cv=10)
 xgb_mdl.fit(X_upsample_train,y_upsample_train )
X = train_encoded.drop(['SalePrice'], axis=1)
 y = train_encoded.SalePrice
model_pipeline = Pipeline(steps=[ 
('model', my_model)
 ])
 scores = np.sqrt(-1*cross_val_score(model_pipeline, X, y, cv=5, scoring = 'neg_mean_squared_error'))
print(f"RMSE scores:
{scores}")
 print("-"*60)
 print(f"Average RMSE score:
{scores.mean() }" )<save_to_csv>
probs_xgb = xgb_mdl.predict_proba(test_data_hv_co )
my_model.fit(X, y)
 preds_test = np.expm1(my_model.predict(test_encoded))
output = pd.DataFrame({'Id': test_ID,
'SalePrice': preds_test})
 output.to_csv('submission_ML.csv', index=False )<set_options>
submission1 = pd.DataFrame({"ID":test_data_hv_co.ID, "TARGET": probs_xgb[:,1]} )
def random_seed(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
random_seed(0)
 dep_var = 'SalePrice'
procs = [Categorify, Normalize]
dataset_cleaned = train_cleaned.drop(['SalePrice'], axis=1)
 cont_names = list(dataset_cleaned.select_dtypes(exclude = ['object', 'bool'] ).columns)
 cat_names = list(dataset_cleaned.select_dtypes(include = ['object', 'bool'] ).columns)
 test = TabularList.from_df(test_cleaned, cont_names=cont_names, cat_names=cat_names, procs=procs)
 data =(TabularList.from_df(train_cleaned, path='.', cont_names=cont_names, cat_names=cat_names, procs=procs)
.split_by_rand_pct(valid_pct = 0.2)
.label_from_df(cols = dep_var, label_cls = FloatList, log = False)
.add_test(test)
.databunch(bs = 128, num_workers=0))
data.show_batch(rows=5, ds_type=DatasetType.Valid )<train_model>
submission1.to_csv("submission_xgb.csv",index=False )
<save_to_csv><EOS> 
submission1.to_csv("/kaggle/working/submission.csv",index=False )