Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<train_model>
os.environ['TF_CPP_MIN_LOG_LEVEL'] = "99"
def to_img_shape(X, y=[]):
"Format matrix of rows, Nx784 to Nx28x28x3"
X = np.array(X ).reshape(-1,28,28)
 X = np.stack(( X,)*3, axis=-1)
 y = np.array(y)
 return X, y
def save_imgs(path:Path, data, labels=[]):

 path.mkdir(parents=True, exist_ok=True)
 for label in np.unique(labels): 
(path / str(label)).mkdir(parents=True, exist_ok=True)
 for i in range(len(data)) :
if(len(labels)!= 0):
imageio.imsave(str(path / str(labels[i])/(str(i)+ '.jpg')) , data[i])
 else:
imageio.imsave(str(path / f'testimg_{str(i ).zfill(5)}.jpg'), data[i] )<categorify>
device_lib.list_local_devices()
X_train, y_train = to_img_shape(X_train, y_train)
 X_valid, y_valid = to_img_shape(X_valid, y_valid)
 X_test, _ = to_img_shape(test )<train_model>
tf.logging.set_verbosity(tf.logging.INFO)
 sess = tf.InteractiveSession()
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 print('Shape of the train data with all features:', train.shape)
 train = train.select_dtypes(exclude=['object'])
 print("")
 print('Shape of the train data with numerical features:', train.shape)
 train.drop('Id',axis = 1, inplace = True)
 train.fillna(0,inplace=True )
%%time
save_imgs(Path('/kaggle/working/data/train'), X_train, y_train )<train_model>
test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 test = test.select_dtypes(exclude=['object'])
 ID = test.Id
test.fillna(0,inplace=True)
 test.drop('Id',axis = 1, inplace = True)
 print("")
 print("List of features contained our dataset:",list(train.columns))
%%time
save_imgs(Path('/kaggle/working/data/valid'), X_valid, y_valid )<train_model>
warnings.filterwarnings('ignore')
 clf = IsolationForest(max_samples = 100, random_state = 42)
 clf.fit(train)
 y_noano = clf.predict(train)
 y_noano = pd.DataFrame(y_noano, columns = ['Top'])
 y_noano[y_noano['Top'] == 1].index.values
train = train.iloc[y_noano[y_noano['Top'] == 1].index.values]
train.reset_index(drop = True, inplace = True)
 print("Number of Outliers:", y_noano[y_noano['Top'] == -1].shape[0])
 print("Number of rows without outliers:", train.shape[0] )
%%time
save_imgs(Path('/kaggle/working/data/test'), X_test )<load_pretrained>
col_train = list(train.columns)
 col_train_bis = list(train.columns)
 col_train_bis.remove('SalePrice')
 mat_train = np.matrix(train)
 mat_test = np.matrix(test)
 mat_new = np.matrix(train.drop('SalePrice',axis = 1))
mat_y = np.array(train.SalePrice ).reshape(( 1314,1))
prepro_y = MinMaxScaler()
prepro_y.fit(mat_y)
 prepro = MinMaxScaler()
prepro.fit(mat_train)
 prepro_test = MinMaxScaler()
prepro_test.fit(mat_new)
 train = pd.DataFrame(prepro.transform(mat_train),columns = col_train)
 test = pd.DataFrame(prepro_test.transform(mat_test),columns = col_train_bis)
 train.head()
path = Path('/kaggle/working/data')
 image_list =(ImageList.from_folder(path)
.split_by_folder() 
.label_from_folder())
 data =(image_list.databunch(bs=1)
.normalize(imagenet_stats))<feature_engineering>
COLUMNS = col_train
FEATURES = col_train_bis
LABEL = "SalePrice"
feature_cols = [tf.contrib.layers.real_valued_column(k)for k in FEATURES]
training_set = train[COLUMNS]
prediction_set = train.SalePrice
x_train, x_test, y_train, y_test = train_test_split(training_set[FEATURES] , prediction_set, test_size=0.33, random_state=42)
 y_train = pd.DataFrame(y_train, columns = [LABEL])
 training_set = pd.DataFrame(x_train, columns = FEATURES ).merge(y_train, left_index = True, right_index = True)
 training_set.head()
training_sub = training_set[col_train]
tfms = get_transforms()
tfms[0]<categorify>
y_test = pd.DataFrame(y_test, columns = [LABEL])
 testing_set = pd.DataFrame(x_test, columns = FEATURES ).merge(y_test, left_index = True, right_index = True)
 testing_set.head()
tfms[1]<feature_engineering>
tf.logging.set_verbosity(tf.logging.ERROR)
 regressor = tf.contrib.learn.DNNRegressor(feature_columns=feature_cols,
activation_fn = tf.nn.relu, hidden_units=[256, 128, 64, 32, 16])
 training_set.reset_index(drop = True, inplace =True )
tfms = get_transforms(do_flip=False)
 tfms[0]<load_pretrained>
def input_fn(data_set, pred = False):
if pred == False:
feature_cols = {k: tf.constant(data_set[k].values)for k in FEATURES}
labels = tf.constant(data_set[LABEL].values)
 return feature_cols, labels
if pred == True:
feature_cols = {k: tf.constant(data_set[k].values)for k in FEATURES}
return feature_cols
image_list =(image_list.transform(get_transforms(do_flip=False), size=28)
.add_test(ItemList.from_folder(path=path/"test"), label=None))
data =(image_list.databunch(bs=256)
.normalize(imagenet_stats))<choose_model_class>
regressor.fit(input_fn=lambda: input_fn(training_set), steps=2000 )
learn = cnn_learner(data, models.resnet18, metrics=accuracy )<find_best_params>
ev = regressor.evaluate(input_fn=lambda: input_fn(testing_set), steps=1 )
learn.model<find_best_params>
loss_score1 = ev["loss"]
print("Final Loss on the testing set: {0:f}".format(loss_score1))
learn.lr_find()<train_model>
y = regressor.predict(input_fn=lambda: input_fn(testing_set))
predictions = list(itertools.islice(y, testing_set.shape[0]))
learn.fit_one_cycle(5, 0.02 )<save_model>
def leaky_relu(x):
return tf.nn.relu(x)- 0.01 * tf.nn.relu(-x )
learn.save('stage-1' )<load_pretrained>
regressor = tf.contrib.learn.DNNRegressor(feature_columns=feature_cols,
activation_fn = leaky_relu, hidden_units=[256, 128, 64, 32, 16])
 regressor.fit(input_fn=lambda: input_fn(training_set), steps=2000)
 ev = regressor.evaluate(input_fn=lambda: input_fn(testing_set), steps=1)
 loss_score2 = ev["loss"]
print("Final Loss on the testing set with Leaky Relu: {0:f}".format(loss_score2))
y_predict = regressor.predict(input_fn=lambda: input_fn(test, pred = True))
learn.load('stage-1');<find_best_params>
regressor = tf.contrib.learn.DNNRegressor(feature_columns=feature_cols,
activation_fn = tf.nn.elu, hidden_units=[200, 100, 50, 25, 12])
 regressor.fit(input_fn=lambda: input_fn(training_set), steps=2000)
 ev = regressor.evaluate(input_fn=lambda: input_fn(testing_set), steps=1)
 loss_score3 = ev["loss"]
learn.unfreeze()
learn.lr_find()<train_model>
print("Final Loss on the testing set with Elu: {0:f}".format(loss_score3))
y_predict = regressor.predict(input_fn=lambda: input_fn(test, pred = True))
%%time
learn.fit_one_cycle(35, max_lr=slice(1e-5, 0.02/10))<save_model>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 train.drop('Id',axis = 1, inplace = True)
 train_numerical = train.select_dtypes(exclude=['object'])
 train_numerical.fillna(0,inplace = True)
 train_categoric = train.select_dtypes(include=['object'])
 train_categoric.fillna('NONE',inplace = True)
 train = train_numerical.merge(train_categoric, left_index = True, right_index = True)
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 ID = test.Id
test.drop('Id',axis = 1, inplace = True)
 test_numerical = test.select_dtypes(exclude=['object'])
 test_numerical.fillna(0,inplace = True)
 test_categoric = test.select_dtypes(include=['object'])
 test_categoric.fillna('NONE',inplace = True)
 test = test_numerical.merge(test_categoric, left_index = True, right_index = True )
learn.save('stage-2' )<choose_model_class>
clf = IsolationForest(max_samples = 100, random_state = 42)
 clf.fit(train_numerical)
 y_noano = clf.predict(train_numerical)
 y_noano = pd.DataFrame(y_noano, columns = ['Top'])
 y_noano[y_noano['Top'] == 1].index.values
train_numerical = train_numerical.iloc[y_noano[y_noano['Top'] == 1].index.values]
train_numerical.reset_index(drop = True, inplace = True)
 train_categoric = train_categoric.iloc[y_noano[y_noano['Top'] == 1].index.values]
train_categoric.reset_index(drop = True, inplace = True)
 train = train.iloc[y_noano[y_noano['Top'] == 1].index.values]
train.reset_index(drop = True, inplace = True )
interp = ClassificationInterpretation.from_learner(learn )<define_variables>
col_train_num = list(train_numerical.columns)
 col_train_num_bis = list(train_numerical.columns)
 col_train_cat = list(train_categoric.columns)
 col_train_num_bis.remove('SalePrice')
 mat_train = np.matrix(train_numerical)
 mat_test = np.matrix(test_numerical)
 mat_new = np.matrix(train_numerical.drop('SalePrice',axis = 1))
mat_y = np.array(train.SalePrice)
 prepro_y = MinMaxScaler()
prepro_y.fit(mat_y.reshape(1314,1))
prepro = MinMaxScaler()
prepro.fit(mat_train)
 prepro_test = MinMaxScaler()
prepro_test.fit(mat_new)
 train_num_scale = pd.DataFrame(prepro.transform(mat_train),columns = col_train)
 test_num_scale = pd.DataFrame(prepro_test.transform(mat_test),columns = col_train_bis )
def make_submission_file(
 learner,
filename=f'submission_{datetime.datetime.now().strftime("%Y%m%d_%H%M%S")}.csv',
preds=None 
):

 if preds is None:
preds, _ = learner.get_preds(ds_type=DatasetType.Test)
 preds = np.argmax(preds, 1)
 test_index = []
num = len(learn.data.test_ds)
 for i in range(num):
test_index.append(str(learner.data.test_ds.items[i] ).split('/')[-1])
 df =(pd.DataFrame(data={"Label": preds, "Filename": test_index})
.sort_values(by='Filename')
.drop('Filename', axis=1)
.assign(ImageId = range(1, len(preds)+ 1)) 
.reset_index(drop=True)
.reindex(columns=['ImageId', 'Label']))
df.to_csv(filename, index=False)
 print(f'Saved predictions as {filename}.' )<define_variables>
train[col_train_num] = pd.DataFrame(prepro.transform(mat_train),columns = col_train_num)
 test[col_train_num_bis] = test_num_scale
make_submission_file(learn, filename="resnet18-fine-tuned.csv" )<create_dataframe>
COLUMNS = col_train_num
FEATURES = col_train_num_bis
LABEL = "SalePrice"
FEATURES_CAT = col_train_cat
engineered_features = []
for continuous_feature in FEATURES:
engineered_features.append(
 tf.contrib.layers.real_valued_column(continuous_feature))
for categorical_feature in FEATURES_CAT:
sparse_column = tf.contrib.layers.sparse_column_with_hash_bucket(
 categorical_feature, hash_bucket_size=1000)
 engineered_features.append(tf.contrib.layers.embedding_column(sparse_id_column=sparse_column, dimension=16,combiner="sum"))
training_set = train[FEATURES + FEATURES_CAT]
prediction_set = train.SalePrice
x_train, x_test, y_train, y_test = train_test_split(training_set[FEATURES + FEATURES_CAT] ,
prediction_set, test_size=0.20, random_state=42)
 y_train = pd.DataFrame(y_train, columns = [LABEL])
 training_set = pd.DataFrame(x_train, columns = FEATURES + FEATURES_CAT ).merge(y_train, left_index = True, right_index = True)
 training_sub = training_set[FEATURES + FEATURES_CAT]
testing_sub = test[FEATURES + FEATURES_CAT]
most_unsure = DatasetFormatter.from_most_unsure(learn )<compute_test_metric>
y_test = pd.DataFrame(y_test, columns = [LABEL])
 testing_set = pd.DataFrame(x_test, columns = FEATURES + FEATURES_CAT ).merge(y_test, left_index = True, right_index = True )
err1 = 1 - 0.99442
err2 = 1 - 0.99571
print(f'Human in the loop improvement: {100*(err1-err2)/err1}%' )<load_from_csv>
training_set[FEATURES_CAT] = training_set[FEATURES_CAT].applymap(str)
 testing_set[FEATURES_CAT] = testing_set[FEATURES_CAT].applymap(str)
 def input_fn_new(data_set, training = True):
continuous_cols = {k: tf.constant(data_set[k].values)for k in FEATURES}
categorical_cols = {k: tf.SparseTensor(
 indices=[[i, 0] for i in range(data_set[k].size)], values = data_set[k].values, dense_shape = [data_set[k].size, 1])for k in FEATURES_CAT}
feature_cols = dict(list(continuous_cols.items())+ list(categorical_cols.items()))
if training == True:
label = tf.constant(data_set[LABEL].values)
 return feature_cols, label
return feature_cols
regressor = tf.contrib.learn.DNNRegressor(feature_columns = engineered_features,
activation_fn =leaky_relu, hidden_units=[256, 128, 64, 32, 16])
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 print(train_df.shape, test_df.shape )<count_values>
categorical_cols = {k: tf.SparseTensor(indices=[[i, 0] for i in range(training_set[k].size)], values = training_set[k].values, dense_shape = [training_set[k].size, 1])for k in FEATURES_CAT}
train_df['label'].value_counts(sort=False )<prepare_x_and_y>
regressor.fit(input_fn = lambda: input_fn_new(training_set), steps=2000 )
train_X = train_df.drop(['label'], axis=1 ).values
train_Y = train_df['label'].values
test_X = test_df.values
print(train_X.shape, train_Y.shape, test_X.shape )<categorify>
ev = regressor.evaluate(input_fn=lambda: input_fn_new(testing_set, training = True), steps=1)
 loss_score4 = ev["loss"]
print("Final Loss on the testing set: {0:f}".format(loss_score4))
n_x = 28
train_X_digit = train_X.reshape(( -1, n_x, n_x, 1))
test_X_digit = test_X.reshape(( -1, n_x, n_x, 1))
print(train_X_digit.shape, test_X_digit.shape)
 train_X_digit = train_X_digit / 255.
 test_X_digit = test_X_digit / 255.
 onehot_labels = to_categorical(train_Y)
 print(onehot_labels.shape)
 print(train_Y[181], onehot_labels[181])
 plt.figure(figsize=(1,1))
plt.imshow(train_X[181].reshape(( 28,28)) ,cmap=plt.cm.binary)
 plt.show()<set_options>
regressor = tf.contrib.learn.DNNRegressor(feature_columns = engineered_features,
activation_fn = tf.nn.relu, hidden_units=[2500] )
data_augment = ImageDataGenerator(rotation_range=10, zoom_range=0.1,
width_shift_range=0.1, height_shift_range=0.1 )<choose_model_class>
regressor.fit(input_fn = lambda: input_fn_new(training_set), steps=2000 )
model = models.Sequential()
model.add(layers.Conv2D(32, kernel_size=5, padding='same', activation='relu', input_shape=(28, 28, 1)))
 model.add(layers.MaxPooling2D(pool_size=(2,2)))
 model.add(layers.Dropout(rate=0.4))
model.add(layers.Conv2D(64, kernel_size=5, activation='relu'))
model.add(layers.MaxPooling2D(pool_size=(2,2)))
 model.add(layers.Dropout(rate=0.4))
model.add(layers.Conv2D(128, kernel_size=3, activation='relu'))
model.add(layers.Flatten())
 model.add(layers.Dense(256, activation='relu'))
model.add(layers.Dropout(rate=0.4))
model.add(layers.Dense(10, activation='softmax'))
model.summary()<choose_model_class>
ev = regressor.evaluate(input_fn=lambda: input_fn_new(testing_set, training = True), steps=1)
 loss_score5 = ev["loss"]
model.compile(optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'] )<choose_model_class>
print("Final Loss on the testing set: {0:f}".format(loss_score5))
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy',patience=3,factor=0.5,min_lr=0.00001,
verbose=1 )<categorify>
list_score = [loss_score1, loss_score2, loss_score3, loss_score4,loss_score5]
list_model = ['Relu_cont', 'LRelu_cont', 'Elu_cont', 'Relu_cont_categ','Shallow_1ku']
X_dev = train_X_digit[:5000]
rem_X_train = train_X_digit[5000:]
print(X_dev.shape, rem_X_train.shape)
 Y_dev = onehot_labels[:5000]
rem_Y_train = onehot_labels[5000:]
print(Y_dev.shape, rem_Y_train.shape )<train_model>
def to_submit(pred_y,name_out):
y_predict = list(itertools.islice(pred_y, test.shape[0]))
y_predict = pd.DataFrame(prepro_y.inverse_transform(np.array(y_predict ).reshape(len(y_predict),1)) , columns = ['SalePrice'])
 y_predict = y_predict.join(ID)
 y_predict.to_csv(name_out + '.csv',index=False )
<predict_on_test><EOS> 
y_predict = regressor.predict(input_fn=lambda: input_fn_new(testing_sub, training = False))
to_submit(y_predict, "submission_shallow" )