Kaggle competition: CareerCon 2019 - Help Navigate Robots
score = rmse_cross_val(gbr)
 print("gbr: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['gbr'] =(score.mean() , score.std() )<predict_on_test>
PATH = '.. /input/career-con-2019'
def blended_predictions(xtrain):
return(( 0.2 * ridge_data.predict(xtrain)) + \ 
(0.15 * svr_data.predict(xtrain)) + \ 
(0.17 * gbr_data.predict(xtrain)) + \ 
(0.1 * xgb_data.predict(xtrain)) + \ 
(0.03 * rf_data.predict(xtrain)) + \ 
(0.35 * stack_reg_model.predict(np.array(xtrain))))<compute_test_metric>
gc.enable()
train_x = pd.read_csv(PATH+'/X_train.csv')
 train_y = pd.read_csv(PATH+'/y_train.csv' )
blended_score = rmsle(train_labels, blended_predictions(xtrain))
scores['blended'] =(blended_score, 0)
 print('RMSE score on train data:')
 print(blended_score )<predict_on_test>
def prepare_data(t):
def f(d):
d=d.sort_values(by=['measurement_number'])
 return pd.DataFrame({
'lx':[ d['linear_acceleration_X'].values ],
'ly':[ d['linear_acceleration_Y'].values ],
'lz':[ d['linear_acceleration_Z'].values ],
'ax':[ d['angular_velocity_X'].values ],
'ay':[ d['angular_velocity_Y'].values ],
'az':[ d['angular_velocity_Z'].values ],
})
 t= t.groupby('series_id' ).apply(f)
 def mfft(x):
return [ x/math.sqrt(128.0)for x in np.absolute(np.fft.fft(x)) ][1:65]
t['lx_f']=[ mfft(x)for x in t['lx'].values ]
t['ly_f']=[ mfft(x)for x in t['ly'].values ]
t['lz_f']=[ mfft(x)for x in t['lz'].values ]
t['ax_f']=[ mfft(x)for x in t['ax'].values ]
t['ay_f']=[ mfft(x)for x in t['ay'].values ]
t['az_f']=[ mfft(x)for x in t['az'].values ]
return t
t=prepare_data(train_x)
 t=pd.merge(t,train_y[['series_id','surface','group_id']],on='series_id')
 t=t.rename(columns={"surface": "y"})
 def aggf(d, feature):
va= np.array(d[feature].tolist())
 mean= sum(va)/va.shape[0]
var= sum([(va[i,:]-mean)**2 for i in range(va.shape[0])])/va.shape[0]
dev= [ math.sqrt(x)for x in var ]
return pd.DataFrame({
'mean': [ mean ],
'dev' : [ dev ],
})
 display={
'hard_tiles_large_space':'r-.',
'concrete':'g-.',
'tiled':'b-.',
'fine_concrete':'r-',
'wood':'g-',
'carpet':'b-',
'soft_pvc':'y-',
'hard_tiles':'r--',
'soft_tiles':'g--',
}
plt.figure(figsize=(14, 8*7))
features=['lx_f','ly_f','lz_f','ax_f','ay_f','az_f']
count=0
for feature in features:
stat= t.groupby('y' ).apply(aggf,feature)
 stat.index= stat.index.droplevel(-1)
 b=[*range(len(stat.at['carpet','mean'])) ]
count+=1
plt.subplot(len(features)+1,1,count)
 for i,(k,v)in enumerate(display.items()):
plt.plot(b, stat.at[k,'mean'], v, label=k)
 leg = plt.legend(loc='best', ncol=3, mode="expand", shadow=True, fancybox=True)
 plt.title("sensor: " + feature)
 plt.xlabel("frequency component")
 plt.ylabel("amplitude")
 count+=1
plt.subplot(len(features)+1,1,count)
 k='concrete'
v=display[k]
feature='lz_f'
stat= t.groupby('y' ).apply(aggf,feature)
 stat.index= stat.index.droplevel(-1)
 b=[*range(len(stat.at['carpet','mean'])) ]
plt.errorbar(b, stat.at[k,'mean'], yerr=stat.at[k,'dev'], fmt=v)
 plt.title("sample for error bars(lz_f, surface concrete)")
 plt.xlabel("frequency component")
 plt.ylabel("amplitude")
 plt.show()
sample.iloc[:,1] = np.floor(np.expm1(blended_predictions(xtest)) )<save_to_csv>
del train_x, train_y
gc.collect()
sample.to_csv("my_submission.csv", index=False )<import_modules>
import os
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import itertools
import math
from sklearn.metrics import *
from sklearn.model_selection import *
import keras
from keras.layers import *
from keras.callbacks import *
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelEncoder
from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from statsmodels.stats.outliers_influence import variance_inflation_factor
from scipy import stats
from scipy.stats import skew, boxcox_normmax
from scipy.special import boxcox1p
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
import xgboost as xgb
from xgboost.sklearn import XGBRegressor
from xgboost import plot_importance
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import RandomForestRegressor
import lightgbm as lgb
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import RidgeCV
from sklearn.linear_model import LassoCV
from sklearn.linear_model import ElasticNetCV
from sklearn.svm import SVR
from mlxtend.regressor import StackingCVRegressor
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn import metrics
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
import math
from math import sqrt
from sklearn.metrics import r2_score<set_options>
X_train = pd.read_csv(PATH+"/X_train.csv")
 X_test = pd.read_csv(PATH+"/X_test.csv")
 y_train = pd.read_csv(PATH+"/y_train.csv")
 sub = pd.read_csv(PATH+"/sample_submission.csv" )
if not sys.warnoptions:
warnings.simplefilter("ignore" )<load_from_csv>
print("Train: measurements = %d , features = %d" % X_train.shape)
 X_train.head()
house_data = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 house_data_test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )<filter>
X_train.drop(['row_id', "series_id", "measurement_number"], axis=1, inplace=True)
 X_train = X_train.values.reshape(( 3810, 128, 10))
X_test.drop(['row_id', "series_id", "measurement_number"], axis=1, inplace=True)
 X_test = X_test.values.reshape(( 3816, 128, 10))
house_data[(house_data['GrLivArea']>5000)&(house_data['SalePrice']<200000)]<drop_column>
encode_dic = {'fine_concrete': 0,
'concrete': 1,
'soft_tiles': 2,
'tiled': 3,
'soft_pvc': 4,
'hard_tiles_large_space': 5,
'carpet': 6,
'hard_tiles': 7,
'wood': 8}
decode_dic = {0: 'fine_concrete',
1: 'concrete',
2: 'soft_tiles',
3: 'tiled',
4: 'soft_pvc',
5: 'hard_tiles_large_space',
6: 'carpet',
7: 'hard_tiles',
8: 'wood'}
y_train = y_train['surface'].map(encode_dic ).astype(int )
house_data = house_data.drop(house_data[(house_data['GrLivArea']>5000)&(house_data['SalePrice']<200000)].index )<concatenate>
class Attention(Layer):
def __init__(self, step_dim, W_regularizer=None, b_regularizer=None, W_constraint=None, b_constraint=None, bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],), initializer=self.init, name='{}_W'.format(self.name), regularizer=self.W_regularizer, constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],), initializer='zero', name='{}_b'.format(self.name), regularizer=self.b_regularizer, constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) , K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias: eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None: a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
house_all_data = pd.concat([house_data, house_data_test], axis=0)
 house_all_data[:1459]<count_missing_values>
def make_model() :
inp = Input(shape=(128, 10))
x = Bidirectional(CuDNNLSTM(32, return_sequences=True))(inp)
 x = Attention(128 )(x)
 x = Dense(9, activation="softmax" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
null_list = house_all_data.isnull().sum()
def find_null_values(list):
for name, value in list.items() :
if value != 0:
print(name, value)
 print(find_null_values(null_list))<data_type_conversions>
def k_folds(X, y, X_test, k=5):
folds = list(StratifiedKFold(n_splits=k ).split(X, y))
y_test = np.zeros(( X_test.shape[0], 9))
y_oof = np.zeros(( X.shape[0]))
for i,(train_idx, val_idx)in enumerate(folds):
print(f"Fold {i+1}")
 model = make_model()
model.fit(X[train_idx], y[train_idx], batch_size=64, epochs=75,
validation_data=[X[val_idx], y[val_idx]], verbose=0)
 pred_val = np.argmax(model.predict(X[val_idx]), axis=1)
 score = accuracy_score(pred_val, y[val_idx])
 y_oof[val_idx] = pred_val
print(f'Scored {score:.3f} on validation data')
 y_test += model.predict(X_test)
 return y_oof, y_test
house_all_data['LotFrontage'] = house_all_data.groupby('Neighborhood')['LotFrontage'].transform(
 lambda x: x.fillna(x.median()))
for col in('MasVnrArea', 'GarageYrBlt'):
house_all_data[col] = house_all_data[col].fillna(0)
 for col in('LotFrontage', 'MasVnrArea', 'GarageYrBlt'):
house_all_data[col] = house_all_data[col].astype(int)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF','BsmtFullBath', 'BsmtHalfBath',
'GarageCars','GarageArea'):
house_all_data[col] = house_all_data[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure','BsmtFinType1','BsmtFinType2', 'FireplaceQu','GarageType',
'GarageFinish','GarageQual','GarageCond', 'Fence', 'Alley', 'PoolQC', 'MiscFeature'):
house_all_data[col] = house_all_data[col].fillna('not_applicable' )<count_values>
y_oof, y_test = k_folds(X_train, y_train, X_test, k=5 )
house_all_data['Utilities'].value_counts()<categorify>
print(f'Local CV is {accuracy_score(y_oof, y_train):.4f}' )
for col in('MSZoning', 'Exterior1st', 'Exterior2nd','MasVnrType','Electrical', 'KitchenQual',
'Functional','SaleType'):
house_all_data[col] = house_all_data[col].fillna(house_all_data[col].mode() [0] )<feature_engineering>
y_test = np.argmax(y_test, axis=1)
 sub['surface'] = y_test
sub['surface'] = sub['surface'].map(decode_dic)
 sub.head()
sub.to_csv('submission.csv', index=False )
house_all_data['Total_SF'] = house_all_data['TotalBsmtSF'] + house_all_data['1stFlrSF'] + house_all_data['2ndFlrSF']
house_all_data['Additional_Area'] = house_all_data['GarageArea'] + house_all_data['PoolArea']
house_all_data['Total_Bathroom'] = house_all_data['BsmtFullBath'] + house_all_data['BsmtHalfBath'] + \
house_all_data['FullBath'] + house_all_data['HalfBath']
house_all_data['Total_PorchSF'] = house_all_data['OpenPorchSF'] + house_all_data['EnclosedPorch'] + \
house_all_data['3SsnPorch'] + house_all_data['ScreenPorch'] + \
house_all_data['WoodDeckSF']<feature_engineering>
gc.collect()
house_all_data['remodel_year'] = house_all_data['YearRemodAdd'] - house_all_data['YearBuilt']
house_all_data ['Remodel_yn'] = house_all_data['remodel_year'].apply(lambda x: 1 if x > 0 else 0)
 house_all_data['Fireplace_yn'] = house_all_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 house_all_data['Pool_yn'] = house_all_data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 house_all_data['Basement_yn'] = house_all_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 house_all_data['Garage_yn'] = house_all_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 house_all_data['Garage_car_yn'] = house_all_data['GarageCars'].apply(lambda x: 1 if x > 0 else 0)
 house_all_data['2ndfloor_yn'] = house_all_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0 )<filter>
%matplotlib inline
le = preprocessing.LabelEncoder()
warnings.filterwarnings('ignore')
 gc.enable()
house_all_data[house_all_data['remodel_year'] < 0]<feature_engineering>
PATH = ".. /input/career-con-2019"
house_all_data.loc[house_all_data['remodel_year'] == -1, 'remodel_year'] = 1<feature_engineering>
testing = True
X_train = pd.read_csv(PATH+'/X_train.csv')
 y_train = pd.read_csv(PATH+'/y_train.csv')
 X_train = pd.merge(X_train,y_train,on='series_id')
 X_train['surface'] = le.fit_transform(X_train['surface'])
 if(testing):
X_test = pd.read_csv(PATH+'/X_test.csv')
 X_test['series_id'] = X_test['series_id']+3810
X_test['group_id'] = 0
X_test['surface'] = 0
frames = [X_train,X_test]
X_train = pd.concat(frames)
 X_train.reset_index(drop=True,inplace=True )
house_all_data.loc[house_all_data['Id'] == 1877, 'Remodel_yn'] = 1<filter>
cols = list(X_train.columns.values)
 cols.remove('orientation_W')
 cols.insert(3,'orientation_W')
 X_train = X_train[cols]
print("New train data")
 X_train.head()
house_all_data.loc[416]<drop_column>
num_meas = 128
num_series = X_train['series_id'].nunique()
print("Number of measurements = ", num_meas)
 print("Number of total sereis = ", num_series )
house_all_data = house_all_data.drop([ 'Id', 'Utilities','PoolQC', 'Heating', 'RoofMatl', 'Condition2', 'Street'], axis=1 )<data_type_conversions>
def q_to_angle(q_val):
roll = np.arctan2(2*(q_val[0]*q_val[1] + q_val[2]*q_val[3]),1 - 2*(q_val[1]*q_val[1] + q_val[2]*q_val[2]))
pitch = np.arcsin(2*(q_val[0]*q_val[2] - q_val[3]*q_val[1]))
yaw = np.arctan2(2*(q_val[0]*q_val[3] + q_val[1]*q_val[2]),1 - 2*(q_val[2]*q_val[2] + q_val[3]*q_val[3]))
return np.array([roll, pitch, yaw] )
for col in ['MSSubClass', 'OverallCond', 'YrSold', 'MoSold']:
house_all_data[col] = house_all_data[col].astype('str' )<categorify>
quat_arr = np.array(X_train[['orientation_W','orientation_X','orientation_Y','orientation_Z']])
 euler_arr = np.zeros([quat_arr.shape[0],3])
 for n,arr in enumerate(quat_arr):
euler_arr[n] = q_to_angle(arr )
label_encoding_feat =('ExterQual', 'ExterCond', 'BsmtQual','BsmtCond', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2','HeatingQC', 'KitchenQual', 'FireplaceQu', 'GarageQual','GarageCond',
'LotShape', 'PavedDrive', 'CentralAir', 'MSSubClass', 'OverallCond', 'YrSold','MoSold',
'Remodel_yn','Fireplace_yn','Pool_yn','Basement_yn', 'Garage_yn', '2ndfloor_yn', 'Garage_car_yn')
 for feat in label_encoding_feat:
labelencoder = LabelEncoder()
house_all_data[feat] = labelencoder.fit_transform(( house_all_data[feat].values))<categorify>
X_train['roll'] = euler_arr[:,0]
X_train['pitch'] = euler_arr[:,1]
X_train['yaw'] = euler_arr[:,2]
def one_hot_encoding(dataset, feature):
dummy_dataset = pd.get_dummies(dataset[[feature]])
 encode_dataset = pd.concat([dataset, dummy_dataset], axis=1)
 encode_dataset = encode_dataset.drop([feature], axis=1)
 return(encode_dataset )<categorify>
cols = list(X_train.columns.values)
 cols.remove('group_id')
 cols.append('group_id')
 cols.remove('surface')
 cols.append('surface')
 X_train = X_train[cols]
print("Train updated")
 X_train.head()
encoding_features = ['MSZoning', 'LandContour', 'LotConfig','BldgType', 'HouseStyle','RoofStyle',
'MasVnrType', 'Foundation', 'Electrical','GarageType','SaleCondition', 'SaleType',
'Functional','GarageFinish', 'LandSlope', 'Condition1', 'Neighborhood','Exterior1st',
'Exterior2nd', 'Fence', 'Alley', 'MiscFeature' ]
for feature in encoding_features:
house_all_data = one_hot_encoding(house_all_data, feature)
<create_dataframe>
feat_cols = ['roll','pitch','yaw','angular_velocity_X','angular_velocity_Y','angular_velocity_Z','linear_acceleration_X','linear_acceleration_Y','linear_acceleration_Z']
feat_array = np.array(X_train[feat_cols])
 feat_array = np.reshape(feat_array,[num_series,128,len(feat_cols)])
 group_array = np.array(X_train['group_id'])
 group_array = np.reshape(group_array,[num_series,128])
 group_array = group_array[:,0]
target_array = np.array(X_train['surface'])
 target_array = np.reshape(target_array,[num_series,128])
 target_array = target_array[:,0]
house_numerical_data = house_all_data[['BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','BsmtFullBath',
'BsmtHalfBath','GarageCars','GarageArea','Total_SF','Additional_Area','LotFrontage',
'LotArea','OverallQual','YearBuilt','YearRemodAdd','MasVnrArea','1stFlrSF',
'2ndFlrSF','LowQualFinSF','GrLivArea','FullBath','HalfBath','BedroomAbvGr',
'KitchenAbvGr','TotRmsAbvGrd','Fireplaces','GarageYrBlt','WoodDeckSF',
'OpenPorchSF','EnclosedPorch', '3SsnPorch','ScreenPorch','PoolArea','MiscVal',
'Total_Bathroom','Total_PorchSF','remodel_year','SalePrice']]<normalization>
delta_cols = ['roll','pitch','yaw']
for dc in delta_cols:
iia = feat_cols.index(dc)
 np_arr = feat_array[:,:,iia]
roll_arr = np.copy(np_arr)
 roll_arr[:,1:] = roll_arr[:,:-1]
np_arr = np_arr - roll_arr
feat_array[:,:,iia] = np_arr
corr_mask_numerical = corr.mask(mask)
 corr_values_numerical = [c for c in corr_mask_numerical.columns if any(corr_mask_numerical[c] > 0.90)]
print(corr_values_numerical )<feature_engineering>
norm_cols = ['linear_acceleration_X','linear_acceleration_Y','linear_acceleration_Z','angular_velocity_X','angular_velocity_Y','angular_velocity_Z']
for norm in norm_cols:
iia = feat_cols.index(norm)
 np_arr = feat_array[:,:,iia]
mean_arr = np.mean(np_arr,1)
 mean_arr = np.expand_dims(mean_arr,1)
 mean_arr = np.repeat(mean_arr,num_meas,1)
 np_arr = np_arr - mean_arr
feat_array[:,:,iia] = np_arr
house_all_data['SalePrice_log'] = np.log1p(house_all_data['SalePrice'] )<sort_values>
def absfft(x):
return np.abs(np.fft.rfft(x))
feat_fft_array = np.copy(feat_array[:,:,3:])
 feat_fft_array = np.apply_along_axis(absfft,1,feat_fft_array )
house_numerical_data = house_numerical_data.drop(['SalePrice'], axis=1)
 skew_features = house_numerical_data.apply(lambda x: skew(x.dropna())).sort_values(ascending=False)
 high_skew = skew_features[skew_features > 0.5]<create_dataframe>
num_sensor = feat_array.shape[2]
for i in range(num_sensor):
mean_s = np.mean(feat_array[:,:,i])
 sd_s = np.std(feat_array[:,:,i])
 feat_array[:,:,i] =(feat_array[:,:,i]-mean_s)/sd_s
num_sensor_fft = feat_fft_array.shape[2]
for i in range(num_sensor_fft):
mean_s = np.mean(feat_fft_array[:,:,i])
 sd_s = np.std(feat_fft_array[:,:,i])
 feat_fft_array[:,:,i] =(feat_fft_array[:,:,i]-mean_s)/sd_s
skewness = pd.DataFrame({'Skew' :skew_features})
 skewness<compute_test_metric>
from keras.layers import Input,Dense, Dropout, BatchNormalization, SeparableConv1D, Reshape, LSTM, DepthwiseConv2D,AveragePooling2D, CuDNNLSTM, Concatenate
from keras.models import Model
from keras.backend import squeeze
from keras.regularizers import l2
for features in high_skew.index:
house_all_data[features]= boxcox1p(house_all_data[features], boxcox_normmax(house_all_data[features] + 1))<drop_column>
kr = None
num_groups = np.unique(group_array ).shape[0]
num_surfaces = np.unique(target_array ).shape[0]
def get_net_with_fft_mag_only(dp):
inputs_t = Input(shape=(128,len(feat_cols)))
 x = SeparableConv1D(32,8,2,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(inputs_t)
 x = Dropout(dp )(x)
 x = SeparableConv1D(64,8,4,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(x)
 x = Dropout(dp )(x)
 x = SeparableConv1D(128,8,4,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(x)
 x = Dropout(dp )(x)
 x = SeparableConv1D(256,8,4,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(x)
 x = Reshape(( 256,))(x)
 x = Dropout(dp )(x)
 x = Dense(64, activation='relu',kernel_regularizer=kr )(x)
 x = Dropout(dp )(x)
 x = Dense(64, activation='relu' )(x)
 inputs_f = Input(shape=(feat_fft_array.shape[1],feat_fft_array.shape[2]))
y = SeparableConv1D(32,8,2,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(inputs_f)
 y = Dropout(dp )(y)
 y = SeparableConv1D(64,8,2,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(y)
 y = Dropout(dp )(y)
 y = SeparableConv1D(128,8,4,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(y)
 y = Dropout(dp )(y)
 y = SeparableConv1D(128,8,4,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(y)
 y = Dropout(dp )(y)
 y = SeparableConv1D(256,8,2,'same',depth_multiplier=1,activation='relu',kernel_regularizer=kr )(y)
 y = Reshape(( 256,))(y)
 y = Dropout(dp )(y)
 y = Dense(64, activation='relu',kernel_regularizer=kr )(y)
 y = Dropout(dp )(y)
 y = Dense(64, activation='relu' )(y)
 inputs = [inputs_t,inputs_f]
z = Concatenate()([x,y])
 z = Dense(64, activation='relu' )(z)
 predictions = Dense(num_surfaces, activation='softmax' )(z)
 model = Model(inputs=inputs, outputs=predictions)
 model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
 return model
house_all_data= house_all_data.drop(['GarageArea','SalePrice'],axis=1 )<split>
depthwise = False
fft_net = True
if(not(testing)) :
gkf = GroupKFold(3)
 train_gen = gkf.split(X=feat_array,groups=group_array)
 preds = np.zeros_like(target_array)
 for train_idx,test_idx in train_gen:
t_feats = feat_array[train_idx]
t_feats_fft = feat_fft_array[train_idx]
v_feats = feat_array[test_idx]
v_feats_fft = feat_fft_array[test_idx]
t_vals = target_array[train_idx]
v_vals = target_array[test_idx]
pred_classes = np.zeros([v_vals.shape[0],num_surfaces,5])
 for k in range(5):
nnet = get_net_with_fft_mag_only(0.5)
 nnet.fit(x=[t_feats,t_feats_fft],y=t_vals,batch_size=256,epochs=3000,validation_data=([v_feats,v_feats_fft],v_vals),verbose=2)
 pred_classes[:,:,k] = nnet.predict([v_feats,v_feats_fft])
 pred_classes = np.mean(pred_classes,axis=2)
 pred_classes = np.argmax(pred_classes,axis=1)
 preds[test_idx] = pred_classes
print('Val accuracy: ',accuracy_score(v_vals,pred_classes))
pred_classes = nnet.predict([t_feats,t_feats_fft])
 pred_classes = np.argmax(pred_classes,axis=1)
 print('Train accuracy: ',accuracy_score(t_vals,pred_classes))
print('5 Fold accuracy: ', accuracy_score(target_array,preds))
else:
t_feats = feat_array[:3810]
t_feats_fft = feat_fft_array[:3810]
t_vals = target_array[:3810]
v_feats = feat_array[3810:]
v_feats_fft = feat_fft_array[3810:]
pred_classes = np.zeros([v_feats.shape[0],num_surfaces,3])
 for k in range(3):
nnet = get_net_with_fft_mag_only(0.5)
 nnet.fit(x=[t_feats,t_feats_fft],y=t_vals,batch_size=256,epochs=3000,verbose=0)
 pred_classes[:,:,k] = nnet.predict([v_feats,v_feats_fft])
 pred_classes = np.mean(pred_classes,axis=2)
 pred_classes = list(np.argmax(pred_classes,axis=1))
pred_classes = [le.inverse_transform([i])[0] for i in pred_classes]
sub_df = pd.read_csv(PATH+'/sample_submission.csv')
 sub_df['surface'] = pred_classes
sub_df.to_csv('submission.csv',index=False )
train_house_data = house_all_data[:1459]
test_house_data = house_all_data.iloc[1459:]<drop_column>
gc.collect()
test_house_data = test_house_data.drop(['SalePrice_log'],axis=1)
 test_house_data<split>
from collections import Counter
from sklearn.neighbors import KNeighborsClassifier
from keras.layers import LSTM,Input,Dense,Flatten,SpatialDropout1D,Dropout,CuDNNLSTM,Reshape,Concatenate
from keras.layers import Lambda,concatenate,BatchNormalization
from keras.models import Model
from keras.utils import to_categorical
from keras.optimizers import Adam
from keras import backend as K
from sklearn.preprocessing import LabelEncoder
train_data, label_data = train_house_data.iloc[:,:-1],train_house_data.iloc[:,-1]<split>
directory = ".. /input/career-con-2019/"
X_test_path = os.path.join(directory,"X_test.csv")
 X_train_path = os.path.join(directory,"X_train.csv")
 X_test_data = pd.read_csv(X_test_path)
 X_train_data = pd.read_csv(X_train_path)
 y_train_data = pd.read_csv(os.path.join(directory,"y_train.csv"))
sample_submission = pd.read_csv(os.path.join(directory,"sample_submission.csv"))
X_train, X_test, y_train, y_test = train_test_split(
 train_data, label_data, test_size=0.3,random_state=42 )<split>
print("Train data: measurements = %d , features=%d" %X_train_data.shape)
 X_train_data.head()
kfold_cv=KFold(n_splits=15, random_state=42, shuffle=True)
 for train_index, test_index in kfold_cv.split(train_data,label_data):
X_train, X_test = train_data.iloc[train_index], train_data.iloc[test_index]
y_train, y_test = label_data.iloc[train_index], label_data.iloc[test_index]<train_model>
le = LabelEncoder()
le.fit(list(y_train_data["surface"]))
y_train_dataset_for_nn = to_categorical(le.transform(list(y_train_data["surface"])) )
model_xgb= XGBRegressor(colsample_bytree = 0.3,
learning_rate = 0.03,
max_depth = 5,
min_child_weight = 5,
n_estimators = 500,
objective ='reg:squarederror')
 model_xgb.fit(X_train, y_train)
 y_pred_xgbm = model_xgb.predict(X_test )<choose_model_class>
def dataset_for_nn(X_dataset):
num_samples = X_dataset.shape[0]//128
X_dataset_for_nn = np.zeros(( num_samples,128,10))
for i in range(num_samples):
subset = np.array(X_dataset.iloc[i*128:(i+1)*128,3:])
 X_dataset_for_nn[i,:,:] = subset
return X_dataset_for_nn
model_lgbm = lgb.LGBMRegressor(boosting_type ='gbdt',
colsample_bytree = 0.3,
learning_rate = 0.03,
n_estimators = 1000,
num_leaves = 6,
objective = 'regression',
reg_alpha = 1,
reg_lambda = 1.4)
 model_lgbm.fit(X_train, y_train)
 y_pred_lgbm = model_lgbm.predict(X_test )<train_model>
X_train_for_nn = dataset_for_nn(X_train_data)
 X_test_for_nn = dataset_for_nn(X_test_data )
model_gbr= GradientBoostingRegressor(learning_rate=0.01,
max_depth=6,
max_features=0.1,
min_samples_leaf=30,
n_estimators=1000)
 model_gbr.fit(X_train, y_train)
 y_pred_gbr = model_gbr.predict(X_test )<define_search_space>
print("Train data for NN")
 X_train_for_nn[0]
ridge_alpha = [1e-15, 1e-10, 1e-8, 9e-4, 7e-4, 5e-4, 3e-4, 1e-4, 1e-3, 5e-2, 1e-2, 0.1, 0.3,
1, 3, 5, 10, 15, 18, 20, 30, 50, 75, 100]
lasso_alpha = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
enet_alpha = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]<find_best_model_class>
def freqs(dataset,width):
X = np.abs(fftpack.fft(dataset))
squeezed_dataset = []
for i in range(64//width):
squeezed_dataset.append(np.mean(X[i*width:(i+1)*width]))
return squeezed_dataset
rr = make_pipeline(RobustScaler() , RidgeCV(alphas = ridge_alpha, cv=10))
rr.fit(X_train, y_train)
 pred_ridge= rr.predict(X_test )<train_on_grid>
def X_features(X_dataset,width=3):
num_samples = len(list(set(X_dataset["series_id"])))
 num_cols = 64//width
features = np.zeros(( num_samples,40+10*num_cols))
for i in range(num_samples):
X_train_subset = np.array(X_dataset.iloc[i*128:(i+1)*128,3:])
 features[i,:10] = np.mean(X_train_subset,axis=0)
 features[i,10:20] = np.std(X_train_subset,axis=0)
 features[i,20:30] = np.max(X_train_subset,axis=0)-np.min(X_train_subset,axis=0)
 features[i,30:40] = X_train_subset[-1,:]-X_train_subset[0,:]
for j in range(X_train_subset.shape[1]):
features[i,40+j*num_cols:40+(j+1)*num_cols] = freqs(X_train_subset[:,j],width)
 return features
model_lasso = make_pipeline(RobustScaler() , LassoCV(alphas = lasso_alpha, cv=10))
model_lasso.fit(X_train, y_train)
 pred_lasso= model_lasso.predict(X_test )<choose_model_class>
X_train_features = X_features(X_train_data)
 X_test_features = X_features(X_test_data )
model_enet = make_pipeline(RobustScaler() ,
ElasticNetCV(fit_intercept=True, alphas=enet_alpha, cv=10))
model_enet.fit(X_train, y_train)
 pred_enet= model_enet.predict(X_test )<train_model>
def LSTM_NN(drop):
inp = Input(shape=(128,10))
x = SpatialDropout1D(0.1 )(inp)
 inp_2 = Input(shape=(250,))
x_2 = Dense(250, input_shape=(250,), activation="sigmoid" )(inp_2)
 x_2 = Dropout(drop )(x_2)
 x_2 = Dense(120, activation="sigmoid" )(x_2)
 x_2 = Dropout(drop )(x_2)
 x_2 = Dense(60, activation="sigmoid" )(x_2)
 x_2 = Dropout(drop )(x_2)
 x_2 = BatchNormalization()(x_2)
 x = CuDNNLSTM(units=200, return_sequences=True, return_state=False, go_backwards=False )(x)
 x = Dropout(drop )(x)
 x = CuDNNLSTM(units=100, return_sequences=False, return_state=False, go_backwards=False )(x)
 x = concatenate([x,x_2])
 x = Dropout(drop )(x)
 outp = Dense(9, activation="sigmoid" )(x)
 model = Model(inputs=[inp,inp_2], outputs=outp)
 model.compile(loss='categorical_crossentropy', optimizer=Adam())
 return model
model_svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0001))
model_svr.fit(X_train, y_train)
 pred_svr= model_svr.predict(X_test )<predict_on_test>
%%time
n_splits=5
kfold = StratifiedKFold(n_splits=n_splits, random_state=10, shuffle=True)
 y_test = np.zeros(( X_test_for_nn.shape[0],9*n_splits))
train_preds = np.zeros(( X_train_for_nn.shape[0],9))
X = X_train_for_nn
X_test = X_test_for_nn
Y = np.array(list(y_train_data["surface"]))
for i,(train_index, valid_index)in enumerate(kfold.split(X, Y)) :
X_train, X_val = X[list(train_index),:,:],X[list(valid_index),:,:]
X_train_feat, X_val_feat = X_train_features[list(train_index),:],X_train_features[list(valid_index),:]
Y_train, Y_val = Y[list(train_index)], Y[list(valid_index)]
Y_train = to_categorical(le.transform(Y_train))
Y_val = to_categorical(le.transform(Y_val))
model = LSTM_NN(0.15)
 model.fit([X_train,X_train_feat], Y_train, epochs=120, validation_data=([X_val,X_val_feat], Y_val), verbose=2)
 y_pred = model.predict([X_val,X_val_feat], verbose=2)
 y_test[:,i*9:(i+1)*9] = model.predict([X_test,X_test_features])
 train_preds[list(valid_index),:] = np.squeeze(y_pred )
y_pred_xgbm_train = model_xgb.predict(X_train)
 y_pred_lgbm_train = model_lgbm.predict(X_train)
 y_pred_gbr_train = model_gbr.predict(X_train)
 pred_ridge_train= rr.predict(X_train)
 pred_lasso_train= model_lasso.predict(X_train)
 pred_enet_train= model_enet.predict(X_train)
 pred_svr_train= model_svr.predict(X_train )<compute_test_metric>
res_train = np.argmax(train_preds,axis=1)
 ans_train = np.argmax(to_categorical(le.transform(Y)) ,axis=1)
 print("CV score",round(accuracy_score(res_train,ans_train),4))
print("Train Error" +", " + 'XGBM– ' + str(math.sqrt(metrics.mean_squared_error(y_train, y_pred_xgbm_train))))
print("Train Error" +", " + 'LGBM– ' + str(math.sqrt(metrics.mean_squared_error(y_train, y_pred_lgbm_train))))
print("Train Error" +", " + 'GBR– ' + str(math.sqrt(metrics.mean_squared_error(y_train, y_pred_gbr_train))))
print("Train Error" +", " + 'Ridge– ' + str(math.sqrt(metrics.mean_squared_error(y_train, pred_ridge_train))))
print("Train Error" +", " + 'Lasso– ' + str(math.sqrt(metrics.mean_squared_error(y_train, pred_lasso_train))))
print("Train Error" +", " + 'Elastic Net– ' + str(math.sqrt(metrics.mean_squared_error(y_train, pred_enet_train))))
print("Train Error" +", " + 'SVR– ' + str(math.sqrt(metrics.mean_squared_error(y_train, pred_svr_train))))<compute_test_metric>
def most_frequent(List):
occurence_count = Counter(List)
 return occurence_count.most_common(1)[0][0]
res_test_inter = np.zeros(( y_test.shape[0],n_splits))
res_test=[]
for i in range(n_splits):
inter_arr = y_test[:,i*9:(i+1)*9]
res_test_inter[:,i] = np.argmax(inter_arr,axis=1)
 for j in range(y_test.shape[0]):
res_test.append(int(most_frequent(res_test_inter[j,:])) )
print("Test Error" +", " + 'XGBM– ' + str(math.sqrt(metrics.mean_squared_error(y_test, y_pred_xgbm))))
print("Test Error" +", " + 'LGBM– ' + str(math.sqrt(metrics.mean_squared_error(y_test, y_pred_lgbm))))
print("Test Error" +", " + 'GBR– ' + str(math.sqrt(metrics.mean_squared_error(y_test, y_pred_gbr))))
print("Test Error" +", " + 'Ridge– ' + str(math.sqrt(metrics.mean_squared_error(y_test, pred_ridge))))
print("Test Error" +", " + 'Lasso– ' + str(math.sqrt(metrics.mean_squared_error(y_test, pred_lasso))))
print("Test Error" +", " + 'Elastic Net– ' + str(math.sqrt(metrics.mean_squared_error(y_test, pred_enet))))
print("Test Error" +", " + 'SVR– ' + str(math.sqrt(metrics.mean_squared_error(y_test, pred_svr))))<compute_test_metric>
test_for_sub=le.inverse_transform(res_test)
 print(test_for_sub[:5])
 test_size = len(list(set(X_test_data["series_id"])))
 Y_test_pred_array = np.zeros(( test_size,2))
Y_test_for_submission = pd.DataFrame(Y_test_pred_array,columns = ["series_id","surface"])
 Y_test_for_submission.iloc[:,0] = list(range(test_size))
Y_test_for_submission.iloc[:,1] = test_for_sub
Y_test_for_submission.to_csv("submission.csv",index=None)
 Y_test_for_submission.head()
def rmse(prediction, target):
return str(math.sqrt(metrics.mean_squared_error(target, prediction)))
 keys = ['XGBM_train', 'LGBM_train', 'GBR_train', 'Ridge_train', 'Lasso_train', 'ElasticN_train','SVR_train',
'XGBM_test', 'LGBM_test', 'GBR_test', 'Ridge_test', 'Lasso_test', 'ElasticN_test', 'SVR_test']
values = [rmse(y_train, y_pred_xgbm_train),rmse(y_train, y_pred_lgbm_train), rmse(y_train, y_pred_gbr_train),
rmse(y_train, pred_ridge_train),rmse(y_train, pred_lasso_train), rmse(y_train, pred_enet_train),
rmse(y_train, pred_svr_train), rmse(y_test, y_pred_xgbm), rmse(y_test, y_pred_lgbm),
rmse(y_test, y_pred_gbr),rmse(y_test, pred_ridge),rmse(y_test, pred_lasso),
rmse(y_test, pred_enet),rmse(y_test, pred_svr)]<train_on_grid>
gc.collect()
stack = StackingCVRegressor(regressors=(model_xgb, model_lgbm, model_gbr,rr, model_lasso, model_enet,model_svr),
meta_regressor=model_xgb, use_features_in_secondary=True)
 stack_model = stack.fit(X_train.values, y_train.values )<split>
data_target = pd.read_csv(PATH+"/y_train.csv")
 train_X = pd.read_csv(PATH+'/X_train.csv' ).iloc[:,3:]
test_X = pd.read_csv(PATH+'/X_test.csv' ).iloc[:,3:]
kfold_cv=KFold(n_splits=10,random_state =42, shuffle=True)
 for train_index, test_index in kfold_cv.split(train_data,label_data):
X_train, X_test = train_data.iloc[train_index], train_data.iloc[test_index]
y_train, y_test = label_data.iloc[train_index], label_data.iloc[test_index]<predict_on_test>
train_X = train_X.append(test_X)
 train_X = train_X.values.reshape(-1,128,10 )
def blend_models_predict(X):
return(( 0.05 * model_xgb.predict(X)) + 
(0.1 * model_lgbm.predict(X)) + 
(0.1 * model_gbr.predict(X)) + 
(0.1 * rr.predict(X)) + 
(0.1 * model_lasso.predict(X)) + 
(0.1 * model_enet.predict(X)) + 
(0.1 * model_svr.predict(X)) + 
(0.35 * stack_model.predict(X.values)))
 blend_model_pred = blend_models_predict(X_test )<compute_test_metric>
train_left_edges, train_left_linked = find_run_edges(train_X, edge='left')
 train_right_edges, train_right_linked = find_run_edges(train_X, edge='right')
 print('Found', len(train_left_edges), 'left edges and', len(train_right_edges), 'right edges.' )
y_pred_blend = blend_models_predict(X_test)
 print('MAE: %2f'% mean_absolute_error(y_test, y_pred_blend))
print('RMSE: %2f'% np.sqrt(mean_squared_error(y_test, y_pred_blend)))
 print('R2 %2f' % r2_score(y_test, y_pred_blend))<compute_test_metric>
train_runs = find_runs(train_X, train_left_edges, train_right_edges )
print('Blend Model– ' + str(math.sqrt(metrics.mean_squared_error(y_test, blend_model_pred))))<predict_on_test>
lost_samples = np.array([ i for i in range(len(train_X)) if i not in np.concatenate(train_runs)])
 print(lost_samples)
 print(len(lost_samples))
test_data_predict = blend_models_predict(test_house_data)
 test_data_predict<load_from_csv>
first_double_surface = 0
for i in range(0, len(train_runs)) :
for x in train_runs[i]:
if x==821:
print(i)
 first_double_surface = i
submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv')
 submission.iloc[:,1] = np.floor(np.expm1(blend_models_predict(test_house_data)) )<predict_on_test>
new_train_runs = [821, 974, 328, 1548, 172,
355, 957, 1481, 1046, 1650, 857, 724, 164, 1092, 1017, 1300,
1212, 536, 531, 1032, 994, 1501, 588, 579, 1177, 812, 1333,
1253]
train_runs[first_double_surface] = train_runs[first_double_surface][0:-len(new_train_runs)]
submission.iloc[:,1] = np.floor(( np.floor(np.expm1(blend_models_predict(test_house_data)))) )<save_to_csv>
train_runs.append(np.array(new_train_runs))
q1 = submission['SalePrice'].quantile(0.0042)
 q2 = submission['SalePrice'].quantile(0.99)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1)
 submission.to_csv("submission.csv", index=False )<load_from_csv>
second_double_surface = 0
for i in range(0, len(train_runs)) :
for x in train_runs[i]:
if x==3055:
print(i)
 second_double_surface = i
train_data = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test_data = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 test_data['SalePrice'] = np.nan
all_data = pd.concat([train_data, test_data])
 print("Rows in train data = {}".format(len(train_data)))
 print("Rows in test data = {}".format(len(test_data)) )<filter>
new_train_runs2 = [3055, 3360, 3662,
3780, 3663, 3091, 3769, 3175, 1957, 2712, 2063, 2708, 3139, 2722]
train_runs[second_double_surface] = train_runs[second_double_surface][0:-len(new_train_runs2)]
print("NaNs in each training Feature")
 dfNull = all_data.isnull().sum().to_frame('nulls')
 print(dfNull.loc[dfNull['nulls'] > 0] )<drop_column>
train_runs.append(np.array(new_train_runs2))
all_data.drop(columns=['Alley', 'FireplaceQu', 'PoolQC', 'Fence', 'MiscFeature'], inplace=True )<categorify>
third_double_surface = 0
for i in range(0, len(train_runs)) :
for x in train_runs[i]:
if x==2484:
print(i)
 third_double_surface = i
mapping = {'Ex': 5.0, 'Gd': 4.0, 'TA': 3.0, 'Fa': 2.0, 'Po': 1.0}
all_data.replace({'KitchenQual': mapping}, inplace=True)
 all_data.replace({'BsmtQual': mapping}, inplace=True)
 all_data.replace({'BsmtCond': mapping}, inplace=True)
 all_data.replace({'GarageQual': mapping}, inplace=True)
 all_data.replace({'GarageCond': mapping}, inplace=True )<categorify>
train_runs[third_double_surface]
all_data['MSZoning'].fillna('RL', inplace=True)
 all_data.drop(columns=['Utilities'], inplace=True)
 all_data['Electrical'].fillna('SBrkr', inplace=True)
 all_data['Functional'].fillna('Typ', inplace=True )<filter>
new_train_runs3 = [2484, 3062, 2290, 3517, 3293, 2651, 3767,
2029, 2558, 3580, 1874, 3373, 2514, 2308, 3160, 3161, 3613, 2511,
2469, 2990, 2780, 3756, 2376, 2616, 2540, 2039, 2219, 3743, 3198,
2584, 2752, 2304, 2887, 2841, 3480, 2517, 3020, 3424, 2027, 2652,
2648, 3433, 2359, 3392, 3164, 3798, 3642, 2713, 3405, 3673, 2369,
3411, 3595, 2242, 2307, 1897, 2834, 2350, 3795, 2948, 1856, 3486,
3353, 1966]
train_runs[third_double_surface] = train_runs[third_double_surface][0:-len(new_train_runs3)]
train_runs.append(np.array(new_train_runs3))
print(all_data['Exterior2nd'].values[all_data['Exterior1st'].isnull() == True] )<data_type_conversions>
fourth_double_surface = 0
for i in range(0, len(train_runs)) :
for x in train_runs[i]:
if x==3501:
print(i)
 fourth_double_surface = i
all_data['Exterior1st'].fillna('VinylSd', inplace=True)
 all_data['Exterior2nd'].fillna('VinylSd', inplace=True )<filter>
train_runs[fourth_double_surface]
print(all_data['MasVnrArea'].values[all_data['MasVnrType'].isnull() == True] )<feature_engineering>
new_train_runs4 = [3501, 2785]
train_runs[fourth_double_surface] = train_runs[fourth_double_surface][0:-len(new_train_runs4)]
train_runs.append(np.array(new_train_runs4))
indices =(np.argwhere(( all_data['MasVnrType'].isnull().values)&(all_data['MasVnrArea'].isnull().values == False)) ).flatten()
print(indices)
 all_data.iloc[indices[0], all_data.columns.get_loc('MasVnrType')] = 'BrkFace'
all_data['MasVnrType'].fillna('None', inplace=True)
 all_data['MasVnrArea'].fillna(0.0, inplace=True )<data_type_conversions>
df_train_y = pd.DataFrame()
df_train_y['run_id'] = 0
df_train_y['run_pos'] = 0
for run_id in range(len(train_runs)) :
for run_pos in range(len(train_runs[run_id])) :
series_id = train_runs[run_id][run_pos]
df_train_y.at[ series_id, 'run_id' ] = run_id
df_train_y.at[ series_id, 'run_pos' ] = run_pos
df_train_y.tail()
all_data['BsmtCond'].fillna(3.0, inplace=True)
 all_data['BsmtQual'].fillna(3.0, inplace=True)
 all_data['BsmtExposure'].fillna('No', inplace=True)
 all_data['BsmtFinType1'].fillna('Unf', inplace=True)
 all_data['BsmtFinType2'].fillna('Unf', inplace=True)
 all_data['BsmtFullBath'].fillna(0.0, inplace=True)
 all_data['BsmtHalfBath'].fillna(0.0, inplace=True )<groupby>
df_train_y['index'] = df_train_y.index
df_train_y = df_train_y.sort_values('index')
 df_train_y.rename(columns={'index':'series_id'}, inplace=True)
 df_train_y['run_id'] = df_train_y['run_id'].apply(lambda x: int(x))
df_train_y['run_pos'] = df_train_y['run_pos'].apply(lambda x: int(x))
df_train_y.tail()
meanKitchenQual = np.rint(all_data.groupby(['OverallQual'])['KitchenQual'].mean())
 print(meanKitchenQual)
 print(all_data['OverallQual'].values[np.isnan(all_data['KitchenQual'].values)])
 all_data['KitchenQual'].fillna(3.0, inplace=True )<data_type_conversions>
run_id_train = df_train_y['run_id'][0:3810].values
run_id_test = df_train_y['run_id'][3810:7626].values
all_data['GarageType'].fillna('Attchd', inplace=True)
 all_data['GarageFinish'].fillna('Unf', inplace=True)
 all_data['GarageCars'].fillna(2.0, inplace=True)
 all_data['GarageQual'].fillna(3.0, inplace=True)
 all_data['GarageCond'].fillna(3.0, inplace=True )<count_missing_values>
data_target['run_id'] = run_id_train
mapping_leak = {}
for i in range(0,3810):
cur_data = data_target.iloc[i]
mapping_leak.update({cur_data['run_id']: cur_data['surface']})
 unknown_run = []
ans_test = []
known_series = []
unknown_series = []
for i in range(0,3816):
if run_id_test[i] in mapping_leak:
ans_test.append(mapping_leak[run_id_test[i]])
 known_series.append(i)
 else:
ans_test.append('unknown')
 unknown_series.append(i)
 unknown_run.append(run_id_test[i] )
for i in keys:
print(np.argwhere(np.isnan(all_data[i].values)) )<correct_missing_values>
sub = pd.read_csv(PATH+"/sample_submission.csv")
 sub['surface'] = ans_test
sub.head()
median_total = np.median(all_data['TotalBsmtSF'].values[~np.isnan(all_data['TotalBsmtSF'].values)])
 median_SF1 = np.median(all_data['BsmtFinSF1'].values[~np.isnan(all_data['BsmtFinSF1'].values)])
 median_Unf = np.median(all_data['BsmtUnfSF'].values[~np.isnan(all_data['BsmtUnfSF'].values)])
 print(median_total, median_SF1, median_Unf)
 all_data['BsmtFinSF1'].fillna(median_SF1, inplace=True)
 all_data['BsmtFinSF2'].fillna(0.0, inplace=True)
 all_data['BsmtUnfSF'].fillna(median_Unf, inplace=True)
 all_data['TotalBsmtSF'].fillna(median_total, inplace=True )<feature_engineering>
best = pd.read_csv('.. /input/robots-best-submission/mybest0.73.csv')
 map_best_ans = {}
for i in range(0, best.shape[0]):
map_best_ans.update({best.iloc[i]['series_id'] : best.iloc[i]['surface'] } )
garage_yr_built = all_data['GarageYrBlt'].values
indices = np.argwhere(np.isnan(garage_yr_built))
yearbuilt = all_data['YearBuilt'].values[indices]
garage_yr_built[indices] = yearbuilt
all_data['GarageYrBlt'] = garage_yr_built
median = np.median(all_data['GarageArea'].values[~np.isnan(all_data['GarageArea'].values)])
 print(median)
 all_data['GarageArea'].fillna(median, inplace=True )<data_type_conversions>
result = []
for i in range(0, sub.shape[0]):
if(sub.surface[i] == 'unknown'):
result.append(map_best_ans[i])
 else:
result.append(sub.surface[i] )
median = np.median(all_data['LotFrontage'].values[~np.isnan(all_data['LotFrontage'].values)])
 print(median)
 all_data['LotFrontage'].fillna(median, inplace=True )<count_missing_values>
sub.surface = result
sub.to_csv('submission3.csv', index=False)
 sub.head()
print(all_data.isnull().sum().values )<categorify>
sub.surface.value_counts()
mapping = {1: 'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', 7:'Jul',
8:'Aug', 9:'Sep', 10:'Oct', 11:'Nov', 12:'Dec'}
all_data.replace({'MoSold': mapping}, inplace=True)
 mapping = {20:'A', 30:'B', 40:'C', 45:'D', 50:'E', 60:'F', 70:'G', 75:'H', 80:'I', 85:'J',
90:'K', 120:'L', 150:'M', 160:'N', 180:'O', 190:'P'}
all_data.replace({'MSSubClass': mapping}, inplace=True)
 all_data['sale_age'] = 2020 - all_data['YrSold']
all_data['house_age'] = 2020 - all_data['YearBuilt']
all_data['remodel_age'] = 2020 - all_data['YearRemodAdd']
all_data['garage_age'] = 2020 - all_data['GarageYrBlt']
all_data.drop(columns=['YrSold', 'YearBuilt', 'YearRemodAdd', 'GarageYrBlt'], inplace=True)
 all_data['TotalArea'] =(all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
+ all_data['GrLivArea'] + all_data['GarageArea'])
 all_data['TotalBathrooms'] = all_data['FullBath'] + all_data['HalfBath']*0.5<categorify>
gc.collect()