Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<categorify>
warnings.filterwarnings(action="ignore")
 %pylab inline
plt.style.use('seaborn-poster')
 sns.set(font_scale=2)
 random_seed = 123456789
np.random.seed(random_seed)
 mean_squared_error,mean_squared_log_error,r2_score,make_scorer
def getMetrics(val_reels, val_predites):
correlations = pd.DataFrame(list(zip(val_predites,val_reels)) , columns=['predictions','valeurs'])
 return sqrt(mean_squared_error(val_reels, val_predites)) ,sqrt(abs(r2_score(val_reels, val_predites))),correlations.astype(float ).corr().valeurs[0]
def affichageMetricsSansLog(val_reels, val_predites):
print("Mean absolute error MAE : %.6f" % mean_absolute_error(val_reels, val_predites))
print("Mean squared error MSE : %.6f" % mean_squared_error(val_reels, val_predites))
print("Root mean squared error RMSE : %.6f" % sqrt(mean_squared_error(val_reels, val_predites)))
 print("-"*50,"
\tPour comparer voici
\tla distribution des valeurs de la variable :
","-"*50,"
",pd.Series(val_reels ).describe() ,"
","-"*50,)
 correlations = pd.DataFrame(list(zip(val_predites,val_reels)) , columns=['predictions','valeurs'])
 sns.set(font_scale=2)
 plt.figure(figsize=(8,8))
plt.title('Correlation Pearson des variables', y=1.05, size=24)
 ax = sns.heatmap(correlations.astype(float ).corr() ,linewidths=0.3,vmax=1.0,fmt ="1.4f",
square=True, cmap='coolwarm', linecolor='white', annot=True)
 ax.set_xlim(0,2);
ax.set_ylim(2,0);
plt.show()
sns.set(font_scale=2)
 sns.jointplot(x='valeurs',y='predictions',
data=correlations, kind='reg', size=25)
 noms = [
"LinearRegressionRidge",
"BayesianRidge",
"SVR_linear",
]
regresseurs = [
Ridge() ,
BayesianRidge() ,
SVR(kernel='linear', C=0.005),
]
regresseurs = { nom:reg for nom,reg in zip(noms,regresseurs)}
modelRFE = {}
def report(results, n_top=3):
for i in range(1, n_top + 1):
candidates = np.flatnonzero(results['rank_test_score'] == i)
 for candidate in candidates:
print("Model with rank: {0}".format(i))
print("Mean validation score: {0:.3f}(std: {1:.3f})".format(
 results['mean_test_score'][candidate],
results['std_test_score'][candidate]))
print("Parameters: {0}".format(results['params'][candidate]))
print("")
 def choixModelsRegressions(regresseurs):

 noms = [x for x in regresseurs]
print('Initialisation des models
---Si vous sesisez un ou plusieurs caratères le modele est effacé !---
')
 for model in noms:
if input('Model : '+model+' -- ')!= '':
regresseurs.pop(model)
 def calculRegressions(X_train, X_test, y_train, y_test):
rmse,r,p = dict() ,dict() ,dict()
yRegressions = pd.DataFrame()
yRegressions['Observations']=y_test
for nom in regresseurs:
t1 = time.time()
yRegressions[nom] = regresseurs[nom].fit(X_train, y_train ).predict(X_test)
 rmse[nom],r[nom],p[nom] = getMetrics(y_test, yRegressions[nom])
 print('Exécution '+nom.ljust(30)+(': %.2fs' %(time.time() - t1)).lstrip('0')+(' p = %.12f' % p[nom] ).lstrip('0'))
resultats = pd.DataFrame(pd.Series(rmse),columns=["RMSE"])
 resultats["R"] = pd.Series(r)
 resultats["Pearson"] = pd.Series(p)
 resultats.sort_values(by='Pearson',ascending=False, inplace=True)
 return resultats,yRegressions, regresseurs
def initRFE(X, y, regresseurs, modelRFE):
for nom in regresseurs:
t1 = time.time()
modelRFE[nom] = RFE(regresseurs[nom])
 modelRFE[nom].fit(X,y)
 return modelRFE
def calculRegressionsRFE(X, y, regresseurs, modelRFE, modelOcSVM=None, taux = 0.3):
rmse,r,p = dict() ,dict() ,dict()
for nom in regresseurs:
t1 = time.time()
X_train, X_test, y_train, y_test = train_test_split(
 X,
y,
test_size=taux,
random_state=random_seed 
)
 if modelOcSVM is not None :
newX = modelOcSVM.predict(X_train)
 X_train, y_train = X_train[newX != -1], y_train[newX != -1]
else :
newX = np.ones(X_train.shape[0])
 X_train, X_test = X_train[X_train.columns[modelRFE[nom].support_]],X_test[X_test.columns[modelRFE[nom].support_]]
yRegressions[nom] = regresseurs[nom].fit(X_train, y_train ).predict(X_test)
 rmse[nom],r[nom],p[nom] = getMetrics(y_test, yRegressions[nom])
 print('Exécution '+nom.ljust(30)+(': %.2fs' %(time.time() - t1)).lstrip('0')+(' p = %.12f' % p[nom] ).lstrip('0'))
resultats = pd.DataFrame(pd.Series(rmse),columns=["RMSE"])
 resultats["R"] = pd.Series(r)
 resultats["Pearson"] = pd.Series(p)
 resultats.sort_values(by='Pearson',ascending=False, inplace=True)
 return resultats,yRegressions,regresseurs,modelRFE
for data in train_test_data:
data['Embarked'] = data['Embarked'].fillna("S")
 embarked_mapping = {"S": 0, "C": 1, "Q": 2}
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].map(embarked_mapping )<feature_engineering>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv', index_col='Id')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv', index_col='Id')
 donnees = pd.concat([test,train],sort=False)
 print('Train size : ',train.shape,'
Test size :',test.shape)
 donnees.tail()
train["FamilySize"] = train['SibSp'] + train['Parch'] + 1
test["FamilySize"] = test['SibSp'] + test['Parch'] + 1<drop_column>
Cible = 'SalePrice'
colQualitatives = ['MSSubClass', 'OverallQual', 'OverallCond'] + list(donnees.dtypes[donnees.dtypes == 'object'].reset_index() ['index'].values)
 colQuantitatives = list(donnees.drop(columns=colQualitatives + [Cible] ).columns )
train.drop(['Name'], axis = 1, inplace=True)
 test.drop(['Name'], axis = 1, inplace=True )<import_modules>
donnees[colQuantitatives].isna().sum() [
donnees[colQuantitatives].isna().sum() > 0
].sort_values()
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from keras.wrappers.scikit_learn import KerasClassifier
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout, Flatten, Conv2D,MaxPool2D
import keras<data_type_conversions>
variables = donnees[colQuantitatives].isna().sum() [ 
(donnees[colQuantitatives].isna().sum() > 0)& 
(donnees[colQuantitatives].isna().sum() < 24)
 ].reset_index() ['index']
for variable in variables:
donnees[variable].fillna(train[variable].mean() ,inplace=True )
continuous = ['Age', 'Fare', 'Parch', 'Pclass', 'SibSp', 'FamilySize']
scaler = StandardScaler()
for var in continuous:
train[var] = train[var].astype('float64')
 train[var] = scaler.fit_transform(train[var].values.reshape(-1, 1))
for var in continuous:
test[var] = test[var].astype('float64')
 test[var] = scaler.fit_transform(test[var].values.reshape(-1, 1))<choose_model_class>
donnees[colQuantitatives].isna().sum() [
donnees[colQuantitatives].isna().sum() > 0
].sort_values()
tf.keras.optimizers.Adam(
 learning_rate=0.003, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False,
name='Adam', 
)
 model = Sequential()
model.add(Flatten())
 model.add(Dense(32, input_dim=train.shape[1],kernel_initializer = 'uniform', activation='relu'))
model.add(Dense(32, kernel_initializer = 'uniform', activation = 'relu'))
model.add(Dropout(0.4))
model.add(Dense(32,kernel_initializer = 'uniform', activation = 'relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
<train_model>
y_pred = np.exp(modelRL.predict(modelSS.fit_transform(X)))
 affichageMetricsSansLog(pd.Series(y), pd.Series(y_pred))
history = model.fit(train, y_train, epochs=20, batch_size=50, validation_split = 0.2)
 %s: %.2f%%" %('val_acc', val_acc*100))<compute_test_metric>
regresseur = make_pipeline(PolynomialFeatures(4), Ridge())
 regresseur.fit(modelSS.fit_transform(X),np.log(y))
y_pred = np.exp(regresseur.predict(modelSS.fit_transform(X)))
 affichageMetricsSansLog(pd.Series(y), pd.Series(y_pred))
scores = model.evaluate(train, y_train, batch_size=32 )<save_to_csv>
X = donnees[(donnees.GarageYrBlt.isna())|(donnees.GarageYrBlt > 2010)][dataGarageYrBlt[1:]]
y = donnees[(donnees.GarageYrBlt.isna())|(donnees.GarageYrBlt > 2010)].GarageYrBlt
y_pred = pd.Series(np.exp(regresseur.predict(modelSS.fit_transform(X))), index=y.index)
 donnees.GarageYrBlt[(donnees.GarageYrBlt.isna())|(donnees.GarageYrBlt > 2010)] = y_pred
donnees.GarageYrBlt = donnees.GarageYrBlt.astype('int16' )
y_pred = model.predict(test)
 y_final =(y_pred > 0.5 ).astype(int ).reshape(test.shape[0])
 output = pd.DataFrame({'PassengerId': test_data['PassengerId'], 'Survived': y_final})
 output.to_csv('prediction-ann.csv', index=False )<set_options>
X = donnees[(~donnees.LotFrontage.isna())&(donnees.LotFrontage < 175)][dataLotFrontage[1:]]
y = donnees[(~donnees.LotFrontage.isna())&(donnees.LotFrontage < 175)].LotFrontage.ravel()
modelSS = StandardScaler()
modelRL = LinearRegression()
modelRL.fit(modelSS.fit_transform(X),np.log(y))
coef = pd.DataFrame({'Coef':modelRL.coef_})
 coef['Label'] = X.columns
coef.sort_values(by='Coef' )
%matplotlib inline 
<load_from_csv>
y_pred = np.exp(modelRL.predict(modelSS.fit_transform(X)))
 affichageMetricsSansLog(pd.Series(y), pd.Series(y_pred))
train = pd.read_csv(".. /input/train.csv" ).values
test = pd.read_csv(".. /input/test.csv" ).values
X_train = train[:, 1:].astype('float32')
 Y_train = train[:, 0].astype('int32')
 X_test = test[:, :].astype('float32' )<train_model>
regresseur = make_pipeline(PolynomialFeatures(4), Ridge())
 regresseur.fit(modelSS.fit_transform(X),np.log(y))
y_pred = np.exp(regresseur.predict(modelSS.fit_transform(X)))
 affichageMetricsSansLog(pd.Series(y), pd.Series(y_pred))
X_train = X_train/255.
 X_test = X_test/255.
 X_train = X_train.reshape(-1, 28, 28, 1)
 X_test = X_test.reshape(-1, 28, 28, 1)
 Y_train = to_categorical(Y_train, num_classes=10)
 X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size=0.15, random_state=23)
 datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range=0.15,
width_shift_range=0.15,
height_shift_range=0.15)
 datagen.fit(X_train )<choose_model_class>
X = donnees[donnees.LotFrontage.isna() ][dataLotFrontage[1:]]
y = donnees[donnees.LotFrontage.isna() ].LotFrontage
y_pred = pd.Series(np.exp(regresseur.predict(modelSS.fit_transform(X))), index=y.index)
 donnees.LotFrontage[ donnees.LotFrontage.isna() ] = y_pred
donnees.LotFrontage = donnees.LotFrontage.astype('int16' )
model = Sequential()
model.add(Conv2D(16, kernel_size=(5, 5),
activation='relu',
input_shape=(28, 28, 1)))
 model.add(Conv2D(32, kernel_size=(3, 3),
padding='same',
activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(64, kernel_size=(3, 3),
padding='same',
activation='relu'))
model.add(Conv2D(64, kernel_size=(3, 3),
padding='same',
activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(512, activation='relu'))
model.add(Dense(256, activation='relu'))
model.add(Dense(128, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
model.compile(loss=categorical_crossentropy,
optimizer=RMSprop(lr=1e-3),
metrics=['accuracy'] )<train_model>
donnees[colQualitatives].isna().sum() [
donnees[colQualitatives].isna().sum() > 0
].sort_values()
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.2,
patience=2)
 history = model.fit_generator(datagen.flow(X_train, Y_train, batch_size=70),
epochs=20,
validation_data=(X_val, Y_val),
verbose=2,
callbacks=[reduce_lr] )<predict_on_test>
for col in donnees[colQualitatives].isna().sum() [
donnees[colQualitatives].isna().sum() > 0
].reset_index() ['index']:
print("%12s"%col,'-',"%2d"%len(donnees[col].unique()),' - ',donnees[col].unique().tolist() [:8])
 donnees[colQualitatives] = donnees[colQualitatives].fillna('Non Affectée' )
Y_val_pred = model.predict_classes(X_val)
 Y_val_true = np.argmax(Y_val, axis=1)
 print(confusion_matrix(Y_val_true, Y_val_pred))<save_to_csv>
donnees.BsmtFullBath = donnees.BsmtFullBath.round(2 ).astype('str')
 colQualitatives.append('BsmtFullBath')
 donnees.BsmtHalfBath = donnees.BsmtHalfBath.round(2 ).astype('str')
 colQualitatives.append('BsmtHalfBath')
 donnees.GarageCars = donnees.GarageCars.round(2 ).astype('str')
 colQualitatives.append('GarageCars' )
Y_test_pred = model.predict_classes(X_test)
 submission = pd.DataFrame({ 'ImageId': range(1, 28001), 'Label': Y_test_pred })
 submission.to_csv("submission.csv", index=False )<import_modules>
donnees.drop(columns=['MoSold','YrSold'],inplace=True )
from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D
from keras.optimizers import RMSprop,Adam
from keras.preprocessing.image import ImageDataGenerator<import_modules>
for col in ['FullBath','HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd','Fireplaces','MiscVal']:
colQualitatives.append(col)
 print("%13s"%col,'-',"%3d"%len(donnees[col].unique()),' - ',
donnees[col].sort_values().unique().tolist() [:15] )
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from mlxtend.plotting import plot_confusion_matrix<load_from_csv>
for col in donnees.dtypes[donnees.dtypes == 'category'].reset_index() ['index'] :
donnees[col] = donnees[col].astype('str' )
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv' )<prepare_x_and_y>
modelVect = DictVectorizer()
modelVect.fit(donnees[colQualitatives].to_dict('records'))
donneesQualitatives = modelVect.transform(donnees[colQualitatives].to_dict('records'))
donneesQualitatives = pd.DataFrame(
 donneesQualitatives.toarray() ,
columns=modelVect.feature_names_,
index=donnees.index
)
X_train = train_data.copy()
y_train = train_data['label']
del X_train['label']
X_test = test_data.copy()
y_train = to_categorical(y_train, num_classes = 10 )<data_type_conversions>
colQuantitatives = list(donnees.drop(columns=colQualitatives + [Cible] ).columns)
 modelSS = StandardScaler()
donneesQuantitatives = pd.DataFrame(
 modelSS.fit_transform(donnees[colQuantitatives]),
columns=donnees[colQuantitatives].columns,
index=donnees.index 
)
 donneesQuantitatives.head()
X_train = X_train.astype('float32')/255
X_test = X_test.astype('float32')/255<split>
yeoJohn = PowerTransformer(method='yeo-johnson')
 donneesQuantitatives = pd.DataFrame(
 yeoJohn.fit_transform(donneesQuantitatives),
columns=donneesQuantitatives.columns,
index=donneesQuantitatives.index
)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.1, random_state=2020 )<choose_model_class>
donneesFinal = donneesQuantitatives.join(donneesQualitatives)
 donneesFinal['SalePrice'] = donnees.SalePrice
donneesTrain = donneesFinal[~donneesFinal.SalePrice.isna() ]
donneesTest = donneesFinal[donneesFinal.SalePrice.isna() ]
donneesTest.drop(columns='SalePrice', inplace=True)
 donneesTrain.shape,donneesTest.shape
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))<choose_model_class>
X_train, X_test, y_train, y_test = train_test_split(
 donneesTrain.drop(columns='SalePrice' ).values,
np.log(donneesTrain['SalePrice'].values),
test_size=0.3,
random_state=random_seed 
)
 X_train.shape, X_test.shape
optimizer = RMSprop(lr=0.001,rho=0.9, epsilon=1e-08, decay=0.0 )<choose_model_class>
resultats,yRegressions,regresseurs = calculRegressions(X_train, X_test, y_train, y_test)
 resultats
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'] )<define_variables>
modelOcSVM = OneClassSVM(nu=0.01, gamma=0.001)
 modelOcSVM.fit(donneesTrain.drop(columns='SalePrice'))
y_pred = modelOcSVM.predict(donneesTrain.drop(columns='SalePrice'))
print('ndividuals considered outlier :',sum(y_pred==-1))
epochs = 50
batch_size = 378<train_model>
newX = modelOcSVM.predict(X_train)
 resultats,yRegressions,regresseurs = calculRegressions(X_train[newX != -1], X_test, y_train[newX != -1], y_test)
 resultats
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<train_model>
modelRFE = initRFE(
 donneesTrain.drop(columns='SalePrice'),
donneesTrain.SalePrice,
regresseurs,
modelRFE
)
history = model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val, y_val),
steps_per_epoch=X_train.shape[0] // batch_size )<predict_on_test>
resultats,yRegressions,regresseurs,modelRFE = calculRegressionsRFE(donneesTrain.drop(columns='SalePrice'),
np.log(donneesTrain['SalePrice'].values), regresseurs, modelRFE, taux = 0.3)
 resultats
y_test = model.predict(X_test )<prepare_output>
affichageMetricsSansLog(np.exp(yRegressions.Observations), np.exp(yRegressions.LinearRegressionRidge))
y_test_classes = np.argmax(y_test, axis = 1 )<save_to_csv>
yRegressions = np.zeros(donneesTest.shape[0])
 for nom in regresseurs:
X = donneesTest
newX = modelOcSVM.predict(X)
 X = X[X.columns[modelRFE[nom].support_]]
yRegressions += regresseurs[nom].predict(X)
 yRegressions /= len(regresseurs)
 yRegressions = np.exp(yRegressions )
<predict_on_test><EOS> 
submission = pd.DataFrame(yRegressions,index = donneesTest.index ).reset_index()
submission.columns = ['Id','SalePrice']
submission.to_csv('submission01.csv',index=False )