Kaggle competition: Mercedes-Benz Greener Manufacturing
<SOS> metric: R2Score Kaggle data source: mercedes-benz-greener-manufacturing<drop_column>
rcParams['figure.figsize'] = 50,20
img=cv2.imread(".. /input/private-score/score.JPG")
 plt.imshow(img )
data.drop(['PoolQC', 'MiscFeature', 'Alley', 'Fence', 'FireplaceQu'], axis=1, inplace = True )<define_variables>
train = pd.read_csv('.. /input/mercedes-benz-greener-manufacturing/train.csv')
 test = pd.read_csv('.. /input/mercedes-benz-greener-manufacturing/test.csv')
 df = train
categorical_variables_string = \
['MSZoning', 'Street', 'LotShape', 'LandContour',
'Utilities', 'LotConfig', 'LandSlope', 'Neighborhood',
'Condition1', 'Condition2', 'BldgType', 'HouseStyle',
'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd',
'MasVnrType', 'ExterQual', 'ExterCond', 'Foundation',
'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2', 'Heating', 'HeatingQC', 'CentralAir',
'Electrical', 'KitchenQual', 'Functional', 'GarageType',
'GarageFinish', 'GarageQual', 'GarageCond', 'PavedDrive',
'SaleType', 'SaleCondition']
for v in categorical_variables_string:
data[v] = data[v].fillna(data[v].mode() [0])
 data = pd.get_dummies(data, columns=[v], drop_first=True)
<define_variables>
for i in df_cat.columns:
print('The unique values in '+i+' are: ',df[i].nunique() ,'
',df_cat[i].unique() ,'
',"--------------------------------------------------------------------------------" )
categorical_variables_int = \
['OverallQual', 'OverallCond', 'MoSold']<define_variables>
print(df.isnull().sum().sum(axis=0))
numerical_variavles = \
['MSSubClass', 'LotFrontage', 'LotArea', 'YearBuilt',
'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2',
'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF',
'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath',
'FullBath', 'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr',
'TotRmsAbvGrd', 'Fireplaces', 'GarageYrBlt', 'GarageCars',
'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch',
'3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal', 'YrSold']
ss = StandardScaler()
for v in numerical_variavles:
data[v] = data[v].fillna(data[v].mean())
 data[v] = ss.fit_transform(data[[v]] )<prepare_x_and_y>
temp=df.y.values
df_cat['y']=temp
print(df_cat.head() )
train = data[:1460]
test = data[1460:]<features_selection>
print(( df.loc[df.y>150,'y'].values))
df=df[df.y<150]
print("Removing outliers based on above information and setting 150 as a threshold value.................... ")
 print(df.shape)
 df_cat=df_cat[df_cat.y<150]
df_num=df_num[df_num.y<150]
possible_features = train.columns.copy().drop('SalePrice' ).drop('Id')
 selector = SelectKBest(f_regression, len(possible_features))
selector.fit(train[possible_features], train['SalePrice'])
 scores = -np.log10(selector.pvalues_)
 indices = np.argsort(scores)[::-1]
print('Feature importances:')
 for i in range(len(scores)) :
print('%.2f %s' %(scores[indices[i]], possible_features[indices[i]]))<train_model>
rcParams['figure.figsize'] = 15, 7
res = stats.probplot(df['y'], plot=plt )
fparams = \
['OverallQual', 'GrLivArea', 'GarageCars', 'GarageArea', 'TotalBsmtSF',
'1stFlrSF', 'ExterQual_TA', 'FullBath', 'TotRmsAbvGrd', 'YearBuilt',
'KitchenQual_TA', 'GarageFinish_Unf', 'YearRemodAdd', 'BsmtQual_TA',
'Foundation_PConc', 'MasVnrArea', 'GarageYrBlt', 'Fireplaces',
'ExterQual_Gd', 'BsmtFinType1_GLQ', 'Neighborhood_NridgHt', 'BsmtFinSF1',
'MasVnrType_None', 'SaleType_New', 'GarageType_Detchd', 'SaleCondition_Partial',
'Foundation_CBlock', 'LotFrontage', 'MasVnrType_Stone', 'Neighborhood_NoRidge',
'WoodDeckSF', 'KitchenQual_Gd', 'BsmtExposure_No', '2ndFlrSF', 'OpenPorchSF',
'HeatingQC_TA', 'BsmtExposure_Gd', 'Exterior2nd_VinylSd', 'Exterior1st_VinylSd',
'MSZoning_RM', 'HalfBath', 'LotShape_Reg', 'LotArea', 'CentralAir_Y', 'MSZoning_RL',
'HouseStyle_2Story', 'SaleType_WD', 'Electrical_SBrkr', 'RoofStyle_Hip', 'GarageType_BuiltIn',
'BsmtQual_Gd', 'GarageType_Attchd', 'PavedDrive_Y', 'BsmtFullBath', 'RoofStyle_Gable',
'Neighborhood_StoneBr', 'BsmtUnfSF', 'MasVnrType_BrkFace', 'Neighborhood_OldTown',
'Neighborhood_NAmes', 'Neighborhood_Edwards', 'GarageFinish_RFn', 'RoofMatl_WdShngl',
'BedroomAbvGr', 'Exterior1st_MetalSd', 'Neighborhood_IDOTRR', 'Exterior2nd_MetalSd',
'Exterior2nd_Wd Sdng', 'Exterior1st_Wd Sdng', 'KitchenQual_Fa', 'SaleCondition_Normal',
'Neighborhood_BrkSide', 'LotConfig_CulDSac', 'Neighborhood_Somerst', 'ExterCond_Fa',
'GarageCond_TA', 'KitchenAbvGr', 'BsmtFinType1_Rec', 'HeatingQC_Gd', 'HeatingQC_Fa',
'Exterior1st_CemntBd', 'GarageQual_Fa', 'BsmtFinType1_Unf', 'BsmtFinType1_BLQ',
'GarageCond_Fa', 'BsmtQual_Fa', 'EnclosedPorch', 'Neighborhood_Sawyer', 'Exterior2nd_CmentBd',
'Electrical_FuseF', 'Neighborhood_Timber', 'LotShape_IR2', 'LandContour_HLS', 'Foundation_Slab',
'Condition1_Feedr', 'Functional_Typ', 'ExterQual_Fa', 'BldgType_Duplex', 'Condition1_Norm',
'Neighborhood_MeadowV', 'ScreenPorch', 'ExterCond_TA', 'RoofMatl_CompShg', 'Neighborhood_BrDale',
'BldgType_Twnhs', 'BldgType_2fmCon', 'GarageQual_TA', 'Exterior1st_HdBoard', 'HouseStyle_SFoyer',
'Heating_GasA', 'PoolArea', 'Heating_Grav', 'MSZoning_FV', 'BsmtCond_Gd', 'PavedDrive_P',
'HouseStyle_1.5Unf', 'BsmtFinType1_LwQ', 'MSSubClass', 'LotConfig_Inside', 'OverallCond',
'Exterior2nd_ImStucc', 'Neighborhood_CollgCr', 'Functional_Min2', 'Neighborhood_Crawfor',
'GarageType_CarPort', 'Functional_Maj2', 'Exterior2nd_HdBoard']
train_target = train["SalePrice"].values
train_features = train[fparams].values
test_features = test[fparams].values<train_on_grid>
temp = []
for i in df_num.columns:
if df[i].var() ==0:
temp.append(i)
 print(len(temp))
print(temp )
parameters = {
'loss': ['ls','huber'],
'n_estimators' : [1000],
'learning_rate': [0.02,0.03,0.04,0.05],
'max_depth': [2,4,6,8],
'max_features': ['sqrt'],
"min_samples_split": [2,3,4,8,16],
'min_samples_leaf': [2,3,4,8,16]
}
cv = GridSearchCV(GradientBoostingRegressor() , parameters, cv=8, scoring='neg_mean_absolute_error')
 cv.fit(train_features, train_target)
 print("GridSearch score: "+str(cv.best_score_))
print("GridSearch params: ")
 print(cv.best_params_ )<save_to_csv>
count=0
low_var_col=[]
for i in test.columns:
if test[i].dtype == 'int64':
if test[i].var() <0.01:
low_var_col.append(i)
 count+=1
print(count)
 df.drop(low_var_col,axis=1,inplace=True)
 df_num.drop(low_var_col,axis=1,inplace=True)
 test.drop(low_var_col,axis=1,inplace=True )
prediction = cv.best_estimator_.predict(test_features)
 pred = pd.DataFrame(pd.read_csv(".. /input/test.csv")['Id'])
 pred['SalePrice'] = prediction
pred.to_csv(".. /working/submission.csv", index = False )<set_options>
dic={}
for i in df_num.columns:
if i!='y':
if df[i].corr(df.y)>0.25 or df[i].corr(df.y)<-0.25:
dic[i]=df[i].corr(df.y)
 print("Important Features with there respective correlations are ",'
','---------------------------------------------------------','
',dic )
warnings.filterwarnings("ignore" )<load_from_csv>
d = {}; done = []
cols = df.columns.values
for c in cols: d[c]=[]
for i in range(len(cols)) :
if i not in done:
for j in range(i+1, len(cols)) :
if all(df[cols[i]] == df[cols[j]]):
done.append(j)
 d[cols[i]].append(cols[j])
 dub_cols = []
for k in d.keys() :
if len(d[k])> 0:
dub_cols += d[k]
print('Dublicates:','
', dub_cols )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<load_from_csv>
df.drop(['X279','X76','X37','X134','X147','X222','X244','X326'] , axis=1 , inplace=True )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<concatenate>
test.drop(['X279','X76','X37','X134','X147','X222','X244','X326'] , axis=1 , inplace=True)
 df_num.drop(['X279','X76','X37','X134','X147','X222','X244','X326'] , axis=1 , inplace=True )
combine = pd.concat([train,test],sort = True,ignore_index=True )<feature_engineering>
categorical=[]
for i in df.columns:
if df[i].dtype=='object':
le = preprocessing.LabelEncoder()
le.fit(list(df[i].values)+ list(test[i].values))
print("Categories in the encoded order from 1 to the size of "+i+" are : ")
 print(le.classes_)
 print("--------------------------------------------------------------------------")
 df[i] = le.transform(list(df[i].values))
test[i] = le.transform(list(test[i].values))
categorical.append(i )
combine["Surname"] = combine["Name"].str.split(",",expand = True)[0]
combine["Surname"] = combine["Surname"].str.split("-",expand = True)[0]
combine[["Name","Surname"]].head(2 )<feature_engineering>
train_y = df["y"].values
train_X = df.drop(['y'], axis=1)
 def xgb_r2_score(preds, final):
labels = dtrain.get_label()
return 'r2', r2_score(labels, preds)
 xgb_params = {
'n_trees': 520,
'eta': 0.0045,
'max_depth': 4,
'subsample': 0.98,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'base_score': np.mean(train_y),
'silent': 1
}
final = xgb.DMatrix(train_X, train_y, feature_names=train_X.columns.values)
 model = xgb.train(dict(xgb_params), final, num_boost_round=200, feval=xgb_r2_score, maximize=True)
 fig, ax = plt.subplots(figsize=(10,10))
xgb.plot_importance(model, max_num_features=40, height=0.8, ax=ax, color = 'coral')
 print("Feature Importance by XGBoost")
 plt.show()
model = RandomForestRegressor(n_estimators=200, max_depth=10, min_samples_leaf=4, max_features=0.2, n_jobs=-1, random_state=0)
 model.fit(train_X, train_y)
 feat_names = train_X.columns.values
importances = model.feature_importances_
std = np.std([tree.feature_importances_ for tree in model.estimators_], axis=0)
 indices = np.argsort(importances)[::-1][:40]
plt.subplots(figsize=(10,10))
plt.title("Feature importances by RandomForestRegressor")
 plt.ylabel("Features")
 plt.barh(range(len(indices)) , importances[indices], color="green", align="center")
 plt.yticks(range(len(indices)) , feat_names[indices], rotation='horizontal')
 plt.ylim([-1, len(indices)])
 plt.show()
for name in combine.Surname.values:
combine.loc[combine.Surname==name,"Surname_Counts"]=combine[combine.Surname==name].shape[0]
combine[["Surname","Surname_Counts"]].head(2 )<feature_engineering>
df['X314_plus_X315'] = df.apply(lambda row: row.X314 + row.X315, axis=1)
 test['X314_plus_X315'] = test.apply(lambda row: row.X314 + row.X315, axis=1 )
combine['Title'] = combine.Name.str.extract('([A-Za-z]+)\.', expand=False )<groupby>
print("Correalation between X314_plus_X315 and y is : ",df.y.corr(df['X314_plus_X315']))
print("Which makes it pretty much high !! Awesome !!" )
for i,j in combine.groupby(["Sex"]):
j_title_list = set(list(j.Title.values))
print(i,j_title_list )<filter>
survival_1 = combine[(combine.Gender=="Boy")&(combine.Pclass.isin([1,2])) ]["Survived"].mean()
survival_2 = combine[(combine.Sex=="male")&(combine.Age<16)&(combine.Pclass.isin([1,2])) ]["Survived"].mean()
survival_3 = combine[(((combine.Sex=="male")&(combine.Age<16)) |(combine.Gender=="Boy")) &(combine.Pclass.isin([1,2])) ]["Survived"].mean()
print("The Survival probability of method1 is %.2f%%,the survival probability of method2 is %.2f%%,and the survival probability of method3 is %.2f%%"
%(survival_1*100,survival_2*100,survival_3*100))<feature_engineering>
df['X118_plus_X314_plus_X315'] = df.apply(lambda row: row.X118 + row.X314 + row.X315, axis=1)
 test['X118_plus_X314_plus_X315'] = test.apply(lambda row: row.X118 + row.X314 + row.X315, axis=1 )
combine.loc[(combine.Sex=="male")&(combine.Age<16),"Gender"]="Boy"<drop_column>
df["X10_plus_X54"] = df.apply(lambda row: row.X10 + row.X54, axis=1)
 test["X10_plus_X54"] = test.apply(lambda row: row.X10 + row.X54, axis=1)
 print("Correalation between X10_plus_X54 and y is : ",df.y.corr(df['X10_plus_X54']))
combine.drop(["Title","Sex"],axis = 1,inplace=True )<feature_engineering>
df["X10_plus_X29"] = df.apply(lambda row: row.X10 + row.X29, axis=1)
 test["X10_plus_X29"] = test.apply(lambda row: row.X10 + row.X29, axis=1)
 print("Correalation between X10_plus_X29 and y is : ",df.y.corr(df['X10_plus_X29']))
combine_female = combine[combine.Gender == "Woman"]
combine_female.reset_index(inplace = True)
 for i,name in enumerate(list(combine_female.Name.values)) :
name = str(name)
 name = re.sub("\\s$","",name)
 name = re.sub(".*[^\\)]$","",name)
 name = re.sub(".*\\s (.*)\\)$","\\1",name)
 name = re.sub("^\\(","",name)
 combine_female.loc[i,"Maiden_name"] = name
combine_female[["Name","Maiden_name","Age"]].head(3 )<feature_engineering>
train_X['X314_plus_X315']=df['X314_plus_X315']
train_X['X118_plus_X314_plus_X315']=df['X118_plus_X314_plus_X315']
train_X["X10_plus_X54"] = df["X10_plus_X54"]
train_X["X10_plus_X29"] = df["X10_plus_X29"]
for id in combine_female.PassengerId.values:
id2 = int(id)
 name = combine_female[combine_female.PassengerId==id]["Maiden_name"].values
combine.loc[combine.PassengerId==id2,"Maiden"] = name
combine[["Name","Maiden"]].head(2 )<feature_engineering>
booler = np.ones(400)
 for i in same_features:
if booler[int(i[1][1:])]==1:
booler[int(i[1][1:])]=0
df_num.drop(i[1],axis=1,inplace=True)
 df.drop(i[1],axis=1,inplace=True)
 test.drop(i[1],axis=1,inplace=True)
 train_X.drop(i[1],axis=1,inplace=True)
 elif booler[int(i[0][1:])]==1:
booler[int(i[0][1:])]=0
df_num.drop(i[0],axis=1,inplace=True)
 df.drop(i[0],axis=1,inplace=True)
 test.drop(i[0],axis=1,inplace=True)
 train_X.drop(i[0],axis=1,inplace=True )
combine.loc[combine.Maiden=="","Maiden"]=nan<feature_engineering>
for mai in combine.Maiden.values:
combine.loc[combine.Maiden==mai,"Maiden_Counts"]=combine[combine.Maiden==mai].shape[0]
combine[["Maiden","Maiden_Counts"]].head(2 )<feature_engineering>
combine["Family_size"] = combine["Parch"]+combine["SibSp"]+1
combine[["Name","Surname_Counts","Maiden_Counts","Family_size"]].head(2 )<define_variables>
x_train, x_valid, y_train, y_valid = train_test_split(train_X, train_y, test_size=0.2, random_state=420)
 d_train = xgb.DMatrix(x_train, label=y_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid)
 d_test = xgb.DMatrix(test)
 xgb_params = {
'n_trees': 500,
'eta': 0.0050,
'max_depth': 3,
'subsample': 0.95,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'base_score': np.mean(train_y),
'silent': 1
}
def xgb_r2_score(preds, dtrain):
labels = dtrain.get_label()
return 'r2', r2_score(labels, preds)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
clf = xgb.train(xgb_params, d_train, 1050 , watchlist, early_stopping_rounds=70, feval=xgb_r2_score, maximize=True, verbose_eval=10 )
pclass_gender_list = []
for pclass,gender in zip(combine.Pclass.values,combine.Gender.values):
pclass_gender = "P"+str(pclass)+"-"+str(gender)
 pclass_gender_list = pclass_gender_list+[pclass_gender]<feature_engineering>
d_train = xgb.DMatrix(train_X, label=train_y)
 d_test = xgb.DMatrix(test)
 xgb_params = {
'n_trees': 500,
'eta': 0.0050,
'max_depth': 3,
'subsample': 0.95,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'base_score': np.mean(train_y),
'silent': 1
}
def xgb_r2_score(preds, dtrain):
labels = dtrain.get_label()
return 'r2', r2_score(labels, preds)
 watchlist = [(d_train, 'train')]
clf = xgb.train(xgb_params, d_train, 1050 , watchlist, early_stopping_rounds=70, feval=xgb_r2_score, maximize=True, verbose_eval=10 )
<count_values><EOS> 
Answer = clf.predict(d_test)
 sub = pd.DataFrame()
sub['ID'] = test.ID
sub['y'] = Answer
sub.to_csv('mercedes_benz_The_best_or_Nothing.csv', index=False )