Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<define_variables>
%matplotlib inline
warnings.filterwarnings(action="ignore")
 input_path1 = '.. /input/house-prices-advanced-regression-techniques/'
input_path2 = '.. /input/ames-housing-dataset/'
order_startline = 2*(n_warehouses+1)+2
print(end_wh, order_startline)
 assert order_startline == end_wh
n_order = line_list[end_wh][0]
print("There is : ", n_order, " orders")
 order_delivery = []
order_n_items = []
order_quantity_items = []
print(3*(n_order)+(order_startline+1))
end_order = 3*(n_order)+(order_startline+1)
 for i in np.arange(order_startline+1, 3*(n_order)+(order_startline+1), 3):
order_delivery.append(line_list[i])
 order_n_items.append(line_list[i+1][0])
 order_quantity_items.append(line_list[i+2])
 end_o = i+3
print(len(order_quantity_items))
assert n_order == len(order_n_items)
 assert n_order == len(order_delivery)
 assert n_order == len(order_quantity_items)
 assert order_n_items[32] == len(order_quantity_items[32] )<create_dataframe>
house_data = pd.read_csv(input_path2 + 'AmesHousing.csv')
 test = pd.read_csv(input_path1 + 'test.csv')
 data_w = house_data.copy()
data_w.columns = data_w.columns.str.replace(' ', '')
 data_w.info()
<prepare_x_and_y>
target = data_w['SalePrice']
test_id = test['Id']
test = test.drop(['Id'],axis = 1)
 data_w2 = data_w.drop(['SalePrice','Order','PID'], axis = 1)
 train_test = pd.concat([data_w2,test], axis=0, sort=False )
d_wh = {
"warehouse" : [i for i in range(0, n_warehouses)],
"x" : [i[0] for i in warehouses],
"y" : [i[1] for i in warehouses],
}
for i in range(n_products):
p_id = 'prod_'+str(i)
 d_wh[p_id] = []
for j in range(n_warehouses):
for i in range(n_products):
p_id = 'prod_'+str(i)
 d_wh[p_id].append(wh_stocks[j][i])
 df_wh = pd.DataFrame(d_wh)
 df_wh.head()<prepare_x_and_y>
nan = pd.DataFrame(train_test.isna().sum() , columns = ['NaN_sum'])
 nan['feat'] = nan.index
nan['Perc(%)'] =(nan['NaN_sum']/1460)*100
nan = nan[nan['NaN_sum'] > 0]
nan = nan.sort_values(by = ['NaN_sum'])
 nan['Usability'] = np.where(nan['Perc(%)'] > 20, 'Discard', 'Keep')
 nan
d_order = {
"order" : [i for i in range(0, n_order)],
"items" : order_n_items,
"total_weight" : [0]*n_order,
"x": [order[0] for order in order_delivery],
"y" : [order[1] for order in order_delivery],
}
for i in range(n_products):
p_id = 'prod_'+str(i)
 d_order[p_id] = [0]*n_order
for j in range(0, n_warehouses):
d_order["wh_"+str(j)] = [0]*n_order
d_order["dist_"+str(j)] = [0]*n_order
for k in range(n_order):
w = 0
for i in order_quantity_items[k]:
p_id = 'prod_'+str(i)
 d_order[p_id][k] += 1
w += product_weight[i]
for j in range(0, n_warehouses):
if df_wh[p_id][j] > 0:
d_order["wh_"+str(j)][k] += 1
else:
d_order["wh_"+str(j)][k] += 0
for j in range(0, n_warehouses):
d_order["dist_"+str(j)][k] = distance(d_order['x'][k], d_order['y'][k],
df_wh['x'][j], df_wh['y'][j])
 d_order["total_weight"][k] += w
df_order = pd.DataFrame(d_order)
 df_order['nearest_wh'] = df_order[['dist_'+str(i)for i in range(0, n_warehouses)]].T.idxmin().apply(lambda x: x.split('_')[1])
 df_order.head()<sort_values>
train_test['MSSubClass'] = train_test['MSSubClass'].apply(str)
 train_test['YrSold'] = train_test['YrSold'].apply(str)
 train_test['MoSold'] = train_test['MoSold'].apply(str)
 train_test['Functional'] = train_test['Functional'].fillna('Typ')
 train_test['Electrical'] = train_test['Electrical'].fillna("SBrkr")
 train_test['KitchenQual'] = train_test['KitchenQual'].fillna("TA")
 train_test['Exterior1st'] = train_test['Exterior1st'].fillna(train_test['Exterior1st'].mode() [0])
 train_test['Exterior2nd'] = train_test['Exterior2nd'].fillna(train_test['Exterior2nd'].mode() [0])
 train_test['SaleType'] = train_test['SaleType'].fillna(train_test['SaleType'].mode() [0])
 train_test["PoolQC"] = train_test["PoolQC"].fillna("None")
 train_test["Alley"] = train_test["Alley"].fillna("None")
 train_test['FireplaceQu'] = train_test['FireplaceQu'].fillna("None")
 train_test['Fence'] = train_test['Fence'].fillna("None")
 train_test['MiscFeature'] = train_test['MiscFeature'].fillna("None")
 for col in('GarageArea', 'GarageCars'):
train_test[col] = train_test[col].fillna(0)
 for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
train_test[col] = train_test[col].fillna('None')
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
train_test[col] = train_test[col].fillna('None')
 for col in train_test:
if train_test[col].isna().sum() > 0:
print(train_test[col][0] )
n = 1150
order_quantity_items[n].sort()
order_quantity_items[n]
for p in order_quantity_items[n]:
assert int(df_order[df_order.order == n]['prod_'+str(p)])> 0<sort_values>
useless = ['GarageYrBlt','YearRemodAdd']
train_test = train_test.drop(useless, axis = 1)
 def impute_knn(df):
ttn = train_test.select_dtypes(include=[np.number])
 ttc = train_test.select_dtypes(exclude=[np.number])
 cols_nan = ttn.columns[ttn.isna().any() ].tolist()
cols_no_nan = ttn.columns.difference(cols_nan ).values
for col in cols_nan:
imp_test = ttn[ttn[col].isna() ]
imp_train = ttn.dropna()
model = KNeighborsRegressor(n_neighbors=5)
 knr = model.fit(imp_train[cols_no_nan], imp_train[col])
 ttn.loc[ttn[col].isna() , col] = knr.predict(imp_test[cols_no_nan])
 return pd.concat([ttn,ttc],axis=1)
 train_test = impute_knn(train_test)
 objects = []
for i in train_test.columns:
if train_test[i].dtype == object:
objects.append(i)
 train_test.update(train_test[objects].fillna('None'))
for col in train_test:
if train_test[col].isna().sum() > 0:
print(train_test[col][0] )
%%time
for i in range(0, 10):
print(df_order[['order', 'items', 'wh_'+str(i), 'dist_'+str(i), 'nearest_wh']].query("nearest_wh == "+str(i)+" and wh_"+str(i)+" == items" ).shape )<define_variables>
train_test["SqFtPerRoom"] = train_test["GrLivArea"] /(train_test["TotRmsAbvGrd"] +
train_test["FullBath"] +
train_test["HalfBath"] +
train_test["KitchenAbvGr"])
 train_test['Total_Home_Quality'] = train_test['OverallQual'] + train_test['OverallCond']
train_test['Total_Bathrooms'] =(train_test['FullBath'] +(0.5 * train_test['HalfBath'])+
train_test['BsmtFullBath'] +(0.5 * train_test['BsmtHalfBath']))
train_test["HighQualSF"] = train_test["1stFlrSF"] + train_test["2ndFlrSF"]
train_test['MSSubClass'] = train_test['MSSubClass'].apply(str)
 train_test['YrSold'] = train_test['YrSold'].apply(str)
 train_test['MoSold'] = train_test['MoSold'].apply(str)
 train_test_dummy = pd.get_dummies(train_test)
 numeric_features = train_test_dummy.dtypes[train_test_dummy.dtypes != object].index
skewed_features = train_test_dummy[numeric_features].apply(lambda x: skew(x)).sort_values(ascending=False)
 high_skew = skewed_features[skewed_features > 0.5]
skew_index = high_skew.index
for i in skew_index:
train_test_dummy[i] = np.log1p(train_test_dummy[i] )
%%time
count = 0
for i in range(0, 10):
n = df_order[['order', 'items', 'wh_'+str(i), 'dist_'+str(i), 'nearest_wh']].query("wh_"+str(i)+" == items" ).shape[0]
print('Number of order which could be supplid by warehouse '+str(i)+' only: ',n,' / '+str(n_order))
count += n
print("
By assigning orders to warehouses we could supply ", count, ' / '+str(n_order), '
' )<import_modules>
import shap
import xgboost as xgb
from catboost import Pool
from sklearn.svm import SVR
from catboost import CatBoostRegressor
from lightgbm import LGBMRegressor
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeRegressor
from mlxtend.regressor import StackingRegressor
from sklearn.linear_model import LinearRegression, BayesianRidge
from sklearn.model_selection import RepeatedKFold
from sklearn.model_selection import KFold, cross_val_score
from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from sklearn.metrics import mean_squared_error, mean_absolute_error, mean_squared_log_error
! pip install pulp 
<feature_engineering>
train = train_test_dummy[0:2930]
test = train_test_dummy[2930:]
test['Id'] = test_id
def rmse(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model):
rmse = np.sqrt(-cross_val_score(model, train, target_log, scoring="neg_mean_squared_error", cv=kf))
return(rmse )
stock = [[0]*n_products]*n_warehouses
for wh in range(n_warehouses):
for prod in range(n_products):
stock[wh][prod] = wh_stocks[wh][prod]
stock = np.array(stock)
 demand = np.array([[0]*n_order]*n_products)
 for order in range(n_order):
for prod in order_quantity_items[order]:
demand[prod][order] += 1
demand = np.array(demand)
 x = [[[LpVariable("x_"+str(k)+"_"+str(i)+"_"+str(j), 0, min(1, demand[k][i]), LpInteger)
 for j in range(n_warehouses)]
for i in range(n_order)]
for k in range(n_products)]
cols0 = ['dist_'+str(j)for j in range(n_warehouses)]
dist = np.array(df_order[cols0] ).astype('float32' )<feature_engineering>
kf = KFold(n_splits=10, random_state=42, shuffle=True)
 cv_scores = []
cv_std = []
baseline_models = ['Linear_Reg.','Bayesian_Ridge_Reg.','LGBM_Reg.','SVR',
'Dec_Tree_Reg.','Random_Forest_Reg.', 'XGB_Reg.',
'Grad_Boost_Reg.','Cat_Boost_Reg.','Stacked_Reg.']
lreg = LinearRegression()
score_lreg = cv_rmse(lreg)
 cv_scores.append(score_lreg.mean())
 cv_std.append(score_lreg.std())
 brr = BayesianRidge(compute_score=True)
 score_brr = cv_rmse(brr)
 cv_scores.append(score_brr.mean())
 cv_std.append(score_brr.std())
 l_gbm = LGBMRegressor(objective='regression')
 score_l_gbm = cv_rmse(l_gbm)
 cv_scores.append(score_l_gbm.mean())
 cv_std.append(score_l_gbm.std())
 svr = SVR()
score_svr = cv_rmse(svr)
 cv_scores.append(score_svr.mean())
 cv_std.append(score_svr.std())
 dtr = DecisionTreeRegressor()
score_dtr = cv_rmse(dtr)
 cv_scores.append(score_dtr.mean())
 cv_std.append(score_dtr.std())
 rfr = RandomForestRegressor()
score_rfr = cv_rmse(rfr)
 cv_scores.append(score_rfr.mean())
 cv_std.append(score_rfr.std())
 xgb = xgb.XGBRegressor()
score_xgb = cv_rmse(xgb)
 cv_scores.append(score_xgb.mean())
 cv_std.append(score_xgb.std())
 gbr = GradientBoostingRegressor()
score_gbr = cv_rmse(gbr)
 cv_scores.append(score_gbr.mean())
 cv_std.append(score_gbr.std())
 catb = CatBoostRegressor()
score_catb = cv_rmse(catb)
 cv_scores.append(score_catb.mean())
 cv_std.append(score_catb.std())
 stack_gen = StackingRegressor(regressors=(CatBoostRegressor() ,
LinearRegression() ,
BayesianRidge() ,
GradientBoostingRegressor()),
meta_regressor = CatBoostRegressor() ,
use_features_in_secondary = True)
 score_stack_gen = cv_rmse(stack_gen)
 cv_scores.append(score_stack_gen.mean())
 cv_std.append(score_stack_gen.std())
 final_cv_score = pd.DataFrame(baseline_models, columns = ['Regressors'])
 final_cv_score['RMSE_mean'] = cv_scores
final_cv_score['RMSE_std'] = cv_std
stock = [[0]*n_products]*n_warehouses
for wh in range(n_warehouses):
for prod in range(n_products):
stock[wh][prod] += wh_stocks[wh][prod]
stock = np.array(stock )<compute_test_metric>
X_train,X_val,y_train,y_val = train_test_split(train,target_log,test_size = 0.1,random_state=42)
 cat = CatBoostRegressor()
cat_model = cat.fit(X_train,y_train,
eval_set =(X_val,y_val),
plot=True,
verbose = 0 )
%%time
prob = LpProblem("assign_product_to_order", LpMinimize)
 prob += lpSum(dist[i][j] * x[k][i][j] for k in range(n_products)for i in range(n_order)for j in range(n_warehouses)) , "obj"
for k in range(n_products):
prob += lpSum(x[k][i][j] for i in range(n_order)for j in range(n_warehouses)) <= lpSum(stock[j][k] for j in range(n_warehouses)) , "Stock cst"+str(k)
 for i in range(n_order):
for k in range(n_products):
prob += lpSum(x[k][i][j] for j in range(n_warehouses)) == demand[k][i], "demand cst"+str(k)+"_"+str(i)
 prob.writeLP("ass_order_item_prob.lp")
 print('ok' )<train_model>
cat_pred = cat_model.predict(X_val)
 cat_score = rmse(y_val, cat_pred)
 cat_score
%%time
prob.solve()
print("Status:", LpStatus[prob.status])
 print("objective=", value(prob.objective))<data_type_conversions>
cat_model.get_all_params()
%%time
z = np.array([[0]*n_warehouses]*n_order ).astype('int32')
 df_assign = pd.DataFrame(columns = ['warehouse', 'order', 'product_id', 'quantity', 'total_weight'])
 for k in range(n_products):
for i in range(n_order):
for j in range(n_warehouses):
if x[k][i][j].varValue > 0:
z[i][j] = 1
df_assign = df_assign.append({
'warehouse': j,
'order': i,
'product_id': k,
'quantity' : int(x[k][i][j].varValue),
'total_weight': int(x[k][i][j].varValue)* product_weight[k]
}, ignore_index=True)
 print(df_assign.shape)
 order = 10
tmp = df_assign.sort_values('warehouse' ).query('order == '+str(order))
prods = tmp.product_id
for p in order_quantity_items[10]:
assert np.isin(p, prods)
<feature_engineering>
grid = {'iterations': [1000,6000],
'learning_rate': [0.05, 0.005, 0.0005],
'depth': [4, 6, 10],
'l2_leaf_reg': [1, 3, 5, 9]}
final_model = CatBoostRegressor()
randomized_search_result = final_model.randomized_search(grid,
X = X_train,
y= y_train,
verbose = False,
plot=True)
%%time
for i in range(n_order):
if sum(z[i])> 1:
print("more than 1 wh ",i, ", sum = ", sum(z[i]))
if sum(z[i])== 0:
print("0 wh", i)
 for j in range(n_warehouses):
df_order['assign_'+str(j)] = z.T[j]<sort_values>
params = {'iterations': 6000,
'learning_rate': 0.005,
'depth': 4,
'l2_leaf_reg': 1,
'eval_metric':'RMSE',
'early_stopping_rounds': 200,
'verbose': 200,
'random_seed': 42}
cat_f = CatBoostRegressor(**params)
 cat_model_f = cat_f.fit(X_train,y_train,
eval_set =(X_val,y_val),
plot=True,
verbose = False)
 catf_pred = cat_model_f.predict(X_val)
 catf_score = rmse(y_val, catf_pred )
wh = 1
n_drone_per_wh = n_drones//10
df = df_assign.query("warehouse == "+str(wh)).sort_values('order')
 order_list = list(df.order)
 print(len(order_list))
df.head()<prepare_x_and_y>
test_pred = cat_f.predict(test)
 submission = pd.DataFrame(test_id, columns = ['Id'])
 test_pred = np.expm1(test_pred)
 submission['SalePrice'] = test_pred
submission.head()
%%time
demands = df['total_weight'].tolist() + [0]
products = df['product_id'].tolist() +[0]
len(demands)
 X = np.array(d_order['x'])
 Y = np.array(d_order['y'])
 ref_dist = np.array([[ distance(X[i], Y[i], X[j], Y[j])for j in range(n_order)] for i in range(n_order)])
 tmp = pd.DataFrame(np.array([[ref_dist[i][j] for j in order_list] for i in order_list]))
tmp[tmp.shape[0]] = pd.Series([df_order.iloc[i]['dist_'+str(wh)] for i in order_list]+[0])
 tmp = tmp.append(pd.Series([df_order.iloc[i]['dist_'+str(wh)] for i in order_list]+[0]), ignore_index=True)
 distance_ = np.array(tmp)
 print(distance_.shape)
 distance_<define_variables>
submission.to_csv("result.csv", index = False, header = True )