Kaggle competition: House Prices - Advanced Regression Techniques
def check_swap_day(data, family, choice):
data_copy = data.copy()
data_copy.loc[family, 'assigned_day'] = data_copy.loc[family, 'choice_'+str(choice)]
data_copy.loc[family, 'penalty_cost'] = calc_family_costs(data_copy.iloc[family])
 penalty_before = data.loc[family, 'penalty_cost']
accounting_before = calc_accounting_cost(data)
 penalty_after = data_copy.loc[family, 'penalty_cost']
accounting_after = calc_accounting_cost(data_copy)
 day_before = check_day(data_copy, data.loc[family, 'assigned_day'])
 day_after = check_day(data_copy, data_copy.loc[family, 'assigned_day'])
 if(day_before==True and day_after==True):
improvement =(penalty_before-penalty_after)+(accounting_before-accounting_after)
 else:
improvement = -1
return improvement<drop_column>
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
import seaborn as sns
from sklearn.preprocessing import PowerTransformer
family_id = 386
check_swap_day(data, family_id, 0 )<create_dataframe>
train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test_x = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv")
 train.columns
def check_swap_family(data, family, choice):
family1 = family
day_family1 = data.loc[family1, 'assigned_day']
penalty1 = data.loc[family1, 'penalty_cost']
member_family1 = data.loc[family1, 'n_people']
day_member_list = data.groupby('assigned_day')['family_id'].apply(list ).to_frame()
improvements = {}
for member in day_member_list.loc[data.loc[family1, 'choice_'+str(choice)], 'family_id']:
family2 = member
day_family2 = data.loc[family2, 'assigned_day']
member_family2 = data.loc[family2, 'n_people']
penalty2 = data.loc[family2, 'penalty_cost']
data_copy = data.copy()
data_copy.loc[family2, 'assigned_day'] = data_copy.loc[family1, 'assigned_day']
data_copy.loc[family1, 'assigned_day'] = data_copy.loc[family1, 'choice_'+str(choice)]
new_penalty1 = calc_family_costs(data_copy.iloc[family1])
 new_penalty2 = calc_family_costs(data_copy.iloc[family2])
 day_before = check_day(data_copy, data.loc[family1, 'assigned_day'])
 day_after = check_day(data_copy, data_copy.loc[family1, 'choice_'+str(choice)])
 accounting_before = calc_accounting_cost(data)
 accounting_after = calc_accounting_cost(data_copy)
 if(day_before==True and day_after==True):
improvement =(penalty1-new_penalty1)+(penalty2-new_penalty2)+(accounting_before-accounting_after)
 else:
improvement = -1
improvements.update({member:improvement})
 maximum = max(zip(improvements.values() , improvements.keys()))
family_swap = maximum[1]
return improvement, family_swap<concatenate>
train_x = train.drop("SalePrice",axis=1)
 train_y = train["SalePrice"]
family_id = 386
check_swap_family(data, family_id, 0 )<normalization>
all_data = pd.concat([train_x,test_x],axis=0,sort=True )
family_id = 386
choice = 0
improvement_day = check_swap_day(data, family_id, choice)
 improvement_family, family_swap = check_swap_family(data, family_id, choice)
 improvement_day, improvement_family, family_swap<categorify>
train_ID = train['Id']
test_ID = test_x['Id']
all_data.drop("Id", axis = 1, inplace = True )
def go_to_bazaar(data, family):
family1 = family
day_family1 = data.loc[family1, 'assigned_day']
penalty1 = data.loc[family1, 'penalty_cost']
member_family1 = data.loc[family1, 'n_people']
status = False
for choice in range(10):

 improvement_day = check_swap_day(data, family1, choice)

improvement_family, family2 = check_swap_family(data, family1, choice)
 if(improvement_day >= 0 or improvement_family >= 0):
if(improvement_day > improvement_family):
data.loc[family, 'assigned_day'] = data.loc[family, 'choice_'+str(choice)]
data.loc[family, 'penalty_cost'] = calc_family_costs(data.iloc[family])
 status = True
else:
data.loc[family2, 'assigned_day'] = data.loc[family1, 'assigned_day']
data.loc[family1, 'assigned_day'] = data.loc[family1, 'choice_'+str(choice)]
data.loc[family1, 'penalty_cost'] = calc_family_costs(data.iloc[family1])
 data.loc[family2, 'penalty_cost'] = calc_family_costs(data.iloc[family2])
 status = True
if(status==True):
break<sort_values>
all_data_na = all_data.isnull().sum() [all_data.isnull().sum() >0].sort_values(ascending=False)
 all_data_na
results = results.reindex(sorted(results.columns), axis=1 )<compute_test_metric>
na_col_list = all_data.isnull().sum() [all_data.isnull().sum() >0].index.tolist()
all_data[na_col_list].dtypes.sort_values()
print('Total costs:', data['penalty_cost'].sum() + calc_accounting_cost(data))<save_to_csv>
all_data['LotFrontage'] = all_data.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
float_list = all_data[na_col_list].dtypes[all_data[na_col_list].dtypes == "float64"].index.tolist()
obj_list = all_data[na_col_list].dtypes[all_data[na_col_list].dtypes == "object"].index.tolist()
all_data[float_list] = all_data[float_list].fillna(0)
 all_data[obj_list] = all_data[obj_list].fillna("None")
 all_data.isnull().sum() [all_data.isnull().sum() > 0]
data = data.sort_index()
output = pd.DataFrame({'family_id': samp_subm.index,
'assigned_day': data['assigned_day']})
 output.to_csv('submission.csv', index=False )<load_from_csv>
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )
santa_workshop_tour_start_solution_01 = pd.read_csv(".. /input/santa-workshop-tour-start-solution/santa_workshop_tour_start_solution_01.csv" )<set_options>
skewed_feats_over = skewed_feats[abs(skewed_feats)> 0.5].index
for i in skewed_feats_over:
print(min(all_data[i]))
from itertools import product
import random
import numpy as np
import pandas as pd
from numba import njit
import matplotlib.pylab as plt
import seaborn as sns
from ortools.linear_solver import pywraplp 
<init_hyperparams>
all_data["FeetPerRoom"] = all_data["TotalSF"]/all_data["TotRmsAbvGrd"]
all_data['YearBuiltAndRemod']=all_data['YearBuilt']+all_data['YearRemodAdd']
all_data['Total_Bathrooms'] =(all_data['FullBath'] +(0.5 * all_data['HalfBath'])+
all_data['BsmtFullBath'] +(0.5 * all_data['BsmtHalfBath']))
all_data['Total_porch_sf'] =(all_data['OpenPorchSF'] + all_data['3SsnPorch'] +
all_data['EnclosedPorch'] + all_data['ScreenPorch'] +
all_data['WoodDeckSF'])
 all_data['haspool'] = all_data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['has2ndfloor'] = all_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasgarage'] = all_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasbsmt'] = all_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasfireplace'] = all_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )
def get_penalty(n, choice):
penalty = None
if choice == 0:
penalty = 0
elif choice == 1:
penalty = 50
elif choice == 2:
penalty = 50 + 9 * n
elif choice == 3:
penalty = 100 + 9 * n
elif choice == 4:
penalty = 200 + 9 * n
elif choice == 5:
penalty = 200 + 18 * n
elif choice == 6:
penalty = 300 + 18 * n
elif choice == 7:
penalty = 300 + 36 * n
elif choice == 8:
penalty = 400 + 36 * n
elif choice == 9:
penalty = 500 + 36 * n + 199 * n
else:
penalty = 500 + 36 * n + 398 * n
return penalty
def GetPreferenceCostMatrix(data):
cost_matrix = np.zeros(( N_FAMILIES, N_DAYS), dtype=np.int64)
 for i in range(N_FAMILIES):
desired = data.values[i, :-1]
cost_matrix[i, :] = get_penalty(FAMILY_SIZE[i], 9)
 for j, day in enumerate(desired):
cost_matrix[i, day-1] = get_penalty(FAMILY_SIZE[i], j)
 return cost_matrix
def GetAccountingCostMatrix() :
ac = np.zeros(( 1000, 1000), dtype=np.float64)
 for n in range(ac.shape[0]):
for n_p1 in range(ac.shape[1]):
diff = abs(n - n_p1)
 ac[n, n_p1] = max(0,(n - 125)/ 400 * n**(0.5 + diff / 50.0))
return ac<set_options>
all_data.dtypes.value_counts()
label_size = 22
mpl.rcParams['xtick.labelsize'] = label_size
mpl.rcParams['ytick.labelsize'] = label_size
sns.set_style("whitegrid")
 ax = plt.figure(figsize=(16,8))
for patch in ax.artists:
r, g, b, a = patch.get_facecolor()
patch.set_facecolor(( r, g, b,.9))
ax = sns.lineplot(x="Family_Size", y="Penalty", data=penalty_values,linewidth=4,hue="Choice",color='
marker='o',markersize=14,palette="Oranges")
 plt.title("Penalty
", loc="center",size=32,color='
plt.xlabel('Family Size',color='
plt.ylabel('Penalty',color='
plt.xticks(size=15,color='
plt.yticks(size=15,color='
plt.ylim(0,2500)
 plt.legend(loc='best', labels=['Choice 0','Choice 1','Choice 2','Choice 3','Choice 4','Choice 5','Choice 6','Choice 7',
'Choice 8','Choice 9'],
handlelength=6,fontsize=10,ncol=2,framealpha=0.99 )<drop_column>
all_data = pd.get_dummies(all_data,columns=cal_list)
 all_data.shape
del penalty_values<compute_train_metric>
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
from sklearn.model_selection import GridSearchCV
import lightgbm as lgb
@njit(fastmath=True)
 def pcost(prediction):
daily_occupancy = np.zeros(N_DAYS+1, dtype=np.int64)
 penalty = 0
for(i, p)in enumerate(prediction):
n = FAMILY_SIZE[i]
penalty += PCOSTM[i, p]
daily_occupancy[p] += n
return penalty, daily_occupancy
@njit(fastmath=True)
 def acost(daily_occupancy):
accounting_cost = 0
n_out_of_range = 0
daily_occupancy[-1] = daily_occupancy[-2]
for day in range(N_DAYS):
n_p1 = daily_occupancy[day + 1]
n = daily_occupancy[day]
n_out_of_range +=(n > MAX_OCCUPANCY)or(n < MIN_OCCUPANCY)
 accounting_cost += ACOSTM[n, n_p1]
return accounting_cost, n_out_of_range
@njit(fastmath=True)
 def acostd(daily_occupancy):
accounting_cost = np.zeros(N_DAYS, dtype=np.float64)
 n_out_of_range = 0
daily_occupancy[-1] = daily_occupancy[-2]
for day in range(N_DAYS):
n_p1 = daily_occupancy[day + 1]
n = daily_occupancy[day]
n_out_of_range +=(n > MAX_OCCUPANCY)or(n < MIN_OCCUPANCY)
 accounting_cost[day] = ACOSTM[n, n_p1]
return accounting_cost, n_out_of_range
@njit(fastmath=True)
 def pcostd(prediction):
daily_occupancy = np.zeros(N_DAYS+1, dtype=np.int64)
 penalty = np.empty_like(prediction)
 for(i, p)in enumerate(prediction):
n = FAMILY_SIZE[i]
penalty[i] = PCOSTM[i, p]
daily_occupancy[p] += n
return penalty, daily_occupancy
@njit(fastmath=True)
 def cost_stats(prediction):
penalty, daily_occupancy = pcostd(prediction)
 accounting_cost, n_out_of_range = acostd(daily_occupancy)
 return penalty, accounting_cost, n_out_of_range, daily_occupancy[:-1]
@njit(fastmath=True)
 def cost_function(prediction):
penalty, daily_occupancy = pcost(prediction)
 accounting_cost, n_out_of_range = acost(daily_occupancy)
 return penalty + accounting_cost + n_out_of_range*100000000<compute_train_metric>
train_x, valid_x, train_y, valid_y = train_test_split(
 train_x,
train_y,
test_size=0.15,
random_state=0 )
@njit(fastmath=True)
 def cost_function_(prediction):
penalty, daily_occupancy = pcost(prediction)
 accounting_cost, n_out_of_range = acost(daily_occupancy)
 return penalty + accounting_cost, n_out_of_range
@njit(fastmath=True)
 def findAnotherDay4Fam(prediction, fam, occupancy):
old_day = prediction[fam]
best_cost = np.inf
best_day = fam
n = FAMILY_SIZE[fam]
daysrange = list(range(0,old_day)) +list(range(old_day+1,N_DAYS))
for day in daysrange:
prediction[fam] = day
new_cost, _ = cost_function_(prediction)
 if(new_cost<best_cost)and(occupancy[day]+n<=MAX_OCCUPANCY):
best_cost = new_cost
best_day = day
prediction[fam] = old_day
return best_day, best_cost
@njit(fastmath=True)
 def bestFamAdd(prediction, day, occupancy):
best_cost = np.inf
best_fam = prediction[day]
for fam in np.where(prediction!=day)[0]:
old_day = prediction[fam]
prediction[fam] = day
new_cost, _ = cost_function_(prediction)
 prediction[fam] = old_day
n = FAMILY_SIZE[fam]
if(new_cost<best_cost)and(occupancy[old_day]-n>=MIN_OCCUPANCY):
best_cost = new_cost
best_fam = fam
return best_fam
@njit(fastmath=True)
 def bestFamRemoval(prediction, day, occupancy):
best_cost = np.inf
best_day = day
for fam in np.where(prediction==day)[0]:
new_day, new_cost = findAnotherDay4Fam(prediction, fam, occupancy)
 if new_cost<best_cost:
best_cost = new_cost
best_fam = fam
best_day = new_day
return best_fam, best_day
@njit(fastmath=True)
 def fixMaxOccupancy(prediction):
penalty, accounting_cost, n_out_of_range, occupancy = cost_stats(prediction)
 for day in np.where(occupancy>MAX_OCCUPANCY)[0]:
while occupancy[day]>MAX_OCCUPANCY:
fam, new_day = bestFamRemoval(prediction, day, occupancy)
 prediction[fam] = new_day
penalty, accounting_cost, n_out_of_range, occupancy = cost_stats(prediction)
 @njit(fastmath=True)
 def fixMinOccupancy(prediction):
penalty, accounting_cost, n_out_of_range, occupancy = cost_stats(prediction)
 for day in np.where(occupancy<MIN_OCCUPANCY)[0]:
while occupancy[day]<MIN_OCCUPANCY:
fam = bestFamAdd(prediction, day, occupancy)
 prediction[fam] = day
penalty, accounting_cost, n_out_of_range, occupancy = cost_stats(prediction )<compute_test_metric>
dtrain = xgb.DMatrix(train_x, label=train_y)
 dvalid = xgb.DMatrix(valid_x,label=valid_y)
 num_round = 5000
evallist = [(dvalid, 'eval'),(dtrain, 'train')]
evals_result = {}
param = {
'max_depth': 3,
'eta': 0.01,
'objective': 'reg:squarederror',
}
bst = xgb.train(
 param, dtrain,
num_round,
evallist,
evals_result=evals_result,
early_stopping_rounds=1000
)
<feature_engineering><EOS> 
dtest = xgb.DMatrix(test_x)
 my_submission = pd.DataFrame()
my_submission["Id"] = test_ID
my_submission["SalePrice"] = np.exp(bst.predict(dtest))
my_submission.to_csv('submission.csv', index=False )