Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_from_csv>
%config InlineBackend.figure_format = 'retina'
%matplotlib inline
NUMBER_DAYS = 100
NUMBER_FAMILIES = 5000
MAX_BEST_CHOICE = 5
data = pd.read_csv('/kaggle/input/santa-workshop-tour-2019/family_data.csv')
 submission = pd.read_csv('/kaggle/input/c-stochastic-product-search-65ns/submission.csv')
 assigned_days = submission['assigned_day'].values
columns = data.columns[1:11]
DESIRED = data[columns].values
COST_PER_FAMILY = [0,50,50,100,200,200,300,300,400,500]
COST_PER_FAMILY_MEMBER = [0, 0, 9, 9, 9, 18, 18, 36, 36,235]
N_PEOPLE = data['n_people'].astype(int ).values
def get_daily_occupancy(assigned_days):
daily_occupancy = np.zeros(100, np.int32)
 for i, r in enumerate(assigned_days):
daily_occupancy[r-1] += N_PEOPLE[i]
return daily_occupancy
def cost_function(prediction):
N_DAYS = 100
MAX_OCCUPANCY = 300
MIN_OCCUPANCY = 125
penalty = 0
days = list(range(N_DAYS,0,-1))
tmp = pd.read_csv('/kaggle/input/santa-workshop-tour-2019/family_data.csv', index_col='family_id')
 family_size_dict = tmp[['n_people']].to_dict() ['n_people']
cols = [f'choice_{i}' for i in range(10)]
choice_dict = tmp[cols].to_dict()
daily_occupancy = {k:0 for k in days}
for f, d in enumerate(prediction):
n = family_size_dict[f]
choice_0 = choice_dict['choice_0'][f]
choice_1 = choice_dict['choice_1'][f]
choice_2 = choice_dict['choice_2'][f]
choice_3 = choice_dict['choice_3'][f]
choice_4 = choice_dict['choice_4'][f]
choice_5 = choice_dict['choice_5'][f]
choice_6 = choice_dict['choice_6'][f]
choice_7 = choice_dict['choice_7'][f]
choice_8 = choice_dict['choice_8'][f]
choice_9 = choice_dict['choice_9'][f]
daily_occupancy[d] += n
if d == choice_0:
penalty += 0
elif d == choice_1:
penalty += 50
elif d == choice_2:
penalty += 50 + 9 * n
elif d == choice_3:
penalty += 100 + 9 * n
elif d == choice_4:
penalty += 200 + 9 * n
elif d == choice_5:
penalty += 200 + 18 * n
elif d == choice_6:
penalty += 300 + 18 * n
elif d == choice_7:
penalty += 300 + 36 * n
elif d == choice_8:
penalty += 400 + 36 * n
elif d == choice_9:
penalty += 500 + 36 * n + 199 * n
else:
penalty += 500 + 36 * n + 398 * n
for _, v in daily_occupancy.items() :
if v > MAX_OCCUPANCY or v < MIN_OCCUPANCY:
penalty += 100000000
accounting_cost = max(0,(daily_occupancy[days[0]]-125.0)/ 400.0 * daily_occupancy[days[0]]**(0.5))
yesterday_count = daily_occupancy[days[0]]
for day in days[1:]:
today_count = daily_occupancy[day]
diff = abs(today_count - yesterday_count)
 accounting_cost += max(0,(today_count-125.0)/ 400.0 * today_count**(0.5 + diff / 50.0))
yesterday_count = today_count
return penalty, accounting_cost, penalty + accounting_cost<categorify>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
%%time
for num_members in range(2, 9):
daily_occupancy = get_daily_occupancy(assigned_days)
 fids = np.where(N_PEOPLE == num_members)[0]
PCOSTM = {}
for fid in range(NUMBER_FAMILIES):
if fid in fids:
for i in range(MAX_BEST_CHOICE):
PCOSTM[fid, DESIRED[fid][i]-1] = COST_PER_FAMILY[i] + N_PEOPLE[fid] * COST_PER_FAMILY_MEMBER[i]
else:
daily_occupancy[assigned_days[fid]-1] -= N_PEOPLE[fid]
offset = fids.shape[0]
solver = pywrapgraph.SimpleMinCostFlow()
for day in range(NUMBER_DAYS):
solver.SetNodeSupply(offset+day, int(daily_occupancy[day]//num_members))
for i in range(offset):
fid = fids[i]
solver.SetNodeSupply(i, -1)
 for j in range(MAX_BEST_CHOICE):
day = DESIRED[fid][j]-1
solver.AddArcWithCapacityAndUnitCost(int(offset+day), i, 1, int(PCOSTM[fid, day]))
solver.SolveMaxFlowWithMinCost()
for i in range(solver.NumArcs()):
if solver.Flow(i)> 0:
assigned_days[fids[solver.Head(i)]] = solver.Tail(i)- offset + 1
print(cost_function(assigned_days))<save_to_csv>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
submission['assigned_day'] = assigned_days
submission.to_csv('submission.csv', index=False )<define_variables>
all_data = pd.concat(( train.loc[:,'MSSubClass':'SaleCondition'],
test.loc[:,'MSSubClass':'SaleCondition']))
root_path = Path(r'.. /input/santa-workshop-tour-2019')
 best_submit_path = Path(r'.. /input/mip-optimization-preference-cost/')
 MAX_CHOICE = 5<load_from_csv>
train["SalePrice"] = np.log1p(train["SalePrice"])
 numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = train[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index
all_data[skewed_feats] = np.log1p(all_data[skewed_feats] )
fpath = root_path / 'family_data.csv'
data = pd.read_csv(fpath, index_col='family_id')
 data_choices = data.values
fpath = root_path / 'sample_submission.csv'
submission = pd.read_csv(fpath, index_col='family_id' )<categorify>
all_data = pd.get_dummies(all_data )
dummies = []
for i in range(MAX_CHOICE):
tmp = pd.get_dummies(data[f'choice_{i}'] ).values*(data['n_people'].values.reshape(-1,1))
dummies.append((
np.concatenate([tmp, tmp[:, -1].reshape(-1,1)], axis=1)
 ).reshape(5000, 101, 1))
dummies = np.concatenate(dummies, axis=2)
 dummies = np.swapaxes(dummies, 1, 2)
 penalties = {n: [0, 50, 50 + 9 * n, 100 + 9 * n, 200 + 9 * n, 200 + 18 * n, 300 + 18 * n, 300 + 36 * n, 400 + 36 * n, 500 + 36 * n + 199 * n] for n in np.unique(data['n_people'])}
mat = []
for i in range(5000):
n = data.iloc[i]['n_people']
mat.append(penalties[n][:MAX_CHOICE])
 mat = np.array(mat)
 def create_init(initial_sub):
fam_choices = data
a = pd.merge(initial_sub, fam_choices, on='family_id')
 initial_choices = []
for i in range(MAX_CHOICE):
initial_choices.append(((a[f'choice_{i}'] == a['assigned_day'])).values.reshape(-1,1))
initial_choices = np.concatenate(initial_choices, axis=1)
 initial_choices = torch.tensor(
 initial_choices*10
, dtype=torch.float32)
 return initial_choices
initial_sub = pd.read_csv(best_submit_path / 'submission.csv')
 initial_choices = create_init(initial_sub )<categorify>
all_data = all_data.fillna(all_data.mean() )
family_sizes = data.n_people.values.astype(np.int8)
 cost_dict = {0: [ 0, 0],
1: [ 50, 0],
2: [ 50, 9],
3: [100, 9],
4: [200, 9],
5: [200, 18],
6: [300, 18],
7: [300, 36],
8: [400, 36],
9: [500, 36 + 199],
10: [500, 36 + 398],
}
def cost(choice, members, cost_dict):
x = cost_dict[choice]
return x[0] + members * x[1]
all_costs = {k: pd.Series([cost(k, x, cost_dict)for x in range(2,9)], index=range(2,9)) for k in cost_dict.keys() }
df_all_costs = pd.DataFrame(all_costs)
 family_cost_matrix = np.zeros(( 100,len(family_sizes)))
 for i, el in enumerate(family_sizes):
family_cost_matrix[:, i] += all_costs[10][el]
for j, choice in enumerate(data.drop("n_people",axis=1 ).values[i,:]):
family_cost_matrix[choice-1, i] = all_costs[j][el]
def accounting(today, previous):
return(( today - 125)/ 400)* today ** (.5 +(abs(today - previous)/ 50))
acc_costs = np.zeros([176,176])
 for i, x in enumerate(range(125,300+1)) :
for j, y in enumerate(range(125,300+1)) :
acc_costs[i,j] = accounting(x,y)
 @njit(fastmath=True)
 def cost_function(prediction, family_size, family_cost_matrix, accounting_cost_matrix):
N_DAYS = 100
MAX_OCCUPANCY = 300
MIN_OCCUPANCY = 125
penalty = 0
accounting_cost = 0
max_occ = False
daily_occupancy = np.zeros(N_DAYS + 1, dtype=np.int16)
 for i,(pred, n)in enumerate(zip(prediction, family_size)) :
daily_occupancy[pred - 1] += n
penalty += family_cost_matrix[pred - 1, i]
daily_occupancy[-1] = daily_occupancy[-2]
for day in range(N_DAYS):
n_next = daily_occupancy[day + 1]
n = daily_occupancy[day]
max_occ += MIN_OCCUPANCY > n
max_occ += MAX_OCCUPANCY < n
accounting_cost += accounting_cost_matrix[n-MIN_OCCUPANCY, n_next-MIN_OCCUPANCY]
if max_occ:
return 1e11
return penalty+accounting_cost<compute_test_metric>
X_train = all_data[:train.shape[0]]
X_test = all_data[train.shape[0]:]
y = train.SalePrice
cost_function(initial_sub['assigned_day'].values, family_sizes, family_cost_matrix, acc_costs )<categorify>
def rmse_cv(model):
rmse= np.sqrt(-cross_val_score(model, X_train, y, scoring="neg_mean_squared_error", cv = 5))
return(rmse )
class Model(nn.Module):
def __init__(self, mat, dummies):
super().__init__()
self.mat = torch.from_numpy(mat ).type(torch.int16)
 self.dummies = torch.from_numpy(dummies ).type(torch.float32)
 self.weight = torch.nn.Parameter(data=torch.Tensor(5000, MAX_CHOICE ).type(torch.float32)
 , requires_grad=True)
 self.weight.data.copy_(initial_choices)
 def forward(self):
prob = F.softmax(self.weight,dim=1)
 x =(prob * self.mat ).sum()
daily_occupancy = torch.zeros(101, dtype=torch.float32)
 for i in range(MAX_CHOICE):
daily_occupancy +=(prob[:, i]@self.dummies[:, i, :])
 diff = torch.abs(daily_occupancy[:-1] - daily_occupancy[1:])
 daily_occupancy = daily_occupancy[:-1]
y =(
 torch.relu(daily_occupancy-125.0)/ 400.0 * daily_occupancy**(0.5 + diff / 50.0)
 ).sum()
v =(( torch.relu(125-daily_occupancy)) **2+(torch.relu(daily_occupancy-300)) **2 ).sum()
entropy_loss = -1.0 *(prob * F.log_softmax(self.weight, dim=1)).sum()
return x, y, v*10000, entropy_loss<choose_model_class>
model_ridge = Ridge()
model = Model(mat, dummies)
 best_score = 10e10
best_pos = None
optimizer = torch.optim.Adam(model.parameters() , lr = 0.1 )<find_best_params>
alphas = [0.05, 0.1, 0.3, 1, 3, 5, 10, 15, 30, 50, 75]
cv_ridge = [rmse_cv(Ridge(alpha = alpha)).mean()
for alpha in alphas]
for epoch in tqdm.tqdm_notebook(range(1_001)) :
optimizer.zero_grad()
x, y, v, ent = model()
loss = x + y + v + 0*ent
loss.backward()
optimizer.step()
pos = model.weight.argmax(1 ).cpu().numpy()
pred = []
for i in range(5000):
pred.append(data_choices[i, pos[i]])
 pred = np.array(pred)
 score = cost_function(pred, family_sizes, family_cost_matrix, acc_costs)
 if(score < best_score):
best_score = score
best_pos = pred
print(best_score)
 submission['assigned_day'] = best_pos
submission.to_csv(f'submission.csv')
 if epoch % 1000 == 0:
x = np.round(x.item() ,1)
 y = np.round(y.item() ,1)
 print(f'{epoch}\t{x}\t{y} \t{np.round(score, 2)}' )<choose_model_class>
model_lasso = LassoCV(alphas = [1, 0.1, 0.001, 0.0005] ).fit(X_train, y )
prev_best_score = best_score
coef = 1
count_failures = 0
for _ in range(10_000):
initial_sub = pd.read_csv('submission.csv')
 initial_choices = create_init(initial_sub)
 model = Model(mat, dummies)
 optimizer = torch.optim.Adam(model.parameters() , lr = 0.1)
 for epoch in tqdm.tqdm_notebook(range(1_001)) :
optimizer.zero_grad()
x, y, v, ent = model()
loss = x + coef*y + v + 0*ent
loss.backward()
optimizer.step()
pos = model.weight.argmax(1 ).cpu().numpy()
pred = []
for i in range(5000):
pred.append(data_choices[i, pos[i]])
 pred = np.array(pred)
 score = cost_function(pred, family_sizes, family_cost_matrix, acc_costs)
 if(score < best_score):
best_score = score
best_pos = pred
print(best_score)
 submission['assigned_day'] = best_pos
submission.to_csv(f'submission.csv')
 if(epoch % 1000 == 0)and epoch != 0:
x = np.round(x.item() ,1)
 y = np.round(y.item() ,1)
 print(f'{epoch}\t{x}\t{y} \t{np.round(score, 2)}')
 if best_score == prev_best_score:
count_failures += 1
if count_failures > 10:
break
coef = coef*1.05
else:
prev_best_score = best_score
count_failures = 0
coef = 1<choose_model_class>
rmse_cv(model_lasso ).mean()
prev_best_score = best_score
coef = 1
count_failures = 0
for _ in range(10_000):
initial_sub = pd.read_csv('submission.csv')
 initial_choices = create_init(initial_sub)
 model = Model(mat, dummies)
 optimizer = torch.optim.Adam(model.parameters() , lr = 0.1)
 for epoch in tqdm.tqdm_notebook(range(1_001)) :
optimizer.zero_grad()
x, y, v, ent = model()
loss = coef*x + y + v + 10*ent
loss.backward()
optimizer.step()
pos = model.weight.argmax(1 ).cpu().numpy()
pred = []
for i in range(5000):
pred.append(data_choices[i, pos[i]])
 pred = np.array(pred)
 score = cost_function(pred, family_sizes, family_cost_matrix, acc_costs)
 if(score < best_score):
best_score = score
best_pos = pred
print(best_score)
 submission['assigned_day'] = best_pos
submission.to_csv(f'submission.csv')
 if(epoch % 1000 == 0)and epoch != 0:
x = np.round(x.item() ,1)
 y = np.round(y.item() ,1)
 print(f'{epoch}\t{x}\t{y} \t{np.round(score, 2)}')
 if best_score == prev_best_score:
count_failures += 1
if count_failures > 10:
break
coef = coef*1.01
else:
prev_best_score = best_score
count_failures = 0
coef = 1<save_to_csv>
print("Lasso picked " + str(sum(coef != 0)) + " variables and eliminated the other " + str(sum(coef == 0)) + " variables" )
submission['assigned_day'] = best_pos
submission.to_csv(f'submission.csv' )<import_modules>
imp_coef = pd.concat([coef.sort_values().head(10),
coef.sort_values().tail(10)] )
import numpy as np
import pandas as pd
import scipy.special
import matplotlib.pyplot as plt
import os<train_model>
import xgboost as xgb
path_in = '.. /input/santa-workshop-tour-2019/'
path_start_solution = '.. /input/santa-workshop-tour-start-solution/'
print('input_data:', os.listdir(path_in))
print('start solution:', os.listdir(path_start_solution))<load_from_csv>
dtrain = xgb.DMatrix(X_train, label = y)
 dtest = xgb.DMatrix(X_test)
 params = {"max_depth":2, "eta":0.1}
model = xgb.cv(params, dtrain, num_boost_round=500, early_stopping_rounds=100 )
data = pd.read_csv(path_in+'family_data.csv')
 data.index = data['family_id']
samp_subm = pd.read_csv(path_in+'sample_submission.csv')
 start_solution = pd.read_csv(path_start_solution+'santa_workshop_tour_start_solution_01.csv', index_col=0 )<define_variables>
model_xgb = xgb.XGBRegressor(n_estimators=360, max_depth=2, learning_rate=0.1)
 model_xgb.fit(X_train, y )
num_days = 100
lower = 125
upper = 300
days = list(range(num_days, 0, -1))<init_hyperparams>
xgb_preds = np.expm1(model_xgb.predict(X_test))
lasso_preds = np.expm1(model_lasso.predict(X_test))
def calc_family_costs(family):
assigned_day = family['assigned_day']
number_member = family['n_people']
if assigned_day == family['choice_0']:
penalty = 0
elif assigned_day == family['choice_1']:
penalty = 50
elif assigned_day == family['choice_2']:
penalty = 50 + 9 * number_member
elif assigned_day == family['choice_3']:
penalty = 100 + 9 * number_member
elif assigned_day == family['choice_4']:
penalty = 200 + 9 * number_member
elif assigned_day == family['choice_5']:
penalty = 200 + 18 * number_member
elif assigned_day == family['choice_6']:
penalty = 300 + 18 * number_member
elif assigned_day == family['choice_7']:
penalty = 300 + 36 * number_member
elif assigned_day == family['choice_8']:
penalty = 400 + 36 * number_member
elif assigned_day == family['choice_9']:
penalty = 500 + 36 * number_member + 199 * number_member
else:
penalty = 500 + 36 * number_member + 398 * number_member
return penalty<categorify>
predictions = pd.DataFrame({"xgb":xgb_preds, "lasso":lasso_preds})
 predictions.plot(x = "xgb", y = "lasso", kind = "scatter" )
def calc_accounting_cost(data):
accounting_cost = 0
daily_occupancy = {k:0 for k in days}
family_size_dict = data[['n_people']].to_dict() ['n_people']
for f, d in enumerate(data['assigned_day']):
n = family_size_dict[f]
daily_occupancy[d] += n
accounting_cost =(daily_occupancy[days[0]]-125.0)/ 400.0 * daily_occupancy[days[0]]**(0.5)
 accounting_cost = max(0, accounting_cost)
 yesterday_count = daily_occupancy[days[0]]
for day in days[1:]:
today_count = daily_occupancy[day]
diff = abs(today_count - yesterday_count)
 accounting_cost += max(0,(daily_occupancy[day]-125.0)/ 400.0 * daily_occupancy[day]**(0.5 + diff / 50.0))
yesterday_count = today_count
return accounting_cost<create_dataframe>
preds = 0.7*lasso_preds + 0.3*xgb_preds
def check_day(data, day):
group_data = data.groupby('assigned_day' ).sum() ['n_people'].to_frame()
if(125 <= group_data.loc[day, 'n_people'])&(group_data.loc[day, 'n_people'] <= 300):
return True
else:
return False<data_type_conversions>
solution = pd.DataFrame({"id":test.Id, "SalePrice":preds})
 solution.to_csv("ridge_sol.csv", index = False )
for i in range(num_days):
data.loc[i*50:(i+1)*50-1, 'assigned_day'] = i+1
data['assigned_day'] = data['assigned_day'].astype(int )<feature_engineering>
from keras.layers import Dense
from keras.models import Sequential
from keras.regularizers import l1
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
data['assigned_day'] = start_solution['assigned_day']<feature_engineering>
X_train = StandardScaler().fit_transform(X_train )
data['penalty_cost'] = data.apply(calc_family_costs, axis=1 )<compute_test_metric>
X_tr, X_val, y_tr, y_val = train_test_split(X_train, y, random_state = 3 )
acc_costs = calc_accounting_cost(data)
 acc_costs<compute_test_metric>
model = Sequential()
model.add(Dense(1, input_dim = X_train.shape[1], W_regularizer=l1(0.001)))
 model.compile(loss = "mse", optimizer = "adam" )
print('Total costs:', data['penalty_cost'].sum() + acc_costs )<feature_engineering>
hist = model.fit(X_tr, y_tr, validation_data =(X_val, y_val))