Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<load_from_csv>
%matplotlib inline
N_DAYS = 100
N_FAMILIES = 5000
MAX_OCCUPANCY = 300
MIN_OCCUPANCY = 125
data = pd.read_csv('/kaggle/input/santa-workshop-tour-2019/family_data.csv', index_col='family_id')
 FAMILY_SIZE = data.n_people.values
DESIRED = data.values[:, :-1] - 1
PCOSTM = GetPreferenceCostMatrix(data)
 ACOSTM = GetAccountingCostMatrix()<create_dataframe>
train = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/train.csv")
 test = pd.read_csv("/kaggle/input/house-prices-advanced-regression-techniques/test.csv" )
pivot_data = data.groupby(['choice_0'])['n_people'].agg(['sum'])
 pivot_data = pd.DataFrame(pivot_data)
 pivot_data.columns = ['Occupancy for all First Choices']<compute_train_metric>
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))
prediction = solveSantaLP()
penalty, accounting_cost, n_out_of_range, occupancy = cost_stats(prediction)
 print('{}, {:.2f},({}, {})'.format(penalty.sum() ,
accounting_cost.sum() ,
occupancy.min() ,
occupancy.max()))<compute_train_metric>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'欠損比率' :all_data_na})
 missing_data.head(100 )
fixMinOccupancy(prediction)
 fixMaxOccupancy(prediction)
 penalty, accounting_cost, n_out_of_range, occupancy = cost_stats(prediction)
 print('{}, {:.2f},({}, {})'.format(penalty.sum() ,
accounting_cost.sum() ,
occupancy.min() ,
occupancy.max()))<choose_model_class>
all_data["PoolQC"] = all_data["PoolQC"].fillna("None" )
round2 = stochastic_product_search(
 top_k=2,
fam_size=6,
original=prediction,
n_iter=350000,
verbose=1000,
verbose2=50000,
random_state=2019
)<choose_model_class>
all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None" )
round3 = stochastic_product_search(
 top_k=2,
fam_size=7,
original=round2,
n_iter=350000,
verbose=1000,
verbose2=50000,
random_state=2019
)<choose_model_class>
all_data["Alley"] = all_data["Alley"].fillna("None" )
round4 = stochastic_product_search(
 top_k=2,
fam_size=8,
original=round3,
n_iter=350000,
verbose=1000,
verbose2=50000,
random_state=2019
)<choose_model_class>
all_data["Fence"] = all_data["Fence"].fillna("None" )
round5 = stochastic_product_search(
 top_k=2,
fam_size=9,
original=round4,
n_iter=350000,
verbose=1000,
verbose2=50000,
random_state=2019
)<choose_model_class>
all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None" )
final = stochastic_product_search(
 top_k=2,
fam_size=12,
original=round5,
n_iter=150000,
verbose=1000,
verbose2=50000,
random_state=2019
)<save_to_csv>
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
sub = pd.DataFrame(range(N_FAMILIES), columns=['family_id'])
 sub['assigned_day'] = final+1
sub.to_csv('submission_5.csv', index=False )<rename_columns>
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None' )
df0 = data[['choice_0','n_people']]
df0.columns = ['choice_0','n_people']
df0<create_dataframe>
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0 )
df1 = pd.DataFrame([prediction] ).transpose()
df1.columns = ['Prediction']
df1<create_dataframe>
for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0 )
df2 = pd.DataFrame([round2] ).transpose()
df2.columns = ['Round2_Days']
df2<create_dataframe>
for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None' )
df3 = pd.DataFrame([round3] ).transpose()
df3.columns = ['Round3_Days']
df3<create_dataframe>
all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0 )
df4 = pd.DataFrame([round4] ).transpose()
df4.columns = ['Round4_Days']
df4<create_dataframe>
all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0] )
df5 = pd.DataFrame([round5] ).transpose()
df5.columns = ['Round5_Days']
df5<create_dataframe>
all_data = all_data.drop(['Utilities'], axis=1 )
df6 = pd.DataFrame([final] ).transpose()
df6.columns = ['Final_Days']
df6<concatenate>
all_data["Functional"] = all_data["Functional"].fillna("Typ" )
df = pd.concat([df0,df1,df2,df3,df4,df5,df6], axis=1)
 df<create_dataframe>
all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0] )
pivot0 = df.groupby(['choice_0'])['n_people'].agg(['sum'])
 pivot0 = pd.DataFrame(pivot0)
 pivot0.columns = ['First Choice']
pivot1 = df.groupby(['Prediction'])['n_people'].agg(['sum'])
 pivot1 = pd.DataFrame(pivot1)
 pivot1.columns = ['Prediction']
pivot2 = df.groupby(['Round2_Days'])['n_people'].agg(['sum'])
 pivot2 = pd.DataFrame(pivot2)
 pivot2.columns = ['Round 2']
pivot3 = df.groupby(['Round3_Days'])['n_people'].agg(['sum'])
 pivot3 = pd.DataFrame(pivot3)
 pivot3.columns = ['Round 3']
pivot4 = df.groupby(['Round4_Days'])['n_people'].agg(['sum'])
 pivot4 = pd.DataFrame(pivot4)
 pivot4.columns = ['Round 4']
pivot5 = df.groupby(['Round5_Days'])['n_people'].agg(['sum'])
 pivot5 = pd.DataFrame(pivot5)
 pivot5.columns = ['Round 5']
pivot6 = df.groupby(['Final_Days'])['n_people'].agg(['sum'])
 pivot6 = pd.DataFrame(pivot6)
 pivot6.columns = ['Final']<import_modules>
all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0] )
import numpy as np
import pandas as pd
from numba import njit
from itertools import product
from ortools.linear_solver import pywraplp<init_hyperparams>
all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0] )
def get_penalty(n, choice):
penalty = None
if choice == 0:
penalty = 0
elif choice == 1:
penalty = 50
elif choice == 2:
penalty = 50 + 9 * n
elif choice == 3:
penalty = 100 + 9 * n
elif choice == 4:
penalty = 200 + 9 * n
elif choice == 5:
penalty = 200 + 18 * n
elif choice == 6:
penalty = 300 + 18 * n
elif choice == 7:
penalty = 300 + 36 * n
elif choice == 8:
penalty = 400 + 36 * n
elif choice == 9:
penalty = 500 + 36 * n + 199 * n
else:
penalty = 500 + 36 * n + 398 * n
return penalty
def GetPreferenceCostMatrix(data):
cost_matrix = np.zeros(( N_FAMILIES, N_DAYS), dtype=np.int64)
 for i in range(N_FAMILIES):
desired = data.values[i, :-1]
cost_matrix[i, :] = get_penalty(FAMILY_SIZE[i], 10)
 for j, day in enumerate(desired):
cost_matrix[i, day-1] = get_penalty(FAMILY_SIZE[i], j)
 return cost_matrix
def GetAccountingCostMatrix() :
ac = np.zeros(( 1000, 1000), dtype=np.float64)
 for n in range(ac.shape[0]):
for n_p1 in range(ac.shape[1]):
diff = abs(n - n_p1)
 ac[n, n_p1] = max(0,(n - 125)/ 400 * n**(0.5 + diff / 50.0))
return ac
@njit(fastmath=True)
 def pcost(prediction):
daily_occupancy = np.zeros(N_DAYS+1, dtype=np.int64)
 penalty = 0
for(i, p)in enumerate(prediction):
n = FAMILY_SIZE[i]
penalty += PCOSTM[i, p]
daily_occupancy[p] += n
return penalty, daily_occupancy
@njit(fastmath=True)
 def acost(daily_occupancy):
accounting_cost = 0
n_out_of_range = 0
daily_occupancy[-1] = daily_occupancy[-2]
for day in range(N_DAYS):
n_p1 = daily_occupancy[day + 1]
n = daily_occupancy[day]
n_out_of_range +=(n > MAX_OCCUPANCY)or(n < MIN_OCCUPANCY)
 accounting_cost += ACOSTM[n, n_p1]
return accounting_cost, n_out_of_range
@njit(fastmath=True)
 def cost_function(prediction):
penalty, daily_occupancy = pcost(prediction)
 accounting_cost, n_out_of_range = acost(daily_occupancy)
 return penalty + accounting_cost + n_out_of_range*100000000<compute_test_metric>
all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0] )
def findBetterDay4Family(pred):
fobs = np.argsort(FAMILY_SIZE)
 score = cost_function(pred)
 original_score = np.inf
while original_score>score:
original_score = score
for family_id in fobs:
for pick in range(10):
day = DESIRED[family_id, pick]
oldvalue = pred[family_id]
pred[family_id] = day
new_score = cost_function(pred)
 if new_score<score:
score = new_score
else:
pred[family_id] = oldvalue
print(score, end='\r')
 print(score)
 def stochastic_product_search(top_k, fam_size, original,
verbose=1000, verbose2=50000,
n_iter=500, random_state=2019):

 best = original.copy()
best_score = cost_function(best)
 np.random.seed(random_state)
 for i in range(n_iter):
fam_indices = np.random.choice(range(DESIRED.shape[0]), size=fam_size)
 changes = np.array(list(product(*DESIRED[fam_indices, :top_k].tolist())))
 for change in changes:
new = best.copy()
new[fam_indices] = change
new_score = cost_function(new)
 if new_score < best_score:
best_score = new_score
best = new
if verbose and i % verbose == 0:
print(f"Iteration
if verbose2 and i % verbose2 == 0:
print(f"Iteration
print(f"Final best score is {best_score:.2f}")
 return best<compute_train_metric>
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'欠損比率' :all_data_na})
 missing_data.head()
def seed_finding(seed, prediction_input):
prediction = prediction_input.copy()
np.random.seed(seed)
 best_score = cost_function(prediction)
 original_score = best_score
best_pred = prediction.copy()
print("SEED: {} ORIGINAL SCORE: {}".format(seed, original_score))
for t in range(100):
for i in range(5000):
for j in range(10):
di = prediction[i]
prediction[i] = DESIRED[i, j]
cur_score = cost_function(prediction)
 KT = 1
if t < 5:
KT = 1.5
elif t < 10:
KT = 4.5
else:
if cur_score > best_score + 100:
KT = 3
elif cur_score > best_score + 50 :
KT = 2.75
elif cur_score > best_score + 20:
KT = 2.5
elif cur_score > best_score + 10:
KT = 2
elif cur_score > best_score:
KT = 1.5
else:
KT = 1
prob = np.exp(-(cur_score - best_score)/ KT)
 if np.random.rand() < prob:
best_score = cur_score
else:
prediction[i] = di
if best_score < original_score:
print("NEW BEST SCORE on seed {}: {}".format(seed, best_score))
original_score = best_score
best_pred = prediction.copy()
return prediction<feature_engineering>
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )
def solveSantaLP() :
S = pywraplp.Solver('SolveAssignmentProblem', pywraplp.Solver.GLOP_LINEAR_PROGRAMMING)
 x = {}
candidates = [[] for _ in range(N_DAYS)]
for i in range(N_FAMILIES):
for j in DESIRED[i, :]:
candidates[j].append(i)
 x[i, j] = S.BoolVar('x[%i,%i]' %(i, j))
daily_occupancy = [S.Sum([x[i, j] * FAMILY_SIZE[i] for i in candidates[j]])
 for j in range(N_DAYS)]
family_presence = [S.Sum([x[i, j] for j in DESIRED[i, :]])
 for i in range(N_FAMILIES)]
preference_cost = S.Sum([PCOSTM[i, j] * x[i,j] for i in range(N_FAMILIES)
 for j in DESIRED[i, :] ])
 S.Minimize(preference_cost)
 for j in range(N_DAYS-1):
S.Add(daily_occupancy[j] - daily_occupancy[j+1] <= 23)
 S.Add(daily_occupancy[j+1] - daily_occupancy[j] <= 23)
 for i in range(N_FAMILIES):
S.Add(family_presence[i] == 1)
 for j in range(N_DAYS):
S.Add(daily_occupancy[j] >= MIN_OCCUPANCY)
 S.Add(daily_occupancy[j] <= MAX_OCCUPANCY)
 res = S.Solve()
resdict = {0:'OPTIMAL', 1:'FEASIBLE', 2:'INFEASIBLE', 3:'UNBOUNDED',
4:'ABNORMAL', 5:'MODEL_INVALID', 6:'NOT_SOLVED'}
print('LP solver result:', resdict[res])
 l = [(i, j, x[i, j].solution_value())for i in range(N_FAMILIES)
 for j in DESIRED[i, :]
if x[i, j].solution_value() >0]
df = pd.DataFrame(l, columns=['family_id', 'day', 'n'])
 return df<feature_engineering>
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))
def solveSantaIP(families, min_occupancy, max_occupancy):
S = pywraplp.Solver('SolveAssignmentProblem', pywraplp.Solver.CBC_MIXED_INTEGER_PROGRAMMING)
 n_families = len(families)
 x = {}
candidates = [[] for _ in range(N_DAYS)]
for i in families:
for j in DESIRED[i, :]:
candidates[j].append(i)
 x[i, j] = S.BoolVar('x[%i,%i]' %(i, j))
daily_occupancy = [S.Sum([x[i, j] * FAMILY_SIZE[i] for i in candidates[j]])
 for j in range(N_DAYS)]
family_presence = [S.Sum([x[i, j] for j in DESIRED[i, :]])
 for i in families]
preference_cost = S.Sum([PCOSTM[i, j] * x[i,j] for i in families
for j in DESIRED[i, :] ])
 S.Minimize(preference_cost)
 for i in range(n_families):
S.Add(family_presence[i] == 1)
 for j in range(N_DAYS):
S.Add(daily_occupancy[j] >= min_occupancy[j])
 S.Add(daily_occupancy[j] <= max_occupancy[j])
 res = S.Solve()
resdict = {0:'OPTIMAL', 1:'FEASIBLE', 2:'INFEASIBLE', 3:'UNBOUNDED',
4:'ABNORMAL', 5:'MODEL_INVALID', 6:'NOT_SOLVED'}
print('MIP solver result:', resdict[res])
 l = [(i, j)for i in families
for j in DESIRED[i, :]
if x[i, j].solution_value() >0]
df = pd.DataFrame(l, columns=['family_id', 'day'])
 return df<feature_engineering>
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
def solveSanta() :
df = solveSantaLP()
THRS = 0.999
assigned_df = df[df.n>THRS].copy()
unassigned_df = df[(df.n<=THRS)&(df.n>1-THRS)]
unassigned = unassigned_df.family_id.unique()
print('{} unassigned families'.format(len(unassigned)))
 assigned_df['family_size'] = FAMILY_SIZE[assigned_df.family_id]
occupancy = assigned_df.groupby('day' ).family_size.sum().values
min_occupancy = np.array([max(0, MIN_OCCUPANCY-o)for o in occupancy])
 max_occupancy = np.array([MAX_OCCUPANCY - o for o in occupancy])
 rdf = solveSantaIP(unassigned, min_occupancy, max_occupancy)
 df = pd.concat(( assigned_df[['family_id', 'day']], rdf)).sort_values('family_id')
 return df.day.values<load_from_csv>
skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam)
N_DAYS = 100
N_FAMILIES = 5000
MAX_OCCUPANCY = 300
MIN_OCCUPANCY = 125
data = pd.read_csv('/kaggle/input/santa-workshop-tour-2019/family_data.csv', index_col='family_id')
 FAMILY_SIZE = data.n_people.values
DESIRED = data.values[:, :-1] - 1
PCOSTM = GetPreferenceCostMatrix(data)
 ACOSTM = GetAccountingCostMatrix()<find_best_params>
all_data = pd.get_dummies(all_data)
 print(all_data.shape )
%%time
prediction = solveSanta()
pc, occ = pcost(prediction)
 ac, _ = acost(occ)
 print('{}, {:.2f},({}, {})'.format(pc, ac, occ.min() , occ.max()))<prepare_output>
train = all_data[:ntrain]
test = all_data[ntrain:]
new = prediction.copy()
findBetterDay4Family(new )<choose_model_class>
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb
final = stochastic_product_search(
 top_k=2,
fam_size=8,
original=new,
n_iter=500000,
verbose=1000,
verbose2=50000,
random_state=2019
)<define_variables>
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )
final = seed_finding(2019, final )<save_to_csv>
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
sub = pd.DataFrame(range(N_FAMILIES), columns=['family_id'])
 sub['assigned_day'] = final+1
sub.to_csv('submission.csv', index=False )<import_modules>
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
import time
import pandas as pd
import torch<set_options>
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5 )
torch.backends.cudnn.benchmark = True
torch.backends.cudnn.deterministic = False<define_variables>
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5 )
N = 25
device = 'cuda'
TEST_CSV = '.. /input/conways-reverse-game-of-life-2020/test.csv'
OUTPUT_CSV = 'submission.csv'<define_search_space>
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1 )
cv = torch.nn.Conv2d(1, 1, kernel_size=3, padding=1, padding_mode='circular', bias=False)
 cv.requires_grad=False
cv.weight = torch.nn.Parameter(
 torch.tensor(
 [[[[ 1., 1., 1.],
[ 1., 0., 1.],
[ 1., 1., 1.]]]],
device=device,
dtype=torch.float16 
),
requires_grad=False, 
)
 @torch.jit.script
def forward(grid, delta: int):
N=25
g = grid.reshape(-1, 1, N, N)
 for _ in torch.arange(delta):
g = g.to(torch.float16)
 neighbor_sum = cv(g)
 g =(( neighbor_sum == 3)|(( g == 1)&(neighbor_sum == 2)))
 return g.reshape(-1, N, N )<randomize_order>
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11 )
@torch.jit.script
def random_parents(n_parents: int, device: str):
N = 25
RANDOM_ALIVE =.2
return torch.rand(( n_parents, N, N), device=device)>(1-RANDOM_ALIVE )<compute_test_metric>
score = rmsle_cv(lasso)
 print("Lasso score: {:.4f}({:.4f})".format(score.mean() , score.std()))
@torch.jit.script
def loss(input, target):
return torch.sum(input ^ target, dim=(-1,-2))<find_best_params>
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})".format(score.mean() , score.std()))
@torch.jit.script
def select_best(parents, delta: int, target, n_best: int):
scores = loss(forward(parents, delta), target)
 best_values, best_indices = torch.topk(scores, n_best, dim=0, largest=False, sorted=True)
 new_parents = parents[best_indices,...]
return new_parents, best_values[0], new_parents[0,...]<statistical_test>
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})".format(score.mean() , score.std()))
@torch.jit.script
def random_combine(parents, n_offsprings: int, device: str, pre_masks):
N = 25
dads = torch.randint(low=0, high=parents.shape[0], size=(n_offsprings,),
device=device, dtype=torch.long)
 dads = parents[dads,...]
moms = torch.randint(low=0, high=parents.shape[0], size=(n_offsprings,),
device=device, dtype=torch.long)
 moms = parents[moms,...]
masks = pre_masks[torch.randint(low=0, high=pre_masks.shape[0], size=(n_offsprings,),
device=device, dtype=torch.long)]
return torch.where(masks, dads, moms )<define_search_space>
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})".format(score.mean() , score.std()))
def precomputes_masks() :
N = 25
BLOCK_SIZE = 17
block = torch.nn.Conv2d(1, 1, kernel_size=BLOCK_SIZE, padding=BLOCK_SIZE//2,
padding_mode='circular', bias=False)
 block.requires_grad=False
block.weight = torch.nn.Parameter(
 torch.ones(( 1, 1, BLOCK_SIZE, BLOCK_SIZE),
device=device,
dtype=torch.float16 
),
requires_grad=False, 
)
 masks = torch.zeros(( N * N, 1, N, N), device=device, dtype=torch.float16)
 for x in range(N):
for y in range(N):
masks[x * N + y, 0, x, y] = 1.
 masks = block(masks)
 return masks[:, 0,...] >.5<normalization>
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})".format(score.mean() , score.std()))
@torch.jit.script
def mutate(parents, device: str):
MUTATION =.0016
mutations = torch.rand(parents.shape, device=device)< MUTATION
return parents ^ mutations<define_search_space>
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})".format(score.mean() , score.std()))
@torch.jit.script
def optimize_one_puzzle(delta: int, data, device: str, pre_masks):
N = 25
N_GENERATION = 30
P = 4_500
N_BEST = P // 30
N_ELITES = 8
best_score = torch.tensor([N*N], device=device)
 best = torch.zeros(( N,N), device=device ).to(torch.bool)
 parents = random_parents(P, device)
 elites = torch.empty(( 1, N, N), dtype=torch.bool, device=device)
 elites[0,...] = data
for i in range(N_GENERATION):
parents = random_combine(parents, P, device, pre_masks)
 parents = mutate(parents, device)
 parents[:N_ELITES,...] = elites
parents, best_score, best = select_best(parents, delta, data, N_BEST)
 elites = parents[:N_ELITES,...]
if best_score == 0:
break
return best_score, best<find_best_params>
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )
@torch.jit.script
def optimize_all_puzzles(deltas, df, device: str, pre_masks):
sub = df.clone()
for n in torch.arange(df.shape[0]):
delta = deltas[n]
data = df[n,...]
_, sub[n,...] = optimize_one_puzzle(delta, data, device, pre_masks)
 return sub<load_from_csv>
averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})".format(score.mean() , score.std()))
df = pd.read_csv(TEST_CSV, index_col='id' )<drop_column>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )
submission = df.copy()
submission.drop(['delta'], inplace=True, axis=1 )<data_type_conversions>
stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))
indexes = df.index
deltas = torch.from_numpy(df.delta.values ).to(device)
 df = torch.BoolTensor(df.values[:, 1:].reshape(( -1, N, N)) ).to(device )<find_best_params>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
start_time = time.time()
pre_masks = precomputes_masks()
sub = optimize_all_puzzles(deltas, df, device, pre_masks)
 print(f'Processed {sub.shape[0]:,} puzzles in {time.time() - start_time:.2f} seconds 🔥🔥🔥' )<save_to_csv>
stacked_averaged_models.fit(train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))
submission.rename(columns={f'stop_{x}': f'start_{x}' for x in range(N*N)}, inplace=True)
 submission.iloc[:sub.shape[0], :] = sub.reshape(( -1, N*N)).cpu().numpy().astype(int)
 submission.to_csv(OUTPUT_CSV )<compute_test_metric>
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))
def leaderboard_score(deltas, df, sub, device: str):
result = torch.empty(sub.shape[0], device=device, dtype=torch.long)
 for delta in range(1, 6):
start = sub[deltas == delta]
end = df[deltas == delta]
result[deltas == delta] = loss(forward(start, delta), end)
 print('Leaderboard score:', torch.sum(result ).item() /(result.shape[0]*N*N))<compute_test_metric>
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))
leaderboard_score(deltas, df, sub, device )<set_options>

 print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.70 +
xgb_train_pred*0.15 + lgb_train_pred*0.15))
init_notebook_mode(connected=True)
 layout=go.Layout(paper_bgcolor='rgba(255,255,2555,0.5)', plot_bgcolor='rgba(250,250,250,0.8)', autosize=True)
 fig = go.Figure(layout=layout)
 templated_fig = pio.to_templated(fig)
 pio.templates['my_template'] = templated_fig.layout.template
pio.templates.default = 'my_template' 
<compute_test_metric>
ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15
<string_transform><EOS> 
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('submission.csv',index=False )