Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<import_modules>
print("scikit-learn version: {}".format(sklearn.__version__))
print("StackingRegressor imported" )
warnings.filterwarnings("ignore")
 pd.set_option('max_columns', None )<load_from_csv>
train_data=pd.read_csv(".. /input/train.csv")
 test_data=pd.read_csv(".. /input/test.csv")
 print("training_data(shape): {}".format(train_data.shape))
print("testing_data(shape): {}".format(test_data.shape))
path1 = '/kaggle/input/nfl-big-data-bowl-2020/train.csv'
path2 = '.. /input/nfl-big-data-bowl-2020/train.csv'
if os.path.exists(path1):
df = pd.read_csv(path1)
 TRAIN_OFFLINE = False
else:
df = pd.read_csv(path2)
 TRAIN_OFFLINE = True
df.info()<feature_engineering>
all_data=pd.concat([train_data,test_data], ignore_index="true", axis=0)
 all_data.reset_index()
print("Shape of all data: {}".format(all_data.shape))
def process_unique_features(df, fillna=-999):
def clean_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
def orientation_to_cat(x):
x = np.clip(x, 0, 360 - 1)
 try:
return str(int(x/15))
except:
return "nan"
def strtofloat(x):
try:
return float(x)
 except:
return fillna
def get_num_offense(row):
splits = row.split(', ')
 cnt = 0
for s in splits:
cnt += int(s[0])
 return cnt
add_new_feas = []
df = df.copy()
df['Temperature'] = df['Temperature'].fillna(60 ).astype(np.float)
 df['GameWeather'] = df['GameWeather'].apply(clean_weather ).astype(np.float)
 df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df["Orientation_ob"] = df["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Dir_ob"] = df["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Orientation_sin"] = df["Orientation"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Orientation_cos"] = df["Orientation"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
df["Dir_sin"] = df["Dir"].apply(lambda x : np.sin(x/360 * 2 * np.pi))
df["Dir_cos"] = df["Dir"].apply(lambda x : np.cos(x/360 * 2 * np.pi))
df['Position'] = df['Position'].apply(lambda x: 1 if x in ['RB', 'TE', 'GB', 'WR'] else 0)
 df["diffScoreBeforePlay"] = df["HomeScoreBeforePlay"] - df["VisitorScoreBeforePlay"]
df['PlayerHeight_dense'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
df['PlayerBMI'] = 703*(df['PlayerWeight']/(df['PlayerHeight_dense'])**2)
 df['PlayerBirthDate'] = df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 formation_dict = {'ACE': 3, 'EMPTY': 9, 'I_FORM': 4, 'JUMBO': 1, 'PISTOL': 7, 'SHOTGUN': 8, 'SINGLEBACK': 5, 'WILDCAT': 6, 'unknown': 2}
df['OffenseFormation'] = df['OffenseFormation'].fillna('unknown' ).map(formation_dict)
 df['kg']=df["PlayerWeight"] * 0.45359237 / 9.8
df['True_Momentum']=df['kg'] * df['S'] * 0.9144
df['Force_Newtons']=df['kg'] * df['A'] * 0.9144
df['JerseyNumber'] = df['JerseyNumber'].fillna(fillna)
 df['DefendersInTheBox'] = df['DefendersInTheBox'].apply(lambda x: 2.0 if x == 1.0 else x)
 df['DL_LB'] = df['DefensePersonnel'] \ 
.str[:10] \ 
.str.replace(' DL, ','-')\ 
.str.replace(' LB','')
 df['SUB_DL_BL'] = df['DL_LB'].apply(lambda x: int(x.split('-')[0])- int(x.split('-')[1]))
df['num_OffensePersonnel'] = df['OffensePersonnel'].apply(get_num_offense)

add_new_feas.append("Dir_sin")
 add_new_feas.append("Dir_cos")
 add_new_feas.append("diffScoreBeforePlay")
 add_new_feas.append('PlayerHeight_dense')
 add_new_feas.append('SUB_DL_BL')
 add_new_feas.append('True_Momentum')
 basic_feas =['GameId','PlayId',
'X','Y',
'S','A',
'Dis',
'Orientation',
'PlayerWeight',
'PlayerAge',
'Dir',
'YardLine',
'OffenseFormation',
'Down',
'Distance',
'DefendersInTheBox',
'JerseyNumber',
]
static_features = df[df['NflId'] == df['NflIdRusher']]\
[add_new_feas+basic_feas].drop_duplicates()
static_features.fillna(fillna,inplace=True)
 return static_features<merge>
print("kurtosis: %f" % SalePrice.kurtosis())
 print("Skewness: %f" % SalePrice.skew())
(mu, sigma)= norm.fit(SalePrice)
 print('Normal dist.( mu = {:.2f} and sigma = {:.2f})'.format(mu, sigma))
def defense_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team','X','Y']]
rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']
defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['GameId','PlayId','X','Y','RusherX','RusherY']]
defense['def_dist_to_back'] = defense[['X','Y','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 defense = defense.groupby(['GameId','PlayId'])\ 
.agg({'def_dist_to_back':['min','max','mean','std']})\ 
.reset_index()
defense.columns = ['GameId','PlayId',
'def_min_dist','def_max_dist',
'def_mean_dist','def_std_dist']
return defense
def features_relative_to_back(df, carriers):
player_distance = df[['GameId','PlayId','NflId','X','Y']]
player_distance = pd.merge(player_distance, carriers, on=['GameId','PlayId'], how='inner')
 player_distance = player_distance[player_distance['NflId'] != player_distance['NflIdRusher']]
player_distance['dist_to_back'] = player_distance[['X','Y','back_X','back_Y']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 player_distance = player_distance.groupby(['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field'])\ 
.agg({'dist_to_back':['min','max','mean','std']})\ 
.reset_index()
player_distance.columns = ['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field',
'min_dist',
'max_dist',
'mean_dist','std_dist']
return player_distance
def update_orientation(df, fix_ori=False, ori_dir_sub=False):
df['X'] = df[['X','PlayDirection']].apply(lambda x: new_X(x[0],x[1]), axis=1)
 df['Orientation'] = df[['Orientation','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df['Dir'] = df[['Dir','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 if fix_ori:
df.loc[df['Season'] == 2017, 'Orientation'] = np.mod(90 + df.loc[df['Season'] ==2017, 'Orientation'], 360)
 df.drop(['Season'], axis=1, inplace=True)
 if ori_dir_sub:
df['Ori_Dir_Sub'] = abs(df['Orientation'] - df['Dir'])
 return df
def back_features(df):
carriers = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','NflIdRusher','X','Y','Orientation','Dir','YardLine']]
carriers['back_from_scrimmage'] = carriers['YardLine'] - carriers['X']
carriers['back_oriented_down_field'] = carriers['Orientation'].apply(lambda x: back_direction(x))
carriers['back_moving_down_field'] = carriers['Dir'].apply(lambda x: back_direction(x))
carriers = carriers.rename(columns={'X':'back_X',
'Y':'back_Y'})
 carriers = carriers[['GameId','PlayId','NflIdRusher','back_X','back_Y','back_from_scrimmage','back_oriented_down_field','back_moving_down_field']]
return carriers
def new_line(rush_team, field_position, yardline):
if rush_team == field_position:
return 10.0 + yardline
else:
return 60.0 +(50 - yardline)
 def new_X(x_coordinate, play_direction):
if play_direction == 'left':
return 120.0 - x_coordinate
else:
return x_coordinate
def combine_features(relative_to_back, defense, personal, dist=True):
if dist:
df = relative_to_back
else:
df = pd.merge(relative_to_back,defense,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,personal,on=['GameId','PlayId'],how='inner')
 new_columns = list(df.columns)
 new_columns.remove('GameId')
 new_columns.remove('PlayId')
 return df, new_columns
def new_orientation(angle, play_direction):
if play_direction == 'left':
new_angle = 360.0 - angle
if new_angle == 360.0:
new_angle = 0.0
return new_angle
else:
return angle
def euclidean_distance(x1,y1,x2,y2):
x_diff =(x1-x2)**2
y_diff =(y1-y2)**2
return np.sqrt(x_diff + y_diff)
 def back_direction(orientation):
if orientation > 180.0:
return 1
else:
return 0 
<merge>
count_outliers(SalePrice,"before" )
def process_player_acc(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team','A']]
rusher.columns = ['GameId','PlayId','RusherTeam','RusherA']
acc = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 acc = acc[['GameId','PlayId','A','RusherA']]
acc['Sub_A'] = acc[['A','RusherA']].apply(lambda x: abs(x[0]-x[1]), axis=1)
 acc = acc.groupby(['GameId','PlayId'])\ 
.agg({'Sub_A':['mean']})\ 
.reset_index()
acc.columns = ['GameId','PlayId','sub_mean_A']
return acc
def process_yardline(df):
df = df.copy()
new_yardline = df[df['NflId'] == df['NflIdRusher']]
new_yardline['YardLine'] = new_yardline[['PossessionTeam','FieldPosition', 'YardLine']]\ 
.apply(lambda x: new_line(x[0],x[1],x[2]), axis=1)
 new_yardline = new_yardline[['GameId','PlayId','YardLine']]
df = df.drop('YardLine', axis=1)
 df = pd.merge(df, new_yardline, on=['GameId','PlayId'], how='inner')
 return df
def rusher_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Dir', 'S', 'A', 'X', 'Y']]
rusher.columns = ['GameId','PlayId', 'RusherDir', 'RusherS', 'RusherA', 'RusherX', 'RusherY']
radian_angle =(90 - rusher['RusherDir'])* np.pi / 180.0
v_horizontal = np.abs(rusher['RusherS'] * np.cos(radian_angle))
v_vertical = np.abs(rusher['RusherS'] * np.sin(radian_angle))
rusher['v_horizontal'] = v_horizontal
rusher['v_vertical'] = v_vertical
rusher.columns = ['GameId','PlayId', 'RusherDir', 'RusherS','RusherA','RusherX', 'RusherY','v_horizontal', 'v_vertical']
return rusher
def ori_dir_sub_features(df):
ori_rusher = df[df['NflId'] == df['NflIdRusher']][['GameId', 'PlayId', 'Ori_Dir_Sub', 'Team']]
ori_rusher.columns = ['GameId', 'PlayId', 'Rusher_Ori_Dir_Sub', 'RusherTeam']
ori_df = pd.merge(df, ori_rusher, on=['GameId', 'PlayId'], how='inner')
 ori_away = ori_df[ori_df['RusherTeam'] != ori_df['Team']][['GameId', 'PlayId', 'Ori_Dir_Sub', 'Rusher_Ori_Dir_Sub']]
ori_away['Ori_Dir_Sub_rusher'] = abs(ori_away['Rusher_Ori_Dir_Sub'] - ori_away['Ori_Dir_Sub'])
 ori_away['Ori_Dir_Sub_rusher'] = ori_away['Ori_Dir_Sub_rusher'].fillna(0 ).astype(np.int16)
 ori_away = ori_away.groupby(['GameId', 'PlayId'])['Ori_Dir_Sub_rusher'].mean().reset_index()
ori_away.columns = ['GameId', 'PlayId', 'Ori_Dir_Sub_rusher']
ori_rusher = pd.merge(ori_rusher, ori_away,
on=['GameId', 'PlayId'], how='inner')
 ori_rusher = ori_rusher[['GameId', 'PlayId', 'Ori_Dir_Sub_rusher', 'Rusher_Ori_Dir_Sub']]
return ori_rusher
def radian_features(df):
rusher = df[df['NflId']==df['NflIdRusher']][['GameId', 'PlayId', 'X', 'Y', 'Team']]
rusher.columns = ['GameId', 'PlayId', 'rusherX', 'rusherY', 'RusherTeam']
tmp_df = pd.merge(df, rusher, on=['GameId', 'PlayId'], how='left')
 tmp_df = tmp_df[tmp_df['RusherTeam'] != tmp_df['Team']][['GameId', 'PlayId', 'X', 'Y', 'rusherX', 'rusherY']]
tmp_df['radian'] = abs(tmp_df['Y'] - tmp_df['rusherY'])/ abs(tmp_df['X'] - tmp_df['rusherX'])
 radian_df = tmp_df.groupby(['GameId', 'PlayId'] ).aggregate({'radian': ['std', 'mean']})
 radian_df.columns = ['radian_std', 'radian_mean']
radian_df = radian_df.replace([np.inf, -np.inf], np.nan ).fillna(-999)
 return radian_df
def std_features(df):
tmp = df[df['NflId'] == df['NflIdRusher']][['GameId', 'PlayId', 'X', 'Team']]
tmp.columns = ['GameId', 'PlayId', 't_rusher_x', 'RusherTeam']
tmp_merge = pd.merge(df[['GameId', 'PlayId', 'X', 'Team']], tmp, on=['GameId', 'PlayId'], how='inner')
 tmp_merge = tmp_merge[tmp_merge['Team'] != tmp_merge['RusherTeam']]
tmp_merge['dist_x'] = abs(tmp_merge['X'] - tmp_merge['t_rusher_x'])
 x_tmp = tmp_merge.groupby(['GameId', 'PlayId'] ).aggregate({'dist_x': ['std']})
 x_tmp.columns = ['dist_x_std']
tmp['dist_x_std'] = x_tmp['dist_x_std'].values
return tmp[['GameId', 'PlayId', 'dist_x_std']]<merge>
def target_transform(target,transformation=None):
if(transformation is None):
print("{} transformation ".format("Log"))
count_zero=sum(val == 0 for val in target)
 if(count_zero > 0):
print("Transformation not applicable for zeroes values")
 else:
SalePrice=np.log1p(target)
 return SalePrice
if(transformation=="sqrt"):
print("{} transformation ".format(transformation))
SalePrice=np.sqrt(target)
 return SalePrice
if(transformation=="boxcox1p"):
print("{} transformation ".format(transformation))
SalePrice=boxcox1p(target , 0.2)
 return SalePrice
def create_feature(df, fix_ori,
ori_dir_sub,
process_acc,
process_ori_sub,
process_radian,
process_xy_std,
fillna=-999):
def combine_df(df1, df2):
return pd.merge(df1, df2, on=['GameId', 'PlayId'], how='inner')
 df = df.copy()
df = process_yardline(df)
 df = update_orientation(df, fix_ori, ori_dir_sub)
 static_feats = process_unique_features(df)
 back_feats = back_features(df)
 rel_back = features_relative_to_back(df, back_feats)
 def_feats = defense_features(df)
 rush_feats = rusher_features(df)
 table = df[df['NflId'] == df['NflIdRusher']][['GameId', 'PlayId']]
table = combine_df(table, rel_back)
 table = combine_df(table, static_feats)
 table = combine_df(table, def_feats)
 table = combine_df(table, rush_feats)
 if process_acc:
acc_feats = process_player_acc(df)
 table = combine_df(table, acc_feats)
 if process_ori_sub:
ori_feats = ori_dir_sub_features(df)
 table = combine_df(table, ori_feats)
 if process_xy_std:
std_feats = std_features(df)
 table = combine_df(table, std_feats)
 if process_radian:
radian_df = radian_features(df)
 table = combine_df(table, radian_df)
 rusher = df[df['NflId'] == df['NflIdRusher']][['GameId', 'PlayId', 'Team']]
rusher.columns = ['GameId', 'PlayId', 'RusherTeam']
tmp = pd.merge(df[['GameId', 'PlayId', 'Team', 'X', 'YardLine']], rusher, on=['GameId', 'PlayId'], how='left')
 tmp['back_scrim'] = tmp['YardLine'] - tmp['X']
tmp_away = tmp[tmp['Team'] != tmp['RusherTeam']].groupby(['GameId', 'PlayId'])['back_scrim'].mean().reset_index()
tmp_away.columns = ['GameId', 'PlayId', 'back_scrim_away']
table = pd.merge(table, tmp_away, on=['GameId', 'PlayId'], how='left')
 tmp_home = tmp[tmp['Team'] == tmp['RusherTeam']].groupby(['GameId', 'PlayId'])['back_scrim'].mean().reset_index()
tmp_home.columns = ['GameId', 'PlayId', 'back_scrim_home']
table = pd.merge(table, tmp_home, on=['GameId', 'PlayId'], how='left')
 table.drop(['GameId','PlayId', 'X', 'Y','A', 'S'], axis=1, inplace=True)
 return table<compute_test_metric>
SalePrice=target_transform(SalePrice )
def get_crps(y_pred, y_valid):
y_valid = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_valid - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * y_valid.shape[0])
 crps = np.round(val_s, 6)
 return crps<train_model>
print("kurtosis: %f" % SalePrice.kurtosis())
 print("Skewness: %f" % SalePrice.skew())
(mu, sigma)= norm.fit(SalePrice)
 print('Normal dist.( mu = {:.2f} and sigma = {:.2f})'.format(mu, sigma))
class CRPSCallback(Callback):
def __init__(self,validation, predict_batch_size=20, include_on_batch=False):
super(CRPSCallback, self ).__init__()
self.validation = validation
self.predict_batch_size = predict_batch_size
self.include_on_batch = include_on_batch
print('validation shape',len(self.validation))
def on_batch_begin(self, batch, logs={}):
pass
def on_train_begin(self, logs={}):
if not('CRPS_score_val' in self.params['metrics']):
self.params['metrics'].append('CRPS_score_val')
 def on_batch_end(self, batch, logs={}):
if(self.include_on_batch):
logs['CRPS_score_val'] = float('-inf')
 def on_epoch_end(self, epoch, logs={}):
logs['CRPS_score_val'] = float('-inf')
 if(self.validation):
X_valid, y_valid = self.validation[0], self.validation[1]
y_pred = self.model.predict(X_valid)
 y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * X_valid.shape[0])
 val_s = np.round(val_s, 6)
 logs['CRPS_score_val'] = val_s 
<compute_train_metric>
categorical_cols=[col for col in all_data.columns if all_data[col].dtype =='O']
numerical_cols=[col for col in all_data.columns if all_data[col].dtype !='O']
print("categorical_columns :
")
 print(categorical_cols)
 print("
")
 print("numerical_columns :
")
 print(numerical_cols )
def get_nn_model(x_tr,y_tr,x_val,y_val,step):
inp = Input(shape =(x_tr.shape[1],))
x = Dense(512, input_dim=x_tr.shape[1], activation='relu' )(inp)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(512, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 x = Dense(256, activation='relu' )(x)
 x = Dropout(0.5 )(x)
 x = BatchNormalization()(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inp,out)
 model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[])
 es = EarlyStopping(monitor='val_loss',
mode='min',
restore_best_weights=True,
verbose=1,
patience=30)
 mc = ModelCheckpoint('best_model_{}.h5'.format(step),monitor='CRPS_score_val',mode='min',
save_best_only=True, verbose=1, save_weights_only=True)
 bsz = 1024
steps = x_tr.shape[0]/bsz
hist = model.fit(x_tr, y_tr,callbacks=[CRPSCallback(validation =(x_val,y_val)) ,es,mc], epochs=250, batch_size=bsz,verbose=1)
 model.load_weights('best_model_{}.h5'.format(step))
y_pred = model.predict(x_val)
 crps = get_crps(y_pred, y_val)
 return model, crps, hist 
<split>
null_percentage =(all_data.isnull().sum() /len(all_data)) *100
null_ratio = null_percentage.sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Values Ratio' :null_ratio})
 print(missing_data.shape)
 missing_data.head(10 )
def train(X, yards, yards_label, step=2, fold=5):
crpses = []
models = []
s_time = time.time()
hists = []
fold = fold
y = get_y(yards)
 for i in range(step):
kfold = StratifiedKFold(n_splits=fold, random_state = 42+i, shuffle = True)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(yards_label,yards_label)) :
tr_x, tr_y = X[tr_inds],y[tr_inds]
val_x, val_y = X[val_inds],y[val_inds]
model,crps, hist = get_nn_model(tr_x,tr_y,val_x,val_y,i*fold+(k_fold+1))
models.append(model)
 crpses.append(crps)
 hists.append(hist)
 print("the %d fold crps is %f"%(( i*(k_fold+1)) +(k_fold+1),crps))
print("mean crps is %f"%np.mean(crpses))
return models, crpses, hists<prepare_x_and_y>
print("Number of columns has a missing data")
 print(len(null_percentage.index)- len([x for x in null_percentage.index if null_percentage.loc[x]==0]))
def get_y(yards):
y = np.zeros(( yards.shape[0], 199))
for idx, target in enumerate(list(yards)) :
y[idx][99 + target] = 1
return y<predict_on_test>
all_data["PoolQC"] = all_data["PoolQC"].fillna("None")
 all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None")
 all_data["Alley"] = all_data["Alley"].fillna("None")
 all_data["Fence"] = all_data["Fence"].fillna("None")
 all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None")
 all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0)
 all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0])
 all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None")
 all_data["Functional"] = all_data["Functional"].fillna("Typ")
 all_data = all_data.drop(['Utilities'], axis=1)
 all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))
for col in ['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']:
all_data[col] = all_data[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None')
 for col in ['Electrical','KitchenQual', 'Exterior1st', 'Exterior2nd', 'SaleType']:
all_data[col] = all_data[col].fillna(all_data[col].mode() [0])
def predict(x_te, models):
model_num = len(models)
 for k,m in enumerate(models):
if k==0:
y_pred = m.predict(x_te,batch_size=1024)
 else:
y_pred+=m.predict(x_te,batch_size=1024)
 y_pred = y_pred / model_num
return y_pred<feature_engineering>
null_percentage =(all_data.isnull().sum() /len(all_data)) *100
null_ratio = null_percentage.sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Values Ratio' :null_ratio})
 print("Number of columns has a missing data")
 print(len(null_percentage.index)- len([x for x in null_percentage.index if null_percentage.loc[x]==0]))
fillna = -999
fix_ori = True
ori_dir_sub = True
process_ori_sub = True
process_acc = True
process_radian = False
process_xy_std = True
input_df = create_feature(df,
fix_ori=fix_ori,
ori_dir_sub=ori_dir_sub,
process_acc=process_acc,
process_ori_sub=process_ori_sub,
process_radian=process_radian,
process_xy_std=process_xy_std,
fillna=fillna)
 print('[*]input_df create')
 print(input_df.shape )<normalization>
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )
scaler = StandardScaler()
X = scaler.fit_transform(input_df)
 yards = np.array([df['Yards'][i] for i in range(0,df.shape[0],22)])
 df['Yards_Label'] = df['Yards'] // 10
yards_label = np.array([df['Yards_Label'][i] for i in range(0,df.shape[0],22)])
 models, crpses, hist = train(X, yards, yards_label=yards_label, step=2, fold=5 )<compute_test_metric>
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
print(X.shape)
 print('crps mean: ', np.mean(crpses))
loss_sum = 0
for i in range(10):
loss_sum += hist[i].history['loss'][hist[i].history['CRPS_score_val'].index(min(hist[i].history['CRPS_score_val'])) ]
print('loss mean: ', loss_sum/10 )<feature_engineering>
cols_label = ['FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold']
enc_label=LabelEncoder()
for columns in cols_label:
all_data[columns] = enc_label.fit_transform(all_data[columns] )
if TRAIN_OFFLINE==True:
basetable = create_feature(df.iloc[:22, :],
fix_ori=fix_ori,
ori_dir_sub=ori_dir_sub,
process_acc=process_acc,
process_ori_sub=process_ori_sub,
process_radian=process_radian,
process_xy_std=process_xy_std,
fillna=fillna)
 scaled_basetable = scaler.transform(basetable)
 y_pred = predict(scaled_basetable, models=models)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1 ).tolist() [0]
print(len(y_pred))
print(y_pred)
 elif TRAIN_OFFLINE==False:
env = nflrush.make_env()
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in tqdm.tqdm(iter_test):
basetable = create_feature(test_df,
fix_ori=fix_ori,
ori_dir_sub=ori_dir_sub,
process_acc=process_acc,
process_ori_sub=process_ori_sub,
process_radian=process_radian,
process_xy_std=process_xy_std,
fillna=fillna)
 scaled_basetable = scaler.transform(basetable)
 y_pred = predict(scaled_basetable, models=models)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1 ).tolist() [0]
preds_df = pd.DataFrame(data=[y_pred], columns=sample_prediction_df.columns)
 env.predict(preds_df)
 env.write_submission_file()
print('done' )<feature_engineering>
skew_feat=skewness.iloc[:5]
for columns in skew_feat.index:
all_data[columns] = boxcox1p(all_data[columns],0.2)
def error_correcting_codes(df):
df = df.replace('BLT', 'BAL')
 df = df.replace('HST', 'HOU')
 df = df.replace('ARZ', 'ARI')
 df = df.replace('CLV', 'CLE')
 return df
def organize_positions(df):
return(df.loc[(df['PossessionTeam']==df['HomeTeamAbbr'])&(df['Team']=='away')|(df['PossessionTeam']==df['VisitorTeamAbbr'])&(df['Team']=='home')].copy().reset_index() ,
df.loc[(( df['PossessionTeam']==df['HomeTeamAbbr'])&(df['Team']=='home')|(df['PossessionTeam']==df['VisitorTeamAbbr'])&(df['Team']=='away')) &(df['NflId']!=df['NflIdRusher'])].copy().reset_index() ,
df.loc[df['NflId']==df['NflIdRusher']].copy().reset_index())
 def doubledown(X, doublings=1):
np.random.seed(3)
 for w in range(doublings):
X_dupe2 = np.concatenate(( X.copy() , X.copy()), axis=0)
 for i in range(X.shape[0]):
X_dupe2[2*i, :] = X[i, :]
X_dupe2[2*i+1, :] = X[i, :]
X = X_dupe2
return X
def physics_init(df):
way = -2*(df['PlayDirection']=='left')+ 1
theta = way*df['Dir']*np.pi/180
df['X'] =(df['PlayDirection']=='right')*df['X'] +(df['PlayDirection']=='left')*(120 - df['X'])
 df['Sx'] = np.sin(theta)*df['S']
df['Sy'] = np.cos(theta)*df['S']
df['Ax'] = np.sin(theta)*df['A']
df['Ay'] = np.cos(theta)*df['A']
df['EquivYardLine'] =(df['PossessionTeam']==df['FieldPosition'])*(df['YardLine']+10)+(df['PossessionTeam']!=df['FieldPosition'])*(110-df['YardLine'])
 defn, off, RBs = organize_positions(df)
 defn['X'] -= RBs.loc[[i//11 for i in defn.index], 'X'].values
defn['Y'] -= RBs.loc[[i//11 for i in defn.index], 'Y'].values
defn = defn.loc[:,('PlayId', 'PlayerWeight', 'Sx', 'Sy', 'Ax', 'Ay', 'X', 'Y')]
defn.fillna(0, inplace=True)
 defn['Infl'] = defn['PlayerWeight']/(np.square(defn['X'])+ np.square(defn['Y'])) **0.5
defn['AngularMomentum'] = -defn['PlayerWeight']*(defn['X']*defn['Sx'] + defn['Y']*defn['Sy'])/(np.square(defn['X'])+ np.square(defn['Y']))
off['X'] -= RBs.loc[[i//10 for i in off.index], 'X'].values
off['Y'] -= RBs.loc[[i//10 for i in off.index], 'Y'].values
off = off.loc[:,('PlayId', 'PlayerWeight', 'Sx', 'Sy', 'Ax', 'Ay', 'X', 'Y')]
off.fillna(0, inplace=True)
 off['Infl'] = off['PlayerWeight']/(np.square(off['X'])+ np.square(off['Y'])) **0.5
off['AngularMomentum'] = -off['PlayerWeight']*(off['X']*off['Sx'] + off['Y']*off['Sy'])/(np.square(off['X'])+ np.square(off['Y']))
RBs['YardsBehindScrimmage'] = RBs['EquivYardLine'] - RBs['X']
RBs['X'] = 0
RBs['Y'] = 0
RBs = RBs.loc[:,('PlayId', 'YardsBehindScrimmage', 'PlayerWeight', 'Sx', 'Sy', 'Ax', 'Ay', 'X', 'Y')]
RBs.fillna(0, inplace=True)
 return defn, off, RBs
def action(defn, off, RBs, timestep=0.1):
t = 0.0
while t<timestep:
for X in(defn, off, RBs):
X['X'] += X['Sx']*0.01 +X['Ax']*0.01**2/2
X['Y'] += X['Sy']*0.01 +X['Ay']*0.01**2/2
X['Sx'] += X['Ax']*0.01
X['Sy'] += X['Ay']*0.01
X['Ax'] *= 0.99
X['Ay'] *= 0.99
t += 0.01
defn['X'] -= RBs.loc[[i//11 for i in defn.index], 'X'].values
defn['Y'] -= RBs.loc[[i//11 for i in defn.index], 'Y'].values
defn['Infl'] = defn['PlayerWeight']/(np.square(defn['X'])+ np.square(defn['Y'])) **0.5
defn['AngularMomentum'] = -defn['PlayerWeight']*(defn['X']*defn['Sx'] + defn['Y']*defn['Sy'])/(np.square(defn['X'])+ np.square(defn['Y']))
off['X'] -= RBs.loc[[i//10 for i in off.index], 'X'].values
off['Y'] -= RBs.loc[[i//10 for i in off.index], 'Y'].values
off['Infl'] = off['PlayerWeight']/(np.square(off['X'])+ np.square(off['Y'])) **0.5
off['AngularMomentum'] = -off['PlayerWeight']*(off['X']*off['Sx'] + off['Y']*off['Sy'])/(np.square(off['X'])+ np.square(off['Y']))
RBs['X'] = 0
RBs['Y'] = 0
return defn, off, RBs
def physics_doubledown(X, doublings, width):
np.random.seed(3)
 for w in range(doublings):
X_dupe = X.copy()
X_dupe2 = np.concatenate(( X.copy() , X.copy()), axis=0)
 numpy_sucks = np.arange(11)
 np.random.shuffle(numpy_sucks)
 for(i,j)in enumerate(numpy_sucks):
X_dupe[:, width*i:width*i+width] = X[:, width*j:width*j+width]
numpy_sucks = np.arange(10)
 np.random.shuffle(numpy_sucks)
 for(i,j)in enumerate(numpy_sucks):
X_dupe[:, width*(i+11):width*(i+12)] = X[:, width*(j+11):width*(j+12)]
for i in range(X.shape[0]):
X_dupe2[2*i, :] = X[i, :]
X_dupe2[2*i+1, :] = X_dupe[i, :]
X = X_dupe2
return X
def generate_physics(df, forward_action=0, timestep=0.1, doublings=0):
d, o, r = physics_init(df)
 df = None
defn = [d.copy() ]
off = [o.copy() ]
RBs = [r.copy() ]
for a in range(forward_action):
d, o, r = action(d, o, r, timestep)
 defn.append(d.copy())
 off.append(o.copy())
 RBs.append(r.copy())
 d, o, r = None, None, None
for X in(defn, off, RBs):
for i in range(len(defn)) :
X[i]['Px'] = X[i]['Sx']*X[i]['PlayerWeight']
X[i]['Py'] = X[i]['Sy']*X[i]['PlayerWeight']
X[i]['Fx'] = X[i]['Ax']*X[i]['PlayerWeight']
X[i]['Fy'] = X[i]['Ay']*X[i]['PlayerWeight']
for i in range(len(defn)) :
if i==0:
bigD = defn[i].loc[:,('Px', 'Py', 'X', 'Y', 'Infl', 'AngularMomentum')].astype(np.float32)
 bigO = off[i].loc[:,('Px', 'Py', 'X', 'Y', 'Infl', 'AngularMomentum')].astype(np.float32)
 backs = RBs[i].loc[:,('YardsBehindScrimmage', 'Px', 'Py', 'Fx', 'Fy')].astype(np.float32)
 if bigD.shape[0]==0 | bigO.shape[0]==0:
bigD = pd.DataFrame(data=np.random.randn(11,6), columns=('Px', 'Py', 'X', 'Y', 'Infl', 'AngularMomentum'))
bigO = pd.DataFrame(data=np.random.randn(10,6), columns=('Px', 'Py', 'X', 'Y', 'Infl', 'AngularMomentum'))
inst = np.concatenate(( np.reshape(bigD.copy().values,(bigD.shape[0]//11, 11*6)) ,
np.reshape(bigO.copy().values,(bigO.shape[0]//10, 10*6)) ,
backs.copy().values), axis=1)
 summary = physics_doubledown(inst, doublings, 6)
 else:
bigD = defn[i].loc[:,('Infl', 'AngularMomentum')].astype(np.float32)
 bigO = off[i].loc[:,('Infl', 'AngularMomentum')].astype(np.float32)
 if bigD.shape[0]==0 | bigO.shape[0]==0:
bigD = pd.DataFrame(data=np.random.randn(11,2), columns=('Infl', 'AngularMomentum'))
bigO = pd.DataFrame(data=np.random.randn(10,2), columns=('Infl', 'AngularMomentum'))
inst = np.concatenate(( np.reshape(bigD.copy().values,(bigD.shape[0]//11, 11*2)) ,
np.reshape(bigO.copy().values,(bigO.shape[0]//10, 10*2))), axis=1)
 summary = np.concatenate(( summary, physics_doubledown(inst, doublings, 2)) , axis=1)
 defn, off, RBs = None, None, None
return summary
def generate_physics_II(df):
defn, off, RBs = physics_init(df)
 bigD = defn[['PlayerWeight', 'X', 'Y', 'Sx', 'Sy', 'Ax', 'Ay']].astype(np.float32)
 bigO = off[['PlayerWeight', 'X', 'Y', 'Sx', 'Sy', 'Ax', 'Ay']].astype(np.float32)
 backs = RBs[['YardsBehindScrimmage', 'PlayerWeight', 'Sx', 'Sy', 'Ax', 'Ay']].astype(np.float32)
 if bigD.shape[0]==0 | bigO.shape[0]==0:
bigD = pd.DataFrame(data=np.random.randn(11,7), columns=('PlayerWeight', 'X', 'Y', 'Sx', 'Sy', 'Ax', 'Ay'))
bigO = pd.DataFrame(data=np.random.randn(10,7), columns=('PlayerWeight', 'X', 'Y', 'Sx', 'Sy', 'Ax', 'Ay'))
summary = np.concatenate(( backs.copy().values,
np.reshape(bigD.copy().values,(bigD.shape[0]//11, 11*7)) ,
np.reshape(bigO.copy().values,(bigO.shape[0]//10, 10*7))), axis=1)
 defn, off, RBs = None, None, None
bigD, bigO, backs = None, None, None
return summary
def down_situation(df):
X = df.loc[::22,('Down', 'Distance')].copy().astype(np.float32)
 X.fillna(-1, inplace=True)
 framer = pd.DataFrame(columns=(1.0, 2.0, 3.0, 4.0, 'Distance'), dtype=np.float32)
 concatenation = pd.concat(( pd.get_dummies(X['Down']), X['Distance']), axis=1, join='outer')
 concatenation = pd.concat(( framer, concatenation), axis=0, join='outer')
 concatenation.fillna(0, inplace=True)
 return concatenation.values
def RB_wins_it(df):
X = df.loc[::22,('NflIdRusher')]
framer = pd.DataFrame(columns=baxxx, dtype=np.float32)
 OHE = pd.get_dummies(X)
 concatenation = pd.concat(( framer, OHE.loc[:, :framer.shape[1]]), axis=0, join='outer')
 concatenation.fillna(0, inplace=True)
 return concatenation.values.astype(np.float32)
 def stats(array):
return array.mean(axis=0), array.std(axis=0)
 def stats_II(X):
means = X[:, :11].mean(axis=0)
 deviants = X[:, :11].std(axis=0)
 for start, step, amt in [(11,7,11),(88,7,10)]:
repmeans = []
repdevs = []
for i in range(step):
repmeans.append(X[:, start+i:start+step*amt:step].mean())
 repdevs.append(X[:, start+i:start+step*amt:step].std())
 for j in range(amt):
means = np.concatenate(( means, repmeans))
deviants = np.concatenate(( deviants, repdevs))
means = np.concatenate(( means, [0 for i in range(158, 529)]), axis=0)
 deviants = np.concatenate(( deviants, [1 for i in range(158, 529)]), axis=0)
 return means, deviants
def normalize(X, mn, stand):
return(X - mn)/ stand
def generate_yardudge(df):
Y = np.zeros(( df.shape[0]//22, 199))
for(i, yerds)in enumerate(df['Yards'][::22]):
Y[i, yerds+99] = 1
return Y.astype(np.float32 )<load_from_csv>
scale=StandardScaler()
numerical_features=all_data.dtypes[all_data.dtypes != "object"].index
all_data[numerical_features]=scale.fit_transform(all_data[numerical_features].values )
<prepare_x_and_y>
all_data=pd.get_dummies(all_data )
global meen_II, sigma,_II, baxxx
train_df = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', low_memory=False)
 baxxx = train_df['NflIdRusher'].unique()
Y = generate_yardudge(train_df)
 train_df = error_correcting_codes(train_df)
 X = np.concatenate(( down_situation(train_df),
generate_physics_II(train_df),
RB_wins_it(train_df)) , axis=1)
 train_df = None
meen_II, sigma_II = stats_II(X)
 X = normalize(X, meen_II, sigma_II)
 gc.collect()<init_hyperparams>
train_set=all_data[:len(train_ID)]
predict_set=all_data[len(test_ID)+1:]
stuff = [4.81725572e-02, 1.48089908e-05, 9.57455511e-01, 1.58078791e+00,
9.93707682e-02, 5.04421303e-01, 6.51849665e+01, 9.68255755e+02,
5.14218542e+02, 5.09462973e+02, 5.17812944e+02, 8.04146527e+02,
1.64480385e+02, 9.80945456e+02, 8.50852251e+01, 1.03668046e+03,
6.57080119e+01]
learning_rate = stuff[0]
lr_decay =stuff[1]
beta = stuff[2]
clipse = stuff[3]
drop_rate1 = stuff[4]
drop_rate2 = stuff[5]
epochs = int(round(stuff[6]))
batch_size = int(round(stuff[7]))
hidden_layer1_nodes = int(round(stuff[8]))
hidden_layer2_nodes = int(round(stuff[9]))
hidden_layer3_nodes = int(round(stuff[10]))
function_layer1_nodes = int(round(stuff[11]))
function_layer2_nodes = int(round(stuff[12]))
function_layer3_nodes = int(round(stuff[13]))
function_layer4_nodes = int(round(stuff[14]))
function_layer5_nodes = int(round(stuff[15]))
function_layer6_nodes = int(round(stuff[16]))
ipt = [Input(shape=(11,)) ] + [Input(shape=(7,)) for i in range(11)] + \
[Input(shape=(7,)) for i in range(10)] + [Input(shape=(baxxx.shape[0],)) ]
np.random.seed(314159)
 expanse = Dense(function_layer1_nodes, activation='relu')
 collapsor = Dense(function_layer2_nodes, activation='linear')
 expanse2 = Dense(function_layer3_nodes, activation='relu')
 collapsor2 = Dense(function_layer4_nodes, activation='linear')
 faux_embedding_expanse = Dense(function_layer5_nodes, activation='relu')
 faux_embedding_collapsor = Dense(function_layer6_nodes, activation='linear')
 D = Add()([collapsor(expanse(ipt[i])) for i in range(1,12)])
 O = Add()([collapsor(expanse(ipt[i])) for i in range(12,22)])
 R = faux_embedding_collapsor(faux_embedding_expanse(ipt[22]))
p = concatenate([ipt[0], D, O, R], axis=-1)
 x = Dropout(drop_rate1 )(p)
 x = Dense(hidden_layer1_nodes, activation='relu' )(x)
 x = Dropout(drop_rate2 )(x)
 x = Dense(hidden_layer2_nodes, activation='relu' )(x)
 x = Dropout(drop_rate2 )(x)
 x = Dense(hidden_layer3_nodes, activation='relu' )(x)
 x = Dropout(drop_rate2 )(x)
 output = Dense(199, activation='softmax' )(x)
 model_NN2 = Model(inputs=ipt, outputs=output)
 model_NN2.compile(optimizer=SGD(lr=learning_rate, decay=lr_decay, momentum=beta, nesterov=True, clipnorm=clipse), loss=categorical_crossentropy)
 model_NN2.fit([X[:,:11]]+[X[:,7*i+11:7*i+18] for i in range(0,11)]+
[X[:,7*i+11:7*i+18] for i in range(11,21)]+[X[:,158:]],
Y, epochs=epochs, batch_size=batch_size)
 gc.collect()<load_from_csv>
X_train, X_test, Y_train, Y_test = train_test_split(train_set, SalePrice, test_size=0.20, random_state=42 )
<train_model>
model_name=[]
r_sqaure_value=[]
mean_sqaure_value=[]
def calculate_metrics(model, predicted_value, test_value):
if model not in model_name:
model_name.append(model)
 r_sqaure_value.append(r2_score(predicted_value,test_value))
mean_sqaure_value.append(mean_squared_error(predicted_value,test_value))
Yxgb = np.argmax(Y, axis=1)- 99
model_xgb1 = XGBClassifier(objective='multi:softmax', seed=27182818, max_depth=1)
 model_xgb1.fit(X, Yxgb )<predict_on_test>
params={'alphas':[0.001, 0.01 , 0.1 ,1.0],
'cv': 5,
'fit_intercept': True,
'gcv_mode': None,
'normalize': False,
'scoring': None,
'store_cv_values': False}
ridge_reg=RidgeCV(**params)
 ridge_reg.fit(X_train,Y_train)
 ridge_predicted=ridge_reg.predict(X_test)
 calculate_metrics("LassoCv", ridge_predicted, Y_test )
<feature_engineering>
params={'alphas':[0.1,0.001,0.003,0.0001],
'copy_X': True,
'cv': 5,
'eps': 0.001,
'fit_intercept': True,
'max_iter': 2000,
'n_alphas': 100,
'n_jobs': None,
'normalize': False,
'positive': False,
'precompute': 'auto',
'random_state': None,
'selection': 'cyclic',
'tol': 0.0001,
'verbose': False}
lasso_reg = LassoCV(**params)
 lasso_reg.fit(X_train,Y_train)
 lasso_predicted=lasso_reg.predict(X_test)
 calculate_metrics("RidgeCV", lasso_predicted, Y_test )
<find_best_params>
params={'alphas': [0.1,0.001, 0.02,1.0],
'copy_X': True,
'cv': 5,
'eps': 0.001,
'fit_intercept': True,
'l1_ratio': [0.5,0.4],
'max_iter': 3000,
'n_alphas': 100,
'n_jobs': None,
'normalize': False,
'positive': False,
'precompute': 'auto',
'random_state': None,
'selection': 'cyclic',
'tol': 0.0001,
'verbose': 0}
Enet=ElasticNetCV(**params)
 Enet.fit(X_train,Y_train)
 Enet_predicted=Enet.predict(X_test)
 calculate_metrics("ElasticNetCV", Enet_predicted, Y_test )
env = nflrush.make_env()<compute_train_metric>
start = time.time()
print("Grid Search Started : {}".format(start))
grid_ser_svr = GridSearchCV(
 estimator=SVR() ,
param_grid={'C': [0.1, 1, 0.2],
'epsilon': [0.0001, 0.001, 0.005],
'gamma': [0.0001, 0.001, 0.1 ],
'kernel': ["linear", "poly", "rbf"]
},
cv=5, scoring='neg_mean_squared_error', verbose=0, n_jobs=-1)
 svr_grid= grid_ser_svr.fit(X_train,Y_train)
 end = time.time()
print("Grid Search ended: {}".format(end))
print("Elapsed time for Grid search: {}".format(end-start))
print("Best parameters for grid search
")
 print(svr_grid.best_params_)
 print("
")
 calculate_metrics("SVR", svr_grid.predict(X_test), Y_test )
def make_my_predictions(model1, model2, model3, test_df, sample_predictions):
test_df = error_correcting_codes(test_df)
 X = np.concatenate(( down_situation(test_df.copy()),
generate_physics_II(test_df.copy()),
RB_wins_it(test_df.copy())) , axis=1)
 X = normalize(X, meen_II, sigma_II)
 my_predictions_II = model2.predict([X[:,:11]]+[X[:,7*i+11:7*i+18] for i in range(0,11)]+
[X[:,7*i+11:7*i+18] for i in range(11,21)]+[X[:,158:]])
 offset = int(( model3.predict(X)- np.argmax(model3.predict_proba(X), axis=1)) [0])
 my_predictions_III = model3.predict_proba(X)
 my_predictions_III = np.concatenate(( np.zeros(( 1, 99+offset)) , my_predictions_III, np.ones(( 1, 100-my_predictions_III.shape[1]-offset))), axis=1)
 my_predictions = np.reshape(0.62*my_predictions_II + 0.38*my_predictions_III,(199,0))
cucumber = 0
for i in range(1, my_predictions.shape[0]-1):
cucumber += np.maximum(my_predictions[i], 0)
 sample_predictions.iloc[0, i] = np.minimum(cucumber, 1)
 return sample_predictions<predict_on_test>
params={'criterion': 'mse',
'max_depth': None,
'max_features': None,
'max_leaf_nodes': None,
'min_impurity_decrease': 0.0,
'min_impurity_split': None,
'min_samples_leaf': 1,
'min_samples_split': 2,
'min_weight_fraction_leaf': 0.0,
'presort': True,
'random_state': None,
'splitter': 'best'}
deci_tree_reg=DecisionTreeRegressor(**params)
 deci_tree_reg.fit(X_train,Y_train)
 decision_predicted=deci_tree_reg.predict(X_test)
 calculate_metrics("Decision Tree", decision_predicted, Y_test )
for(test_df, sample_prediction_df)in env.iter_test() :
predictions_df = make_my_predictions(model_xgb1, model_NN2, model_xgb1, test_df, sample_prediction_df)
 env.predict(predictions_df)
 env.write_submission_file()<import_modules>
start = time.time()
print("Grid Search Started : {}".format(start))
grid_ser_rand = GridSearchCV(
 estimator=RandomForestRegressor() ,
param_grid={'max_depth': [60, 50, 40, 45],
'n_estimators': [5,6,7, 8],
'max_leaf_nodes': [2, 3, 4,5,6]},
cv=5, scoring='neg_mean_squared_error', verbose=0, n_jobs=-1)
 random_grid= grid_ser_rand.fit(X_train,Y_train)
 end = time.time()
print("Grid Search ended: {}".format(end))
print("Time taken by Grid search: {}".format(end-start))
print("Best parameters for grid search :
")
 print(random_grid.best_params_)
 calculate_metrics("Random Forest", random_grid.predict(X_test), Y_test )
import datetime
import matplotlib.pyplot as plt
import re
from string import punctuation
import time
import tqdm<categorify>
start = time.time()
print("Grid Search Started : {}".format(start))
grid_ser_boost = GridSearchCV(
 estimator=GradientBoostingRegressor() ,
param_grid={'max_depth': [3, 4, 6, 8],
'n_estimators': [100,120,80,90]},
cv=5, scoring='neg_mean_squared_error', verbose=0, n_jobs=-1)
 gra_grid= grid_ser_boost.fit(X_train,Y_train)
 end = time.time()
print("Grid Search ended: {}".format(end))
print("Time taken by Grid search: {}".format(end-start))
print("Best parameters for grid search
")
 print(gra_grid.best_params_)
 calculate_metrics("GDBRT", gra_grid.predict(X_test), Y_test )
env = nflrush.make_env()<load_from_csv>
print("Predicting price of house")
 final_price=(np.expm1(ridge_reg.predict(predict_set))
+np.expm1(svr_grid.predict(predict_set))
+np.expm1(lasso_reg.predict(predict_set))
+np.expm1(Enet.predict(predict_set))
+np.expm1(gra_grid.predict(predict_set))
+np.expm1(deci_tree_reg.predict(predict_set)))/6
final_price
<feature_engineering><EOS> 
submit = pd.DataFrame()
submit['Id'] = test_ID
submit['SalePrice'] = final_price
submit.to_csv('submission.csv',index=False )