Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<train_on_grid>
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
%matplotlib inline
sns.set(style='darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
warnings.filterwarnings('ignore')
 print('All relevant packages imported successfully' )
def get_conv_net(num_classes_y):
inputdense_players = Input(shape=(11,10,10), name = "playersfeatures_input")
 X = Conv2D(128, kernel_size=(1,1), strides=(1,1), activation='relu' )(inputdense_players)
 X = Conv2D(160, kernel_size=(1,1), strides=(1,1), activation='relu' )(X)
 X = Conv2D(128, kernel_size=(1,1), strides=(1,1), activation='relu' )(X)
 Xmax = MaxPooling2D(pool_size=(1,10))(X)
 Xmax = Lambda(lambda x1 : x1*0.3 )(Xmax)
 Xavg = AvgPool2D(pool_size=(1,10))(X)
 Xavg = Lambda(lambda x1 : x1*0.7 )(Xavg)
 X = Add()([Xmax, Xavg])
 X = Lambda(lambda y : K.squeeze(y,2))(X)
 X = BatchNormalization()(X)
 X = Conv1D(160, kernel_size=1, strides=1, activation='relu' )(X)
 X = BatchNormalization()(X)
 X = Conv1D(96, kernel_size=1, strides=1, activation='relu' )(X)
 X = BatchNormalization()(X)
 X = Conv1D(96, kernel_size=1, strides=1, activation='relu' )(X)
 X = BatchNormalization()(X)
 Xmax = MaxPooling1D(pool_size=11 )(X)
 Xmax = Lambda(lambda x1 : x1*0.3 )(Xmax)
 Xavg = AvgPool1D(pool_size=11 )(X)
 Xavg = Lambda(lambda x1 : x1*0.7 )(Xavg)
 X = Add()([Xmax, Xavg])
 X = Lambda(lambda y : K.squeeze(y,1))(X)
 X = Dense(96, activation="relu" )(X)
 X = BatchNormalization()(X)
 X = Dense(256, activation="relu" )(X)
 X = LayerNormalization()(X)
 X = Dropout(0.3 )(X)
 outsoft = Dense(num_classes_y, activation='softmax', name = "output" )(X)
 model = Model(inputs = [inputdense_players], outputs = outsoft)
 return model<prepare_x_and_y>
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 print('Data imported')
 print('Checking shape of datasets')
 print("The are {} rows of data in train dataset and {} columns in training dataset".format(str(df_train.shape[0]), str(df_train.shape[1])))
 print("The are {} rows of data in train dataset and {} columns in training dataset".format(str(df_test.shape[0]), str(df_test.shape[1])))
 train_ID = df_train.iloc[:,0]
test_ID = df_test.iloc[:,0]
class Metric(Callback):
def __init__(self, model, callbacks, data):
super().__init__()
self.model = model
self.callbacks = callbacks
self.data = data
def on_train_begin(self, logs=None):
for callback in self.callbacks:
callback.on_train_begin(logs)
 def on_train_end(self, logs=None):
for callback in self.callbacks:
callback.on_train_end(logs)
 def on_epoch_end(self, batch, logs=None):
X_valid, y_valid = self.data[0], self.data[1]
y_pred = self.model.predict(X_valid)
 y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * X_valid.shape[0])
 logs['val_CRPS'] = val_s
for callback in self.callbacks:
callback.on_epoch_end(batch, logs )<split>
unique_ID = len(set(df_train.Id))
total_ID = df_train.shape[0]
duplicate_ID = total_ID - unique_ID
print("There are " + str(duplicate_ID)+ " duplicate IDs for " + str(total_ID)+ " total rows" )
%%time
models = []
kf = KFold(n_splits=6, shuffle=True)
 score = []
for i,(tdx, vdx)in enumerate(kf.split(train_x, train_y)) :
print(f'Fold : {i}')
 X_train, X_val = train_x[tdx], train_x[vdx],
y_train, y_val = train_y.iloc[tdx]['YardIndexClipped'].values, train_y.iloc[vdx]['YardIndexClipped'].values
season_val = df_season.iloc[vdx]['Season'].values
y_train_values = np.zeros(( len(y_train), num_classes_y), np.int32)
 for irow, row in enumerate(y_train):
y_train_values[(irow, row - min_idx_y)] = 1
y_val_values = np.zeros(( len(y_val), num_classes_y), np.int32)
 for irow, row in enumerate(y_val - min_idx_y):
y_val_values[(irow, row)] = 1
val_idx = np.where(season_val!=2017)
 X_val = X_val[val_idx]
y_val_values = y_val_values[val_idx]
y_train_values = y_train_values.astype('float32')
 y_val_values = y_val_values.astype('float32')
 model = get_conv_net(num_classes_y)
 es = EarlyStopping(monitor='val_CRPS',
mode='min',
restore_best_weights=True,
verbose=0,
patience=10)
 es.set_model(model)
 metric = Metric(model, [es], [X_val, y_val_values])
 lr_i = 1e-3
lr_f = 5e-4
n_epochs = 30
decay =(1-lr_f/lr_i)/(( lr_f/lr_i)* n_epochs - 1)
 alpha =(lr_i*(1+decay))
opt = Adam(learning_rate=1e-3)
 model.compile(loss=crps,
optimizer=opt)
 model.fit(X_train,
y_train_values,
epochs=n_epochs,
batch_size=64,
verbose=0,
callbacks=[metric],
validation_data=(X_val, y_val_values))
val_crps_score = min(model.history.history['val_CRPS'])
 print("Val loss: {}".format(val_crps_score))
score.append(val_crps_score)
 models.append(model)
 print(np.mean(score))<compute_test_metric>
df_train['Above Ground Standardised'] = np.abs(stats.zscore(df_train.GrLivArea))
df_train['Above Ground Standardised'].nlargest(10 ).to_frame()
print(np.mean(score))<feature_engineering>
df_train['Selling Price Standardised'] = np.abs(stats.zscore(df_train.SalePrice))
df_train[df_train['Above Ground Standardised']>4]
def split_play_and_player_cols(df,predicting=False):
df['IsRusher'] = df['NflId'] == df['NflIdRusher']
play_ids = df["PlayId"].unique()
player_cols = [
'PlayId',
'Season',
'Team',
'X',
'Y',
'S',
'A',
'Dis',
'Dir',
'NflId',
'IsRusher',
]
df_players = df[player_cols]
play_cols = [
'PlayId',
'Season',
'PossessionTeam',
'HomeTeamAbbr',
'VisitorTeamAbbr',
'PlayDirection',
'FieldPosition',
'YardLine',
]
if not predicting:
play_cols.append('Yards')
 df_play = df[play_cols].copy()
df_play = df_play.groupby('PlayId' ).first().reset_index()
assert df_play.PlayId.nunique() == df.PlayId.nunique() , "Play/player split failed?"
return df_play, df_players<split>
df_train = df_train.drop(df_train[(df_train['Above Ground Standardised']> 4)].index)
 print('Outliers dropped successfully' )
def extract_features_one_play(df_test):
df_play, df_players_split = split_play_and_player_cols(df_test, predicting=True)
 process_team_abbr(df_play)
 process_play_direction(df_play)
 process_yard_til_end_zone(df_play)
 df_players = df_players_split.merge(
 df_play[['PlayId','PossessionTeam','HomeTeamAbbr','PlayDirection', 'Yardline100']],
how='left', on='PlayId')
 standarize_direction(df_players)
 process_tracking_data(df_players)
 df_all_feats = df_players[tracking_level_features]
train_x = np.zeros([1,11,10,10])
 [[rusher_x, rusher_y, rusher_Sx, rusher_Sy]] = df_all_feats.loc[df_all_feats.IsRusher==1,['X_std', 'Y_std','Sx','Sy']].values
offense_ids = df_all_feats[df_all_feats.IsOnOffense & ~df_all_feats.IsRusher].index
defense_ids = df_all_feats[~df_all_feats.IsOnOffense].index
for j, defense_id in enumerate(defense_ids):
[def_x, def_y, def_Sx, def_Sy] = df_all_feats.loc[defense_id,['X_std', 'Y_std','Sx','Sy']].values
[def_rusher_x, def_rusher_y] = df_all_feats.loc[defense_id,['player_minus_rusher_x', 'player_minus_rusher_y']].values
[def_rusher_Sx, def_rusher_Sy] = df_all_feats.loc[defense_id,['player_minus_rusher_Sx', 'player_minus_rusher_Sy']].values
train_x[0,j,:,:4] = df_all_feats.loc[offense_ids,['Sx','Sy','X_std', 'Y_std']].values - np.array([def_Sx, def_Sy, def_x,def_y])
 train_x[0,j,:,-6:] = [def_rusher_Sx, def_rusher_Sy, def_rusher_x, def_rusher_y, def_Sx, def_Sy]
yardline100 = df_play['Yardline100'].values
return [train_x, yardline100]<feature_engineering>
df_train = df_train.drop(['Above Ground Standardised','Selling Price Standardised', "TotalBathroom"], axis = 1 )
iter_test = env.iter_test()
for iplay,(test_df, sample_prediction_df)in enumerate(iter_test):
if iplay % 10 == 0:
print('Processing test play {}...'.format(iplay))
[inference_row, yardline100] = extract_features_one_play(test_df)
 cdfs_arr = []
for model in models:
cdfs_, pdfs_ = get_cdf_prediction_model(
 inference_row,
n_classes=199,
model=model,
min_idx=min_idx_y,
max_idx=max_idx_y,
yardline100=yardline100)
 cdfs_arr.append(cdfs_)
 cdfs = np.mean(cdfs_arr, axis=0)
 cdf_val = cdfs[0].clip(0.0,1.0)
 for icol, col in enumerate(sample_prediction_df.columns):
sample_prediction_df.loc[:, col] = cdf_val[icol]
env.predict(sample_prediction_df )<save_to_csv>
df_train_ends = df_train.shape[0]
df_train1 = df_train.copy()
df_test1 = df_test.copy()
df_total = pd.concat(( df_train1, df_test1))
df_total = df_total.drop('Id', axis = 1 )
env.write_submission_file()
print([filename for filename in os.listdir('/kaggle/working')if '.csv' in filename] )<import_modules>
print("There are {} anomalies in the YearBuilt variable".format(df_total[(df_total['YearBuilt'] > 2010)|(df_total['YearBuilt'] < 1858)]['YearBuilt'].count()))
print("There are {} anomalies in the YearRemodAdd variable".format(df_total[(df_total['YearRemodAdd'] > 2010)|(df_total['YearRemodAdd'] < 1858)]['YearRemodAdd'].count()))
print("There are {} anomalies in the GarageYrBlt variable".format(df_total[(df_total['GarageYrBlt'] > 2010)|(df_total['GarageYrBlt'] < 1858)]['GarageYrBlt'].count()))
print("There are {} anomalies in the YrSold variable".format(df_total[(df_total['YrSold'] > 2010)|(df_total['YrSold'] < 2006)]['YrSold'].count()))
start_time = time.time()
time_limit = 9000
warnings.filterwarnings("ignore")
 env = nflrush.make_env()<set_options>
df_total[(df_total['GarageYrBlt'] > 2010)|(df_total['GarageYrBlt'] < 1858)][['YearBuilt', 'YearRemodAdd', 'GarageYrBlt']]
random.seed(721)
 os.environ['PYTHONHASHSEED'] = str(831)
 np.random.seed(101)
 torch.manual_seed(1111)
 torch.cuda.manual_seed(1117)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False<set_options>
df_total.loc[1132,'GarageYrBlt'] = '2007'
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 if device == "cuda:0":
torch.cuda.set_device(0)
 debug = True
else:
debug = False
debug = False
device<normalization>
df_total_na =(df_total.isnull().sum() / len(df_total)) * 100
df_total_na = df_total_na.drop(df_total_na[df_total_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Percentage of data missing' :df_total_na})
 missing_data.head(20 )
class Mish(nn.Module):
def __init__(self):
super().__init__()
def forward(self, x):
return x *(torch.tanh(F.softplus(x)))
 def gelu(x):
return 0.5 * x *(1 + torch.tanh(math.sqrt(2 / math.pi)*(x + 0.044715 * torch.pow(x, 3))))
def _get_activation_fn(activation):
if activation == "relu":
return F.relu
elif activation == "gelu":
return gelu
else:
raise RuntimeError("activation should be relu/gelu, not %s." % activation)
 def multi_head_attention_forward(query,
key,
value,
embed_dim_to_check,
num_heads,
in_proj_weight,
in_proj_bias,
bias_k,
bias_v,
add_zero_attn,
dropout_p,
out_proj_weight,
out_proj_bias,
training=True,
key_padding_mask=None,
need_weights=True,
attn_mask=None,
use_separate_proj_weight=False,
q_proj_weight=None,
k_proj_weight=None,
v_proj_weight=None,
static_k=None,
static_v=None 
):
r
qkv_same = torch.equal(query, key)and torch.equal(key, value)
 kv_same = torch.equal(key, value)
 tgt_len, bsz, embed_dim = query.size()
assert embed_dim == embed_dim_to_check
assert list(query.size())== [tgt_len, bsz, embed_dim]
assert key.size() == value.size()
head_dim = embed_dim // num_heads
assert head_dim * num_heads == embed_dim, "embed_dim must be divisible by num_heads"
scaling = float(head_dim)** -0.5
if use_separate_proj_weight is not True:
if qkv_same:
q, k, v = F.linear(query, in_proj_weight, in_proj_bias ).chunk(3, dim=-1)
 elif kv_same:
_b = in_proj_bias
_start = 0
_end = embed_dim
_w = in_proj_weight[_start:_end, :]
if _b is not None:
_b = _b[_start:_end]
q = F.linear(query, _w, _b)
 if key is None:
assert value is None
k = None
v = None
else:
_b = in_proj_bias
_start = embed_dim
_end = None
_w = in_proj_weight[_start:, :]
if _b is not None:
_b = _b[_start:]
k, v = F.linear(key, _w, _b ).chunk(2, dim=-1)
 else:
_b = in_proj_bias
_start = 0
_end = embed_dim
_w = in_proj_weight[_start:_end, :]
if _b is not None:
_b = _b[_start:_end]
q = F.linear(query, _w, _b)
 _b = in_proj_bias
_start = embed_dim
_end = embed_dim * 2
_w = in_proj_weight[_start:_end, :]
if _b is not None:
_b = _b[_start:_end]
k = F.linear(key, _w, _b)
 _b = in_proj_bias
_start = embed_dim * 2
_end = None
_w = in_proj_weight[_start:, :]
if _b is not None:
_b = _b[_start:]
v = F.linear(value, _w, _b)
 else:
q_proj_weight_non_opt = torch.jit._unwrap_optional(q_proj_weight)
 len1, len2 = q_proj_weight_non_opt.size()
assert len1 == embed_dim and len2 == query.size(-1)
 k_proj_weight_non_opt = torch.jit._unwrap_optional(k_proj_weight)
 len1, len2 = k_proj_weight_non_opt.size()
assert len1 == embed_dim and len2 == key.size(-1)
 v_proj_weight_non_opt = torch.jit._unwrap_optional(v_proj_weight)
 len1, len2 = v_proj_weight_non_opt.size()
assert len1 == embed_dim and len2 == value.size(-1)
 if in_proj_bias is not None:
q = F.linear(query, q_proj_weight_non_opt, in_proj_bias[0:embed_dim])
 k = F.linear(key, k_proj_weight_non_opt, in_proj_bias[embed_dim:(embed_dim * 2)])
 v = F.linear(value, v_proj_weight_non_opt, in_proj_bias[(embed_dim * 2):])
 else:
q = F.linear(query, q_proj_weight_non_opt, in_proj_bias)
 k = F.linear(key, k_proj_weight_non_opt, in_proj_bias)
 v = F.linear(value, v_proj_weight_non_opt, in_proj_bias)
 q = q * scaling
if bias_k is not None and bias_v is not None:
if static_k is None and static_v is None:
k = torch.cat([k, bias_k.repeat(1, bsz, 1)])
 v = torch.cat([v, bias_v.repeat(1, bsz, 1)])
 if attn_mask is not None:
attn_mask = torch.cat([attn_mask,
torch.zeros(( attn_mask.size(0), 1),
dtype=attn_mask.dtype,
device=attn_mask.device)], dim=1)
 if key_padding_mask is not None:
key_padding_mask = torch.cat(
 [key_padding_mask, torch.zeros(( key_padding_mask.size(0), 1),
dtype=key_padding_mask.dtype,
device=key_padding_mask.device)], dim=1)
 else:
assert static_k is None, "bias cannot be added to static key."
assert static_v is None, "bias cannot be added to static value."
else:
assert bias_k is None
assert bias_v is None
q = q.contiguous().view(tgt_len, bsz * num_heads, head_dim ).transpose(0, 1)
 if k is not None:
k = k.contiguous().view(-1, bsz * num_heads, head_dim ).transpose(0, 1)
 if v is not None:
v = v.contiguous().view(-1, bsz * num_heads, head_dim ).transpose(0, 1)
 if static_k is not None:
assert static_k.size(0)== bsz * num_heads
assert static_k.size(2)== head_dim
k = static_k
if static_v is not None:
assert static_v.size(0)== bsz * num_heads
assert static_v.size(2)== head_dim
v = static_v
src_len = k.size(1)
 if key_padding_mask is not None:
assert key_padding_mask.size(0)== bsz
assert key_padding_mask.size(1)== src_len
if add_zero_attn:
src_len += 1
k = torch.cat([k, torch.zeros(( k.size(0), 1)+ k.size() [2:], dtype=k.dtype, device=k.device)], dim=1)
 v = torch.cat([v, torch.zeros(( v.size(0), 1)+ v.size() [2:], dtype=v.dtype, device=v.device)], dim=1)
 if attn_mask is not None:
attn_mask = torch.cat([attn_mask, torch.zeros(( attn_mask.size(0), 1),
dtype=attn_mask.dtype,
device=attn_mask.device)], dim=1)
 if key_padding_mask is not None:
key_padding_mask = torch.cat(
 [key_padding_mask, torch.zeros(( key_padding_mask.size(0), 1),
dtype=key_padding_mask.dtype,
device=key_padding_mask.device)], dim=1)
 attn_output_weights = torch.bmm(q, k.transpose(1, 2))
assert list(attn_output_weights.size())== [bsz * num_heads, tgt_len, src_len]
if attn_mask is not None:
attn_mask = attn_mask.unsqueeze(0)
 attn_output_weights += attn_mask
if key_padding_mask is not None:
attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
 attn_output_weights = attn_output_weights.masked_fill(
 key_padding_mask.unsqueeze(1 ).unsqueeze(2),
float('-inf'), 
)
 attn_output_weights = attn_output_weights.view(bsz * num_heads, tgt_len, src_len)
 attn_output_weights = F.softmax(
 attn_output_weights, dim=-1)
 attn_output_weights = F.dropout(attn_output_weights, p=dropout_p, training=training)
 attn_output = torch.bmm(attn_output_weights, v)
 assert list(attn_output.size())== [bsz * num_heads, tgt_len, head_dim]
attn_output = attn_output.transpose(0, 1 ).contiguous().view(tgt_len, bsz, embed_dim)
 attn_output = F.linear(attn_output, out_proj_weight, out_proj_bias)
 if need_weights:
attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
 return attn_output, attn_output_weights.sum(dim=1)/ num_heads
else:
return attn_output, None
class MultiheadAttention(nn.Module):
r
def __init__(self, embed_dim, num_heads, dropout=0., bias=True, add_bias_kv=False, add_zero_attn=False, kdim=None, vdim=None):
super(MultiheadAttention, self ).__init__()
self.embed_dim = embed_dim
self.kdim = kdim if kdim is not None else embed_dim
self.vdim = vdim if vdim is not None else embed_dim
self._qkv_same_embed_dim = self.kdim == embed_dim and self.vdim == embed_dim
self.num_heads = num_heads
self.dropout = dropout
self.head_dim = embed_dim // num_heads
assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads"
self.in_proj_weight = Parameter(torch.empty(3 * embed_dim, embed_dim))
if self._qkv_same_embed_dim is False:
self.q_proj_weight = Parameter(torch.Tensor(embed_dim, embed_dim))
self.k_proj_weight = Parameter(torch.Tensor(embed_dim, self.kdim))
self.v_proj_weight = Parameter(torch.Tensor(embed_dim, self.vdim))
if bias:
self.in_proj_bias = Parameter(torch.empty(3 * embed_dim))
else:
self.register_parameter('in_proj_bias', None)
 self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
 if add_bias_kv:
self.bias_k = Parameter(torch.empty(1, 1, embed_dim))
self.bias_v = Parameter(torch.empty(1, 1, embed_dim))
else:
self.bias_k = self.bias_v = None
self.add_zero_attn = add_zero_attn
self._reset_parameters()
def _reset_parameters(self):
if self._qkv_same_embed_dim:
xavier_uniform_(self.in_proj_weight)
 else:
xavier_uniform_(self.q_proj_weight)
 xavier_uniform_(self.k_proj_weight)
 xavier_uniform_(self.v_proj_weight)
 if self.in_proj_bias is not None:
constant_(self.in_proj_bias, 0.)
constant_(self.out_proj.bias, 0.)
if self.bias_k is not None:
xavier_normal_(self.bias_k)
 if self.bias_v is not None:
xavier_normal_(self.bias_v)
 def forward(self, query, key, value, key_padding_mask=None,
need_weights=True, attn_mask=None):
r
if hasattr(self, '_qkv_same_embed_dim')and self._qkv_same_embed_dim is False:
return multi_head_attention_forward(
 query, key, value, self.embed_dim, self.num_heads,
self.in_proj_weight, self.in_proj_bias,
self.bias_k, self.bias_v, self.add_zero_attn,
self.dropout, self.out_proj.weight, self.out_proj.bias,
training=self.training,
key_padding_mask=key_padding_mask, need_weights=need_weights,
attn_mask=attn_mask, use_separate_proj_weight=True,
q_proj_weight=self.q_proj_weight, k_proj_weight=self.k_proj_weight,
v_proj_weight=self.v_proj_weight)
 else:
if not hasattr(self, '_qkv_same_embed_dim'):
warnings.warn('A new version of MultiheadAttention module has been implemented.\
Please re-train your model with the new module',
UserWarning)
 return multi_head_attention_forward(
 query, key, value, self.embed_dim, self.num_heads,
self.in_proj_weight, self.in_proj_bias,
self.bias_k, self.bias_v, self.add_zero_attn,
self.dropout, self.out_proj.weight, self.out_proj.bias,
training=self.training,
key_padding_mask=key_padding_mask, need_weights=need_weights,
attn_mask=attn_mask)
 class TransformerEncoderLayer(nn.Module):
r
def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation="relu"):
super(TransformerEncoderLayer, self ).__init__()
self.self_attn = MultiheadAttention(d_model, nhead, dropout=dropout)
 self.linear1 = nn.Linear(d_model, dim_feedforward)
 self.dropout = nn.Dropout(dropout)
 self.linear2 = nn.Linear(dim_feedforward, d_model)
 self.norm1 = nn.LayerNorm(d_model)
 self.norm2 = nn.LayerNorm(d_model)
 self.dropout1 = nn.Dropout(dropout)
 self.dropout2 = nn.Dropout(dropout)
 self.activation = _get_activation_fn(activation)
 def forward(self, src, src_mask=None, src_key_padding_mask=None):
r
src2 = self.self_attn(src, src, src, attn_mask=src_mask,
key_padding_mask=src_key_padding_mask)[0]
src = src + self.dropout1(src2)
 src = self.norm1(src)
 if hasattr(self, "activation"):
src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))
else:
src2 = self.linear2(self.dropout(F.relu(self.linear1(src))))
src = src + self.dropout2(src2)
 src = self.norm2(src)
 return src
class TransformerDecoderLayer(nn.Module):
r
def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation="relu"):
super(TransformerDecoderLayer, self ).__init__()
self.self_attn = MultiheadAttention(d_model, nhead, dropout=dropout)
 self.multihead_attn = MultiheadAttention(d_model, nhead, dropout=dropout)
 self.linear1 = nn.Linear(d_model, dim_feedforward)
 self.dropout = nn.Dropout(dropout)
 self.linear2 = nn.Linear(dim_feedforward, d_model)
 self.norm1 = nn.LayerNorm(d_model)
 self.norm2 = nn.LayerNorm(d_model)
 self.norm3 = nn.LayerNorm(d_model)
 self.dropout1 = nn.Dropout(dropout)
 self.dropout2 = nn.Dropout(dropout)
 self.dropout3 = nn.Dropout(dropout)
 self.activation = _get_activation_fn(activation)
 def forward(self, tgt, memory, tgt_mask=None, memory_mask=None,
tgt_key_padding_mask=None, memory_key_padding_mask=None):
r
tgt2 = self.self_attn(tgt, tgt, tgt, attn_mask=tgt_mask,
key_padding_mask=tgt_key_padding_mask)[0]
tgt = tgt + self.dropout1(tgt2)
 tgt = self.norm1(tgt)
 tgt2 = self.multihead_attn(tgt, memory, memory, attn_mask=memory_mask,
key_padding_mask=memory_key_padding_mask)[0]
tgt = tgt + self.dropout2(tgt2)
 tgt = self.norm2(tgt)
 if hasattr(self, "activation"):
tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
else:
tgt2 = self.linear2(self.dropout(F.relu(self.linear1(tgt))))
tgt = tgt + self.dropout3(tgt2)
 tgt = self.norm3(tgt)
 return tgt
class RAdam(Optimizer):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):
defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 self.buffer = [[None, None, None] for ind in range(10)]
super(RAdam, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(RAdam, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
buffered = self.buffer[int(state['step'] % 10)]
if state['step'] == buffered[0]:
N_sma, step_size = buffered[1], buffered[2]
else:
buffered[0] = state['step']
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 buffered[1] = N_sma
if N_sma >= 5:
step_size = group['lr'] * math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 else:
step_size = group['lr'] /(1 - beta1 ** state['step'])
 buffered[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma >= 5:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 else:
p_data_fp32.add_(-step_size, exp_avg)
 p.data.copy_(p_data_fp32)
 return loss
class Lookahead(Optimizer):
def __init__(self, optimizer, k=5, alpha=0.5):
self.optimizer = optimizer
self.k = k
self.alpha = alpha
self.param_groups = self.optimizer.param_groups
self.state = defaultdict(dict)
 self.fast_state = self.optimizer.state
for group in self.param_groups:
group["counter"] = 0
def update(self, group):
for fast in group["params"]:
param_state = self.state[fast]
if "slow_param" not in param_state:
param_state["slow_param"] = torch.zeros_like(fast.data)
 param_state["slow_param"].copy_(fast.data)
 slow = param_state["slow_param"]
slow +=(fast.data - slow)* self.alpha
fast.data.copy_(slow)
 def update_lookahead(self):
for group in self.param_groups:
self.update(group)
 def step(self, closure=None):
loss = self.optimizer.step(closure)
 for group in self.param_groups:
if group["counter"] == 0:
self.update(group)
 group["counter"] += 1
if group["counter"] >= self.k:
group["counter"] = 0
return loss
def state_dict(self):
fast_state_dict = self.optimizer.state_dict()
slow_state = { 
(id(k)if isinstance(k, torch.Tensor)else k): v
for k, v in self.state.items()
}
fast_state = fast_state_dict["state"]
param_groups = fast_state_dict["param_groups"]
return {
"fast_state": fast_state,
"slow_state": slow_state,
"param_groups": param_groups,
}
def load_state_dict(self, state_dict):
slow_state_dict = {
"state": state_dict["slow_state"],
"param_groups": state_dict["param_groups"],
}
fast_state_dict = {
"state": state_dict["fast_state"],
"param_groups": state_dict["param_groups"],
}
super(Lookahead, self ).load_state_dict(slow_state_dict)
 self.optimizer.load_state_dict(fast_state_dict)
 self.fast_state = self.optimizer.state
def add_param_group(self, param_group):
param_group["counter"] = 0
self.optimizer.add_param_group(param_group )<load_from_csv>
df_total['LotFrontage'] = df_total.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))
train_df = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', low_memory=False )<define_variables>
df_total['MSZoning'] = df_total['MSZoning'].fillna(df_total['MSZoning'].mode() [0] )
num_cols = ['IsRusher',
'IsRusherTeam',
'X','Y',
'Dir_X','Dir_Y',
'Orientation_X','Orientation_Y',
'S',
'DistanceToBall',
'BallDistanceX','BallDistanceY',
'BallAngleX','BallAngleY',
'related_horizontal_v','related_vertical_v',
'related_horizontal_A','related_vertical_A',
'TeamDistance','EnermyTeamDistance',
'TeamXstd','EnermyXstd',
'EnermyYstd','TeamYstd',
'DistanceToBallRank',
'DistanceToBallRank_AttTeam','DistanceToBallRank_DefTeam',
'YardLine',
'NextX','NextY',
'NextDistanceToBall',
'BallNextAngleX','BallNextAngleY',
'BallNextDistanceX','BallNextDistanceY',
'A',
]
map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}
for abb in train_df['PossessionTeam'].unique() :
map_abbr[abb] = abb
def Create_group_features(df,train=True):
df['PossessionTeam'] = df['PossessionTeam'].map(map_abbr)
 df['HomeTeamAbbr'] = df['HomeTeamAbbr'].map(map_abbr)
 df['Season'] =(df['Season'] == 2017)
 df.loc[df['Season'],'A'] = np.nan
df['S'] = 10*df['Dis']
df.loc[df['Season'],'Orientation'] = np.mod(90+df.loc[df['Season'],'Orientation'],360)
 mask = df['PlayDirection'] == 'left'
df.loc[mask, 'X'] = 120 - df.loc[mask, 'X']
df.loc[mask, 'Y'] =(160 / 3)- df.loc[mask, 'Y']
df.loc[mask, 'Orientation'] = np.mod(180 + df.loc[mask, 'Orientation'], 360)
 df.loc[mask, 'Dir'] = np.mod(180 + df.loc[mask, 'Dir'], 360)
 df['FieldPosition'].fillna('', inplace=True)
 mask = df['PossessionTeam'] != df['FieldPosition']
df.loc[mask, 'YardLine'] = 100 - df.loc[mask, 'YardLine']
df['PlayDirection'] = df['PlayDirection'].map({'left':1,'right':0})
 df['Team'] = df['Team'].map({'away':1,'home':0})
 df['Dir'] = np.mod(90+df['Dir'],360)
 df['Orientation'] = np.mod(90+df['Orientation'],360)
 df['Orientation_X'] = np.cos(df['Orientation']*2*np.pi/360)
 df['Orientation_Y'] = np.sin(df['Orientation']*2*np.pi/360)
 df['Dir_X'] = np.cos(df['Dir']*2*np.pi/360)
 df['Dir_Y'] = np.sin(df['Dir']*2*np.pi/360)
 df['IsRusher'] =(df['NflId'] == df['NflIdRusher'])
 rushers = df[df['IsRusher']][['PlayId', 'Team', 'X', 'Y', 'A', 'S', 'Dir_X','Dir_Y']]
rushers.columns = ['PlayId', 'RusherTeam', 'BallX', 'BallY', 'BallA', 'BallS', 'BallDir_X', 'BallDir_Y']
df = df.join(rushers.set_index('PlayId'), on='PlayId')
 df['IsRusherTeam'] =(df['Team'] == df['RusherTeam'])
 df['DistanceToBall'] =(( df['X'] - df['BallX'])**2 +(df['Y'] - df['BallY'])**2)**0.5
dx = df['BallX']-df['X']
dy = df['BallY']-df['Y']
angle = np.arctan(dy/(1e-8+dx)) *360/(2*np.pi)
 angle[dx<0] += 180
angle[(dx>0)&(dy<0)] += 360
angle = np.mod(angle,360)
 df['BallAngleX'] = np.sin(angle*2*np.pi/360)
 df['BallAngleY'] = np.cos(angle*2*np.pi/360)
 df['BallDistanceX'] = -dx
df['BallDistanceY'] = -dy
df['related_horizontal_v'] = df['S']*df['Dir_X'] - df['BallS']*df['BallDir_X']
df['related_vertical_v'] = df['S']*df['Dir_Y'] - df['BallS']*df['BallDir_Y']
df['related_horizontal_A'] = df['A']*df['Dir_X'] - df['BallA']*df['BallDir_X']
df['related_vertical_A'] = df['A']*df['Dir_Y'] - df['BallA']*df['BallDir_Y']
df['DistanceToBallRank'] = df.groupby('PlayId')['DistanceToBall'].rank()
Distance_Ball_Team = df.groupby(['PlayId','Team'])['DistanceToBall'].rank()
df['DistanceToBallRank_AttTeam'] = np.nan
df['DistanceToBallRank_DefTeam'] = np.nan
df.loc[df['IsRusherTeam'],'DistanceToBallRank_AttTeam'] = Distance_Ball_Team.loc[df['IsRusherTeam']]
df.loc[~df['IsRusherTeam'],'DistanceToBallRank_DefTeam'] = Distance_Ball_Team.loc[~df['IsRusherTeam']]
TeamXY = df.groupby(['PlayId','Team'])[['X','Y']].transform(np.mean)
 df['TeamDistance'] =(( df['X'] - TeamXY['X'])**2 +(df['Y'] - TeamXY['Y'])**2)**0.5
df['TeamXstd'] = df['X'] - TeamXY['X']
df['TeamYstd'] = df['Y'] - TeamXY['Y']
EnermyTeamXY = TeamXY.groupby(df['PlayId'] ).apply(lambda x: x.iloc[::-1] ).values
EnermyTeamXY = pd.DataFrame(EnermyTeamXY,index = TeamXY.index,columns = TeamXY.columns)
 df['EnermyTeamDistance'] =(( df['X'] - EnermyTeamXY['X'])**2 +(df['Y'] - EnermyTeamXY['Y'])**2)**0.5
df['EnermyXstd'] = df['X'] - EnermyTeamXY['X']
df['EnermyYstd'] = df['Y'] - EnermyTeamXY['Y']
df['NextX'] = df['S']*df['Dir_X']+df['X']
df['NextY'] = df['S']*df['Dir_Y']+df['Y']
rushers = df[df['IsRusher']][['PlayId', 'NextX', 'NextY']]
rushers.columns = ['PlayId', 'BallNextX', 'BallNextY']
df = df.join(rushers.set_index('PlayId'), on='PlayId')
 df['NextDistanceToBall'] =(( df['NextX'] - df['BallNextX'])**2 +(df['NextY'] - df['BallNextY'])**2)**0.5
dx = df['BallNextX']-df['NextX']
dy = df['BallNextY']-df['NextY']
angle = np.arctan(dy/(1e-8+dx)) *360/(2*np.pi)
 angle[dx<0] += 180
angle[(dx>0)&(dy<0)] += 360
angle = np.mod(angle,360)
 df['BallNextAngleX'] = np.sin(angle*2*np.pi/360)
 df['BallNextAngleY'] = np.cos(angle*2*np.pi/360)
 df['BallNextDistanceX'] = -dx
df['BallNextDistanceY'] = -dy
return df[num_cols]<prepare_x_and_y>
df_total['MSSubClass'] = df_total.groupby('Neighborhood')['MSSubClass'].transform(lambda x: x.fillna(x.mode()))
%%time
groups =(train_df['GameId'].astype(str ).str[:4] + train_df['Week'].astype(str))
groups = groups.groupby(train_df['PlayId'] ).apply(lambda x:x.iloc[0] ).values
train_y = train_df['Yards'].groupby(train_df['PlayId'] ).apply(lambda x:x.iloc[0] ).values
train_X_df = Create_group_features(train_df.copy())
 bool_cols = ['IsRusher','IsRusherTeam']
Scale_cols = [col for col in train_X_df.columns if col not in bool_cols]
X_mean = train_X_df[Scale_cols].mean()
X_std = train_X_df[Scale_cols].std()
train_X = train_X_df.copy()
train_X[bool_cols] = train_X[bool_cols].astype(np.float32)
 train_X[Scale_cols] =(train_X_df[Scale_cols] - X_mean[Scale_cols])/X_std[Scale_cols]
train_X.fillna(0,inplace=True)
 train_X = train_X.values.reshape(len(train_y),22,-1)
 train_X.shape<concatenate>
df_total['GarageYrBlt'] = df_total['GarageYrBlt'].fillna(df_total['YearBuilt'] )
n = np.arange(-99, 100)
 n = np.row_stack([n] * train_y.shape[0])
 ym = train_y.reshape(train_y.shape[0], 1)
 train_y = np.heaviside(n - ym, 1 )<define_variables>
modes = ["Functional", 'Electrical', "Utilities",'KitchenQual', 'Exterior1st', 'Exterior2nd', 'SaleType']
for col in modes:
df_total[col] = df_total[col].fillna(df_total[col].mode() [0] )
Lower_yards = np.heaviside(99 - train_X_df['YardLine'].values[::22].reshape(train_y.shape[0], 1)- np.row_stack([np.arange(199)] * train_y.shape[0]),0)
 Upper_yards = np.heaviside(np.row_stack([np.arange(199)] * train_y.shape[0])- 199 + train_X_df['YardLine'].values[::22].reshape(train_y.shape[0], 1),0)
 Yard_mask = Lower_yards + Upper_yards<prepare_x_and_y>
cats = ["PoolQC", "Fence", "MiscFeature", "Alley", "FireplaceQu", "MasVnrType", 'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2']
for col in cats:
df_total[col] = df_total[col].fillna('None' )
train_y_OH = np.zeros([train_y.shape[0],199])
 train_y_OH[np.arange(len(ym)) ,(ym+99 ).reshape(-1)] = 1<choose_model_class>
nums = ['MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath', 'GarageCars', 'GarageArea']
for col in nums:
df_total[col] = df_total[col].fillna(0)
 print('Finished imputing' )
class Model(nn.Module):
def __init__(self,params):
super(Model, self ).__init__()
Activation = params['Activation']
self.layers = params['layer']
self.emb = Seq(nn.Dropout(params['dropout_0']),Lin(params['input_size'],params['hidden_0']))
self.transformer_encoder = nn.ModuleList()
self.transformer_decoder = nn.ModuleList()
for l in range(self.layers):
self.transformer_encoder.append(TransformerEncoderLayer(params['hidden_0'],\
nhead=params['head'],\
dim_feedforward=params['hidden_1'],\
dropout=params['dropout_1'],\
activation = params['transformer_activation']))
self.transformer_decoder.append(TransformerDecoderLayer(params['hidden_0'],\
nhead=params['head'],\
dim_feedforward=params['hidden_1'],\
dropout=params['dropout_1'],\
activation = params['transformer_activation']))
self.mlp = Seq(Lin(params['hidden_0'], params['hidden_3']),Activation,nn.Dropout(params['dropout_3']),Lin(params['hidden_3'], 199))
def forward(self, X):
rusher_mask = X[:,:,0]>0
combine = self.emb(X)
 x = combine.permute(1,0,2)
 for l in range(self.layers):
x = self.transformer_encoder[l](x)
 rusher =(x.permute(1,0,2)[rusher_mask].unsqueeze(1)).permute(1,0,2)
 for l in range(self.layers):
rusher = self.transformer_decoder[l](rusher,x)
 x = rusher.permute(1,0,2)
 y = x.squeeze(1)
 y = self.mlp(y)
 y = F.softmax(y,dim=-1)
 return y,torch.cumsum(y,dim = -1 )<choose_model_class>
df_total_na =(df_total.isnull().sum() / len(df_total)) * 100
df_total_na = df_total_na.drop(df_total_na[df_total_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Percentage missing' :df_total_na})
 missing_data.head()
def Schedule_scheme(x):
if x%3==0:
return 1
elif x%3==1:
return 0.5
else:
return 0.2
def Create_model(params,input_size = train_X.shape[-1]):
params['input_size'] = input_size
clf = Model(params ).to(device)
 vv = 0
for v in list(clf.parameters()):
vv += len(v.view(-1))
print("Num_Variables:",vv)
 base_opt = RAdam(clf.parameters() ,lr=params['lr'],\
weight_decay=params['weight_decay'],\
betas=(1-params['alpha'], 1-params['beta']), eps=params['eps'] 
)
 optimizer = Lookahead(base_opt, k=params['lookahead_k'], alpha=params['lookahead_alpha'])
 scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda= lambda x: 1 if x%2==0 else 0.5)
 return clf,optimizer,scheduler<set_options>
df_total['MSSubClass'] = df_total['MSSubClass'].apply(str)
 df_total['YrSold'] = df_total['YrSold'].astype(str)
 df_total['MoSold'] = df_total['MoSold'].astype(str )
config = {'Activation': 'lrelu1',
'alpha': 0.06177912913861566,
'batch_size': 82,
'beta': 0.025662159487370977,
'cosine_T': 67,
'dropout_0': 0.023898897998374913,
'dropout_1': 0.06080646510619758,
'dropout_2': 0.025011299009768506,
'dropout_3': 0.22165717850445166,
'eps': 5.884062604808768e-07,
'head': 6,
'hidden_0': 111,
'hidden_1': 2040,
'hidden_2': 1742,
'hidden_3': 933,
'layer': 4,
'lookahead_alpha': 0.5422623511158232,
'lookahead_k': 3,
'loss_1': 0.48305609964511376,
'loss_2': 0.2353079388007755,
'lr': 0.00014789689708999082,
'min_lr': 0.0038268377963604707,
'transformer_activation': 'gelu',
'weight_decay': 2.2445535063229495e-05}
act_map = {'lrelu1':LeakyReLU(0.1),
'lrelu2':LeakyReLU(0.2),
'relu':ReLU() ,
'selu':SELU() ,
'prelu':nn.PReLU() ,
'elu':nn.ELU() ,
'mish':Mish() }
params = {'Activation': Mish() ,
'dropout_0': config['dropout_0'],
'hidden_0': 128,
'head': 32,
'hidden_1': 512,
"dropout_1": 0.2,
'transformer_activation': config['transformer_activation'],
"hidden_2": config['hidden_2'],
"dropout_2": config['dropout_2'],
'hidden_3': 512,
'dropout_3': 0.3,
"layer": 2,
"lr": 5*config['lr'],
'weight_decay': 1e-1,
'alpha': config['alpha'],
'beta': config['beta'],
'eps': config['eps'],
'lookahead_k': 5,
'lookahead_alpha': 0.5,
'cosine_T': config['cosine_T'],
'min_lr': 0.01*config['lr'] * config['min_lr'],
'batch_size': 32,
'loss_1': 0.5,
'loss_2': 0.5
}<init_hyperparams>
df_total = df_total.replace({"BsmtCond" : {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"BsmtExposure" : {"None" : 0, "No" : 1, "Mn" : 1, "Av": 2, "Gd" : 3},
"BsmtFinType1" : {"None" : 0, "Unf" : 1, "LwQ": 1, "Rec" : 2, "BLQ" : 3,
"ALQ" : 4, "GLQ" : 5},
"BsmtFinType2" : {"None" : 0, "Unf" : 1, "LwQ": 1, "Rec" : 2, "BLQ" : 3,
"ALQ" : 4, "GLQ" : 5},
"BsmtQual" : {"None" : 0, "Po" : 1, "Fa" : 2, "TA": 3, "Gd" : 4, "Ex" : 5},
"ExterCond" : {"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"ExterQual" : {"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"FireplaceQu" : {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"GarageCond" : {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"GarageQual" : {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
'GarageFinish' : {"None" : 0, "Unf" : 1, "RFn" : 2, "Fin" : 3},
"HeatingQC" : {"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"KitchenQual" : {"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"PoolQC" : {"None" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"CentralAir" : {"N" : 0, "Y" : 1}} )
model_list = []
oof = np.zeros([len(train_X),199])
 batch_size = params['batch_size']
for fold_ in range(999):
trn_idx = np.arange(len(train_X))
X_tr,y_tr,y_OH_tr = train_X[trn_idx], train_y[trn_idx],train_y_OH[trn_idx]
tmp = deepcopy(y_OH_tr)
 y_OH_tr[:,:-1] += 0.5 * tmp[:,1:]
y_OH_tr[:,1:] += 0.5 * tmp[:,:-1]
params['cosine_T'] = len(y_tr)//batch_size
clf,optimizer,scheduler = Create_model(params)
 best_loss = 9999
for epoch in tqdm_notebook(range(30)) :
clf.train()
batches = np.arange(len(trn_idx))
np.random.shuffle(batches)
 batches = batches[:batch_size *(len(batches)//batch_size)]
batches = batches.reshape(-1,batch_size)
 Train_pred = []
MA_loss = 0
for batch in batches:
optimizer.zero_grad()
X = torch.Tensor(X_tr[batch] ).to(device)
 y = torch.Tensor(y_tr[batch] ).to(device)
 y_OH = torch.Tensor(y_OH_tr[batch] ).to(device)
 y_pred_OH,y_pred = clf(X)
 loss = torch.mean(( y - y_pred)**2)
 loss2 = -torch.mean(y_OH * torch.log(1e-8 + y_pred_OH)+(1-y_OH)* torch.log(1-y_pred_OH+1e-8)) 
(params['loss_1']*loss+params['loss_2']*loss2 ).backward()
optimizer.step()
MA_loss += loss.item()
if epoch >= 10:
scheduler.step()
MA_loss /= len(batches)
 print("epoch",epoch,"Train:",MA_loss)
 if epoch >= 10 and epoch%2 == 1:
model_list.append(deepcopy(clf))
if time.time() -start_time > time_limit:
break
if time.time() -start_time > time_limit:
print("Overtime")
 break<prepare_x_and_y>
df_total = df_total.replace({"Alley" : {"None" :0, "Grvl" : 1, "Pave" : 2}, "PavedDrive" : {"N" : 1, "P" : 2, "Y" : 3},
"Street" : {"Grvl" : 1, "Pave" : 2}} )
for(test_df, sample_prediction_df)in tqdm_notebook(env.iter_test()):
test_X = Create_group_features(test_df.copy() ,train=False)
 yardline = test_X['YardLine'].iloc[-1]
test_X[bool_cols] = test_X[bool_cols].astype(np.float32)
 test_X[Scale_cols] =(test_X[Scale_cols] - X_mean[Scale_cols])/X_std[Scale_cols]
test_X = torch.Tensor(test_X.fillna(0 ).values.reshape(-1,22,train_X.shape[-1]))
y_pred_p = np.zeros(199)
 for i,model in enumerate(model_list):
model.eval()
pred = model(test_X)[1][-1].detach().numpy()
y_pred_p += pred / len(model_list)
 y_pred_p = np.clip(y_pred_p,0,1)
 y_pred_p[:int(99-yardline)] = 0
y_pred_p[int(199-yardline):] = 1
env.predict(pd.DataFrame(data=[y_pred_p],columns=sample_prediction_df.columns))
env.write_submission_file()<import_modules>
df_total = df_total.replace({"LandSlope" : {"Sev" : 1, "Mod" : 2, "Gtl" : 3} ,
"LotShape" : {"IR3" : 1, "IR2" : 2, "IR1" : 3, "Reg" : 4}} )
%matplotlib inline<define_variables>
df_total = df_total.replace({"Functional" : {"Sal" : 0, "Sev" : 1, "Maj2" : 2, "Maj1" : 2, "Mod":3,
"Min2" : 4, "Min1" : 4, "Typ" : 5}} )
TRAIN_DATA_PATH = Path('.. /input/nfl-big-data-bowl-2020/train.csv')
 SEED = 0<load_from_csv>
df_total = df_total.replace({"Electrical" : {"FuseP" : 1, "FuseF" : 2, "FuseA" : 3, "Mix" : 4, "SBrkr" : 5},
'Utilities': {'NoSeWa':0, "AllPub": 1}} )
train = pd.read_csv(TRAIN_DATA_PATH, dtype={'WindSpeed': 'object'})
 train.head()<filter>
df_total['OverallQual_simple'] = df_total.OverallQual.replace({1:1, 2:1, 3:1,
4:2, 5:2, 6:2,
7:3, 8:3, 9:3, 10:3
})
 df_total['OverallCond_simple'] = df_total.OverallCond.replace({1:1, 2:1, 3:1,
4:2, 5:2, 6:2,
7:3, 8:3, 9:3, 10:3
})
 df_total['ExterQual_simple'] = df_total.ExterQual.replace({5:3, 4:3, 3:2, 2:2, 1:1})
 df_total['ExterCond_simple'] = df_total.ExterCond.replace({5:3, 4:3, 3:2, 2:2, 1:1})
 df_total['BsmtQual_simple'] = df_total.BsmtQual.replace({5:3, 4:3, 3:2, 2:2, 1:1})
 df_total['BsmtCond_simple'] = df_total.BsmtCond.replace({5:3, 4:3, 3:2, 2:2, 1:1})
 df_total['BsmtFinType1_simple'] = df_total.BsmtFinType1.replace({6:3, 5:3, 4:2, 3:2, 2:1, 1:1})
 df_total['BsmtFinType2_simple'] = df_total.BsmtFinType2.replace({6:3, 5:3, 4:2, 3:2, 2:1, 1:1})
 df_total['HeatingQC_simple'] = df_total.HeatingQC.replace({5:3, 4:3, 3:2, 2:2, 1:1})
 df_total['KitchenQual_simple'] = df_total.KitchenQual.replace({5:3, 4:3, 3:2, 2:2, 1:1})
 df_total['Functional_simple'] = df_total.Functional.replace({8:4, 7:3, 7:3, 6:3, 5:2,4:2, 3:1, 2:1})
 df_total['GarageQual_simple'] = df_total.GarageQual.replace({5:3, 4:3, 3:2, 2:2, 1:1})
 df_total['GarageCond_simple'] = df_total.GarageCond.replace({5:3, 4:3, 3:2, 2:2, 1:1})
 df_total['PoolQC_simple'] = df_total.PoolQC.replace({5:3, 4:3, 3:2, 2:2, 1:1})
 df_total['FireplaceQu_simple'] = df_total.FireplaceQu.replace({5:3, 4:3, 3:2, 2:2, 1:1} )
train.loc[509755:509769, ['WindSpeed', 'WindDirection']]<define_variables>
year = df_total[df_total['GarageYrBlt']!='None'][['YearBuilt','YearRemodAdd', 'GarageYrBlt']]
year['GarageYrBlt'] = year['GarageYrBlt'].astype(int)
 year.corr()
replacements = [ 
('LSU', 'Louisiana State'), 
('Southern California', 'USC'), 
('Miami(Fla.) ', 'Miami'), 
('Miami, O.', 'Miami OH'), 
('Miami(Ohio)', 'Miami OH'), 
('Texas-El Paso', 'Texas')
 ]<define_variables>
categorical_features = df_total.select_dtypes(include = ["object"] ).columns
numerical_features = df_total.select_dtypes(exclude = ["object"] ).columns
numerical_features = numerical_features.drop("SalePrice")
 price = df_total['SalePrice']
print("Numerical features : " + str(len(numerical_features)))
 print("Categorical features : " + str(len(categorical_features)))
 df_total_num = df_total[numerical_features]
df_total_cat = df_total[categorical_features]
position_groups = {
'CB' : 'DB',
'FS' : 'DB',
'SAF' : 'DB',
'S' : 'DB',
'SS' : 'DB',
'DB' : 'DB',
'OLB' : 'LB',
'ILB' : 'LB',
'MLB' : 'LB',
'LB' : 'LB',
'DE' : 'DL',
'DT' : 'DL',
'NT' : 'DL',
'DL' : 'DL',
'G' : 'OL',
'OG' : 'OL',
'T' : 'OL',
'OT' : 'OL',
'C' : 'OL',
'RB' : 'RB',
'FB' : 'RB',
'HB' : 'RB',
'WR' : 'SK',
'QB' : 'SK',
'TE' : 'SK'
}
posession_positions = {
'DB': 0,
'LB': 0,
'DL': 0,
'OL': 1,
'RB': 1,
'SK': 1
}<define_variables>
skewness = df_total_num.apply(lambda x: skew(x))
skewness = skewness[abs(skewness)> 0.5]
print(str(skewness.shape[0])+ " skewed numerical features need to be transform")
 skewed_features = skewness.index
df_total_num[skewed_features] = boxcox1p(df_total_num[skewed_features], 0.15 )
COLS_TO_DROP = [
'Position', 'IsRusher', 'WindDirection', 'WindSpeed'
]
CATEGORICAL_VARS = [
'DisplayName', 'PlayerCollegeName', 'Location',
'OffensePersonnel', 'Stadium', 'DefensePersonnel',
'HomeTeamAbbr', 'VisitorTeamAbbr', 'FieldPosition', 'PossessionTeam',
'StadiumType', 'Position', 'Turf', 'PlayerHeight', 'OffenseFormation',
'JerseyNumber', 'NflId'
]
TRANSFORMED_CATEGORICALS = [
'OffensePersonnel', 'DefensePersonnel', 'PlayerHeight',
'NflId', 'Turf', 'FieldPosition', 'PlayerCollegeName', 'JerseyNumber',
'PossessionTeam', 'GameWeather'
]
VARS_TO_ONE_HOT = [var for var in CATEGORICAL_VARS
if var not in TRANSFORMED_CATEGORICALS
and var not in COLS_TO_DROP]
MAX_CARDINALITY = 24<count_unique_values>
df_total = pd.concat([df_total_num, df_total_cat, price], axis = 1 )
cardinalities = train[VARS_TO_ONE_HOT].nunique().sort_values(ascending=False)
 cardinalities<compute_test_metric>
df_total['HouseSF'] = df_total["GrLivArea"] + df_total["TotalBsmtSF"]
def shoelace(x, y):
return 0.5 * np.abs(np.dot(x, np.roll(y,1)) -np.dot(y,np.roll(x,1)) )<categorify>
df_total["AllPorchSF"] = df_total["OpenPorchSF"] + df_total["EnclosedPorch"] + df_total["ScreenPorch"]
def func_timer(fn):
@wraps(fn)
 def timer(*args, **kwargs):
print('
' + fn.__name__.upper())
 start_time = datetime.now()
res = fn(*args, **kwargs)
 end_time = datetime.now()
time_taken = end_time - start_time
time_taken = time_taken.total_seconds() / 60
print('{} time taken: {:.2f} mins'.format(fn.__name__, time_taken))
return res
return timer
def transformation_check(fn):
@wraps(fn)
 def checker(*args, **kwargs):
print(' -- {} -- '.format(fn.__name__))
res = fn(*args, **kwargs)
 end_shape = res.shape
null_prc = 100 *(res.isnull().sum() / len(res)).mean()
print('\tshape = {}'.format(end_shape))
print('\tnull % = {:.2f}'.format(null_prc))
return res
return checker<categorify>
df_total["TotalBathroom"] = df_total["BsmtFullBath"] +(0.5 * df_total["BsmtHalfBath"])+ df_total["FullBath"] +(0.5 * df_total["HalfBath"] )
class Preprocessor(TransformerMixin):
def __init__(self, one_hot_vars, max_cardinality, cols_to_drop=[]):
super().__init__()
self.target = 'Yards'
self.one_hot_vars = one_hot_vars
self.max_cardinality = max_cardinality
self.cols_to_drop = cols_to_drop
self.player_cols = []
self.one_hot_encoder = OneHotEncoder(
 sparse=False,
dtype=np.int,
handle_unknown='ignore' 
)
 self.college_encoding = {}
self.jersey_encoding = {}
self.map_abbr = None
@func_timer
def initial_cleaning(self, X):

 X = self._correct_team_abbreviations(X)
 X = self._encode_player_height(X)
 X = self._process_time_variables(X)
 X = self._fix_wind_variables(X)
 X = self._fix_stadium_type_and_turf(X)
 X = self._map_college_and_pos(X)
 X = self._encode_personnel(X)
 X = self._normalise_positional_data(X)
 X = self._calc_voronoi(X)
 X = self._misc_engineering(X)
 X.drop(columns=self.cols_to_drop, inplace=True)
 return X
@func_timer
def fit(self, X, y=None):
self._get_player_cols(X)
 cardinalities = X[self.one_hot_vars].nunique().to_dict()
one_hot_vars = [var for var in self.one_hot_vars
if cardinalities[var] <= self.max_cardinality]
self.one_hot_encoder.fit(X[one_hot_vars].fillna('unknown'))
self.oh_cols_to_drop = [var for var in self.one_hot_vars
if var not in one_hot_vars]
self.one_hot_vars = one_hot_vars
self.college_encoding = \
X.groupby('PlayerCollegeName')['PlayId'].count().to_dict()
self.jersey_encoding = \
X.groupby('JerseyNumber')['PlayId'].count().to_dict()
return self
@func_timer
def transform(self, X):
X['PlayerCollegeName'] = X['PlayerCollegeName'].map(self.college_encoding)
 X['PlayerCollegeNameRusher'] = \
X['PlayerCollegeNameRusher'].map(self.college_encoding)
 X['JerseyNumber'] = X['JerseyNumber'].map(self.jersey_encoding)
 X = self._apply_one_hot_encoder(X)
 X = self._flatten_player_vars(X)
 return X
@transformation_check
def _correct_team_abbreviations(self, X):
if self.map_abbr is None:
self.map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}
for abb in X['PossessionTeam'].unique() :
self.map_abbr[abb] = abb
X['PossessionTeam'] = X['PossessionTeam'].map(self.map_abbr)
 X['HomeTeamAbbr'] = X['HomeTeamAbbr'].map(self.map_abbr)
 X['VisitorTeamAbbr'] = X['VisitorTeamAbbr'].map(self.map_abbr)
 return X
@transformation_check
def _encode_player_height(self, X):
def string_to_inches(x):
feet, inch = x.split('-')
 return int(inch)+ 12 * int(feet)
 X['PlayerHeight'] = X['PlayerHeight'].apply(string_to_inches)
 return X
@transformation_check
def _process_time_variables(self, X):
for col in ['TimeHandoff', 'TimeSnap', 'PlayerBirthDate']:
X[col] = pd.to_datetime(X[col], utc=True, infer_datetime_format=True)
 X['TimeUntilHandoff'] = X['TimeSnap'] - X['TimeHandoff']
X['TimeUntilHandoff'] = X['TimeUntilHandoff'].dt.total_seconds()
X['PlayerAge'] = X['TimeSnap'] - X['PlayerBirthDate']
X['PlayerAge'] = X['PlayerAge'].dt.total_seconds() / 31556952
X['GameClock'] = 360 * X['GameClock'].str[:2].astype(int)\
+ 60 * X['GameClock'].str[3:5].astype(int)\
+ X['GameClock'].str[6:8].astype(int)
 X['SecondsRemaining'] = 0
X.loc[X['Quarter'] == 1, 'SecondsRemaining'] = 45 * 60 + X['GameClock']
X.loc[X['Quarter'] == 2, 'SecondsRemaining'] = 30 * 60 + X['GameClock']
X.loc[X['Quarter'] == 3, 'SecondsRemaining'] = 15 * 60 + X['GameClock']
X.loc[X['Quarter'] == 4, 'SecondsRemaining'] = 0 * 60 + X['GameClock']
X.loc[X['Quarter'] == 5, 'SecondsRemaining'] = 0 * 60 + X['GameClock']
X.drop(columns=['TimeHandoff', 'TimeSnap', 'PlayerBirthDate'], inplace=True)
 return X
@transformation_check
def _fix_wind_variables(self, X):
def average_ranges(x):
x = str(x)
 if '-' in x:
low, high = x.split('-')
 return str(( int(high)+ int(low)) / 2)
 elif ' gusts up to ' in x:
low, high = x.split(' gusts up to ')
 return str(( int(high)+ int(low)) / 2)
 else:
return x
def coerce_to_int(x):
try:
x = int(x)
 except:
x = np.nan
return x
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
X['WindSpeed'] = X['WindSpeed'].str.lower().str.replace('mph', '')
 X['WindSpeed'] = X['WindSpeed'].str.strip()
X['WindSpeed'] = X['WindSpeed'].apply(average_ranges)
 X['WindSpeed'] = X['WindSpeed'].apply(coerce_to_int)
 X['GameWeather'] = X['GameWeather'].apply(map_weather)
 acceptable_directions = [
'NE', 'SW', 'S', 'NW', 'WSW', 'SE', 'W', 'N', 'NNE', 'WNW', 'SSW',
'NNW', 'SSE', 'E', 'ENE', 'ESE'
]
X['WindDirection'] = X['WindDirection'].str.upper()
X['WindDirection'] = X['WindDirection'].str.replace('FROM ', '' ).str.replace('-', '')
 X.loc[~X['WindDirection'].isin(acceptable_directions), 'WindDirection'] = np.nan
return X
@transformation_check
def _fix_stadium_type_and_turf(self, X):
stadium_type_map = {
'Outdoor': 'Outdoor',
'Outdoors': 'Outdoor',
'Indoors': 'Indoor',
'Dome': 'Indoor',
'Indoor': 'Indoor',
'Retractable Roof': 'Retr Open',
'Open': 'Retr Open',
'Retr.Roof-Closed': 'Retr Closed',
'Retr.Roof - Closed': 'Retr Closed',
'Domed, closed': 'Retr Closed',
'Domed, open': 'Retr Open',
'Closed Dome': 'Retr Closed',
'Dome, closed': 'Retr Closed',
'Domed': 'Indoor',
'Oudoor': 'Outdoor',
'Indoor, Roof Closed': 'Retr Closed',
'Retr.Roof Closed': 'Retr Closed',
'Retr.Roof-Open': 'Retr Open',
'Bowl': 'Outdoor',
'Outddors': 'Outdoor',
'Heinz Field': 'Outdoor',
'Outdoor Retr Roof-Open': 'Retr Open',
'Retr.Roof - Open': 'Retr Open',
'Indoor, Open Roof': 'Retr Open',
'Ourdoor': 'Outdoor',
'Outdor': 'Outdoor',
'Outside': 'Outdoor',
'Cloudy': 'Outdoor',
'Domed, Open': 'Retr Open'
}
X['StadiumType'] = X['StadiumType'].map(stadium_type_map)
 grass_labels = ['grass', 'natural grass', 'natural', 'naturall grass']
X['Turf'] = np.where(X['Turf'].str.lower().isin(grass_labels), 1, 0)
 def get_city(x):
x = x.replace('e.', 'east' ).replace('.', ',')
 return x.split(',')[0].strip().lower()
X['Location'] = X['Location'].apply(get_city)
 return X
@transformation_check
def _map_college_and_pos(self, X):
for replacement in replacements:
X['PlayerCollegeName'] = X['PlayerCollegeName']\ 
.replace(replacement[0], replacement[1])
 X['PositionGroup'] = X['Position'].map(position_groups)
 X['InPossesion'] = X['PositionGroup'].map(posession_positions)
 return X
@transformation_check
def _encode_personnel(self, X):
def count_positions(x, offensive):
offensive_counts = {'DB': 0, 'DL': 0, 'LB': 0, 'OL': 0, 'QB': 0, 'RB': 0, 'TE': 0, 'WR': 0}
defensive_counts = {'DB': 0, 'DL': 0, 'LB': 0, 'OL': 0}
if offensive:
val_counts=offensive_counts
else:
val_counts=defensive_counts
if isinstance(x, str):
for position_val in x.split(','):
val, pos = position_val.strip().split(' ')
 if pos in val_counts:
val_counts[pos] += int(val)
 return val_counts
X['OffensePersonnel'] = X['OffensePersonnel'] \ 
.apply(count_positions, offensive=True)
 off_personnel_df = pd.DataFrame().from_records(X['OffensePersonnel'].values)
 off_personnel_df.index = X.index
off_personnel_df.columns = ['NOffensive' + col for col in off_personnel_df.columns]
X['DefensePersonnel'] = X['DefensePersonnel'] \ 
.apply(count_positions, offensive=False)
 def_personnel_df = pd.DataFrame().from_records(X['DefensePersonnel'].values)
 def_personnel_df.index = X.index
def_personnel_df.columns = ['NDefensive' + col for col in def_personnel_df.columns]
X.drop(columns=['OffensePersonnel', 'DefensePersonnel'], inplace=True)
 X = pd.concat([X, off_personnel_df, def_personnel_df], axis=1)
 return X
@transformation_check
def _normalise_positional_data(self, X):
X['IsLeftDirection'] = X['PlayDirection'] == 'left'
X['PossesionInOwnHalf'] = X['PossessionTeam'] == X['FieldPosition']
possession_in_own_half = X.groupby(['GameId', 'PlayId'])['PossesionInOwnHalf'].max().reset_index()
X = X.drop(columns='PossesionInOwnHalf')\ 
.merge(possession_in_own_half, on=['GameId', 'PlayId'])
 X['DistToEndZone'] = X['YardLine']
X.loc[X['PossesionInOwnHalf'], 'DistToEndZone'] = 50 +(50 - X['YardLine'])
 X.loc[X['YardLine'] == 50, 'DistToEndZone'] = 50
X['YardLineStd'] = 100 - X['DistToEndZone']
X['XStd'] = X['X']
X.loc[X['IsLeftDirection'], 'XStd'] = 120 - X.loc[X['IsLeftDirection'], 'X']
X['YStd'] = X['Y']
X.loc[X['IsLeftDirection'], 'YStd'] = 160 / 3 - X.loc[X['IsLeftDirection'], 'Y']
X['PlayerDistToEndZone'] = 100 -(X['XStd'] - 10)
 X['DirRad'] = np.mod(90 - X['Dir'], 360)* math.pi / 180.0
X['DirStd'] = X['DirRad']
X.loc[X['IsLeftDirection'], 'DirStd'] = \
np.mod(np.pi + X.loc[X['IsLeftDirection'], 'DirRad'], 2*np.pi)
 X['OrientationRad'] = np.mod(X['Orientation'], 360)* math.pi / 180.0
X.loc[X['Season'] >= 2018, 'OrientationRad'] \
= np.mod(X.loc[X['Season'] >= 2018, 'Orientation'] - 90, 360)* math.pi / 180.0
X['OrientationStd'] = X['OrientationRad']
X.loc[X['IsLeftDirection'], 'OrientationStd'] = \
np.mod(np.pi + X.loc[X['IsLeftDirection'], 'OrientationRad'], 2 * np.pi)
 X.drop(columns=['OrientationRad'], inplace=True)
 X['IsLeftDirection'] =(X['IsLeftDirection'] ).astype(int)
 X['IsRusher'] =(X['NflId'] == X['NflIdRusher'] ).astype(int)
 return X
@transformation_check
def _calc_voronoi(self, X):
max_voronoi = 120 * 53.3
X['VoronoiArea'] = 0
X['VoronoiAreaNoOffence'] = 0
for play_id in X['PlayId'].unique() :
play = X.loc[X['PlayId'] == play_id].copy()
x_cut_off = play['XStd'].min() - 5
no_offence_play = play[play['IsRusher'].astype(bool)|
~play['InPossesion'].astype(bool)]
def mirror_boundary(xy):
xy = xy.values
n_points = xy.shape[0]
xy1 = xy.copy()
xy1[:,1] = -xy[:,1]
xy2 = xy.copy()
xy2[:,1] = 320/3 - xy[:,1]
xy3 = xy.copy()
xy3[:,0] = 2 * x_cut_off - xy[:,0]
xy4 = xy.copy()
xy4[:,0] = 220 - xy[:,0]
return np.concatenate(( xy, xy1, xy2, xy3, xy4), axis=0), n_points
xy, n = mirror_boundary(play[['XStd', 'YStd']])
 vor = Voronoi(xy)
 no_off_xy, _ = mirror_boundary(no_offence_play[['XStd', 'YStd']])
 no_off_vor = Voronoi(no_off_xy)
 areas = np.zeros([play.shape[0], ])
 for i in range(n):
player_point = vor.point_region[i]
vertices = vor.vertices[vor.regions[player_point]]
areas[i] = shoelace(vertices[:, 0], vertices[:, 1])
 rusher_index = np.argmax(no_offence_play['IsRusher'].values)
 rusher_index_in_df = no_offence_play.index.values[rusher_index]
rusher_region = no_off_vor.point_region[rusher_index]
rusher_vertex_index = no_off_vor.regions[rusher_region]
rusher_vertices = no_off_vor.vertices[rusher_vertex_index]
rusher_area = shoelace(rusher_vertices[:, 0], rusher_vertices[:, 1])
 X.loc[play.index, 'VoronoiArea'] = areas
X.loc[rusher_index_in_df, 'VoronoiAreaNoOffence'] = min(rusher_area, max_voronoi)
 X.loc[X['VoronoiArea'] > max_voronoi, 'VoronoiArea'] = max_voronoi
return X
@transformation_check
def _misc_engineering(self, X):
X['ScoreDiff'] = X['HomeScoreBeforePlay'] - X['VisitorScoreBeforePlay']
X['IsAwayTeam'] =(X['Team'] == 'away' ).astype(int)
 X['IsInAwayEnd'] =(X['FieldPosition'] == X['VisitorTeamAbbr'] ).astype(int)
 X['HomePossesion'] =(X['PossessionTeam'] == X['HomeTeamAbbr'] ).astype(int)
 X['AwayInPosession'] =(X['InPossesion'] == X['IsAwayTeam'] ).astype(int)
 X['SX'] = X['S'] * np.cos(X['DirStd'])
 X['SY'] = X['S'] * np.sin(X['DirStd'])
 X['AX'] = X['A'] * np.cos(X['DirStd'])
 X['AY'] = X['A'] * np.sin(X['DirStd'])
 X['OrientationCos'] = X['OrientationStd'] * np.cos(X['DirStd'])
 X['OrientationSin'] = X['OrientationStd'] * np.sin(X['DirStd'])
 X['XToScrimmage'] = X['YardLineStd'] -(X['XStd'] - 10)
 rusher_pos = X.loc[
X['IsRusher'] == 1,
['XStd', 'YStd', 'DirStd', 'S', 'SX', 'SY', 'A', 'AX', 'AY',
'Position', 'GameId', 'PlayId', 'PlayerCollegeName', 'Dis',
'XToScrimmage', 'PlayerDistToEndZone', 'OrientationStd', 'OrientationCos',
'OrientationSin', 'VoronoiArea', 'VoronoiAreaNoOffence']
]
X = X.merge(rusher_pos, on=['GameId', 'PlayId'], suffixes=['', 'Rusher'])
 if 'PositionRusher' not in self.one_hot_vars:
self.one_hot_vars += ['PositionRusher']
X['XFromRusher'] = abs(X['XStd'] - X['XStdRusher'])
 X['YFromRusher'] = abs(X['YStd'] - X['YStdRusher'])
 X['DistFromRusher'] = np.sqrt(np.square(X['XFromRusher'])
 + np.square(X['YFromRusher']))
X['TimeToRusher'] = X['DistFromRusher'] / X['S']
X.loc[np.isinf(X['TimeToRusher']), 'TimeToRusher'] = 1000
X['Force'] = X['PlayerWeight'] * X['A']
X['Momentum'] = X['PlayerWeight'] * X['S']
cols_to_drop = ['NflId', 'NflIdRusher', 'Team', 'PlayDirection',
'FieldPosition', 'PossessionTeam', 'OrientationCos',
'OrientationSin', 'VoronoiAreaNoOffence',
'PlayerDistToEndZone']
X.drop(columns=cols_to_drop, inplace=True)
 return X
@transformation_check
def _apply_one_hot_encoder(self, X):
col_names = []
print('\tdropping columns: {} for having cardinality > {}' 
.format(' '.join(self.oh_cols_to_drop), self.max_cardinality))
X.drop(columns=self.oh_cols_to_drop, inplace=True)
 self.player_cols = [col for col in self.player_cols
if col not in self.oh_cols_to_drop]
print('\tone hot encoding columns: {}'.format(' '.join(self.one_hot_vars)))
 X_1h = self.one_hot_encoder.transform(X[self.one_hot_vars].fillna('unknown'))
for i, col in enumerate(self.one_hot_vars):
new_var_names = \
[col + '_' + val for val in self.one_hot_encoder.categories_[i]]
col_names += new_var_names
if col in self.player_cols:
self.player_cols.remove(col)
 self.player_cols += new_var_names
X_1h = pd.DataFrame(data=X_1h, index=X.index, columns=col_names)
 X = pd.concat([X.drop(columns=self.one_hot_vars), X_1h], axis=1)
 return X
@transformation_check
def _flatten_player_vars(self, X):
college_mask = ['CollegeConference' in col for col in X.columns]
college_cols = X.columns[college_mask]
college_agg = X.groupby(['GameId', 'PlayId'])[college_cols].sum()
X.drop(columns=college_cols, inplace=True)
 mechanics_cols = [
'X', 'XStd', 'Y', 'YStd', 'A', 'S', 'Dir', 'DirRad', 'DirStd',
'PlayerHeight', 'PlayerWeight', 'PlayerAge', 'Force',
'Momentum', 'OrientationStd', 'Dis', 'XFromRusher', 'XToScrimmage',
'YFromRusher', 'DistFromRusher', 'AX', 'AY', 'SX', 'SY', 'TimeToRusher'
]
mech_agg = X.groupby(['GameId', 'PlayId', 'InPossesion'])[mechanics_cols]\ 
.agg(['mean', 'std'])
 mech_agg.columns = ['_'.join(col ).strip() for col in mech_agg.columns.values]
mech_agg.reset_index(inplace=True)
 mech_agg.set_index(['GameId', 'PlayId'], inplace=True)
 mech_agg['InPossesion'] = mech_agg['InPossesion'].map({1: 'off', 0: 'def'})
 mech_agg = mech_agg.pivot(columns='InPossesion')
 mech_agg.columns = ['_'.join(col ).strip() for col in mech_agg.columns.values]
mech_agg.fillna(0, inplace=True)
 college_like_cols = [
'PlayerCollegeName', 'JerseyNumber'
]
coll_mean = X.groupby(['GameId', 'PlayId', 'InPossesion'])[college_like_cols]\ 
.agg(['mean'])
 coll_mean.columns = ['_'.join(col ).strip() for col in coll_mean.columns.values]
coll_mean.reset_index(inplace=True)
 coll_mean.set_index(['GameId', 'PlayId'], inplace=True)
 coll_mean['InPossesion'] = coll_mean['InPossesion'].map({1: 'off', 0: 'def'})
 coll_mean = coll_mean.pivot(columns='InPossesion')
 coll_mean.columns = ['_'.join(col ).strip() for col in coll_mean.columns.values]
coll_mean.fillna(0, inplace=True)
 X.drop(columns=college_like_cols, inplace=True)
 mechanics_cols += ['VoronoiArea']
mech_agg_pos = X.groupby(['GameId', 'PlayId', 'PositionGroup', 'InPossesion'])[mechanics_cols]\ 
.agg(['mean', 'std'])
 mech_agg_pos.columns = ['_'.join(col ).strip() for col in mech_agg_pos.columns.values]
mech_agg_pos.reset_index(inplace=True)
 mech_agg_pos.set_index(['GameId', 'PlayId'], inplace=True)
 mech_agg_pos['PositionPossession'] = mech_agg_pos['InPossesion'].map({1: 'off', 0: 'def'})\
+ '_' + mech_agg_pos['PositionGroup']
mech_agg_pos.drop(columns=['InPossesion', 'PositionGroup'], inplace=True)
 mech_agg_pos = mech_agg_pos.pivot(columns='PositionPossession')
 mech_agg_pos.columns = ['_'.join(col ).strip() for col in mech_agg_pos.columns.values]
mech_agg_pos.fillna(0, inplace=True)
 X.drop(columns=mechanics_cols, inplace=True)
 away_ind = X.groupby(['GameId', 'PlayId'])['AwayInPosession', 'ScoreDiff'].max()
away_ind['PosessionTeamLeading'] = 0
mask =(( away_ind['AwayInPosession'] == 1)&(away_ind['ScoreDiff'] < 0)) | \ 
(( away_ind['AwayInPosession'] == 0)&(away_ind['ScoreDiff'] > 0))
away_ind.loc[mask, 'PosessionTeamLeading'] = 1
away_ind['PosessionTeamLead'] = away_ind['ScoreDiff']
away_ind.loc[away_ind['AwayInPosession'] == 1, 'PosessionTeamLead'] = \
-away_ind.loc[away_ind['AwayInPosession'] == 1, 'PosessionTeamLead']
away_ind['GameWithinConvTouchdown'] = away_ind['ScoreDiff'].abs() <= 8
ignore_cols = ['IsAwayTeam', 'InPossesion', 'AwayInPosession',
'PositionGroup', 'ScoreDiff', 'Orientation']
X.drop(columns=ignore_cols, inplace=True)
 self.player_cols = [col for col in self.player_cols
if col not in college_cols
and col not in mechanics_cols
and col not in college_like_cols
and col not in ignore_cols]
n_player_cols = len(self.player_cols)
 if self.player_cols:
player_data = X[self.player_cols].values.reshape(-1, n_player_cols * 22)
 new_col_names = [col + '_' + str(player_num)
 for player_num in range(22)
 for col in self.player_cols]
player_data = pd.DataFrame(
 data=player_data,
columns=new_col_names 
)
 player_data = player_data.infer_objects()
X.drop(columns=self.player_cols, inplace=True)
 X = X.drop_duplicates().reset_index(drop=True)
 if self.player_cols:
X = pd.concat([X, player_data], axis=1)
 X.set_index(['GameId', 'PlayId'], inplace=True)
 X = X.merge(college_agg, how='left', left_index=True, right_index=True, suffixes=['', 'College'])
 X = X.merge(mech_agg, how='left', left_index=True, right_index=True, suffixes=['', 'TeamMech'])
 X = X.merge(coll_mean, how='left', left_index=True, right_index=True, suffixes=['', 'PlayerCollege'])
 X = X.merge(mech_agg_pos, how='left', left_index=True, right_index=True, suffixes=['', 'TeamPosMech'])
 X = X.merge(away_ind, how='left', left_index=True, right_index=True, suffixes=['', 'GameInds'])
 return X
@transformation_check
def _previous_play_data(self, X):
X[['YardLineLastRush', 'GameClockLastRush', 'DistanceLastRush']] = \
X.groupby(['GameId', 'PlayId'])['YardLineStd', 'GameClock', 'Distance'].shift()
X['TimeSinceLastRush'] = X['GameClock'] - X['GameClockLastRush']
X.drop(columns=['YardLineLastRush', 'GameClockLastRush', 'DistanceLastRush'], inplace=True)
 return X
def _get_player_cols(self, X):
max_vals = X.groupby(['GameId', 'PlayId'] ).nunique().max()
self.player_cols = max_vals[max_vals > 1].index.tolist()
self.team_cols = max_vals[max_vals == 2].index.tolist()<train_model>
df_total['QualperSF'] = df_total['HouseSF'] /(1/df_total['OverallQual'])
 df_total['BathroomGrade'] = df_total["TotalBathroom"] * df_total['OverallQual']
df_total['PorchGrade'] = df_total['ExterQual'] * df_total["AllPorchSF"]
df_total['GarageSFQual'] = df_total['GarageCars'] * df_total['GarageQual'] * df_total['GarageFinish']
class DropColinear(TransformerMixin):
def __init__(self, max_corr=1):
self.max_corr = max_corr
self.all_corellated_cols = []
def fit(self, X, y=None):
corr = X.corr()
corr = pd.DataFrame(np.triu(corr), columns=corr.columns, index=corr.index)
 for col in corr.index:
correlated_cols = corr.columns[corr[col].abs() >= self.max_corr].tolist()
correlated_cols = [c_col for c_col in correlated_cols if c_col != col]
if correlated_cols:
self.all_corellated_cols += correlated_cols
self.all_corellated_cols = np.unique(self.all_corellated_cols)
 return self
def transform(self, X):
print('Dropping following columns for having a correlation of over {} with '
'another variable:
{}'.format(self.max_corr, ', '.join(self.all_corellated_cols)))
 X.drop(columns=self.all_corellated_cols, inplace=True)
 return X<categorify>
df_total['HouseSFSq'] = df_total['HouseSF']**2
df_total['HouseSFCb'] = df_total['HouseSF']**3
processor = Preprocessor(
 one_hot_vars=VARS_TO_ONE_HOT,
max_cardinality=MAX_CARDINALITY,
cols_to_drop=COLS_TO_DROP 
)
 print(train.shape)
 train = processor.initial_cleaning(train)
 train = processor.fit_transform(train )<normalization>
df_total['OverQualSq'] = df_total['OverallQual']**2
df_total['OverCondSq'] = df_total['OverallCond']**2
dropper = DropColinear(max_corr=1)
 train = dropper.fit_transform(train)
 print(train.shape )<normalization>
df_total['HasPool'] = df_total['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 df_total['Has2ndFloor'] = df_total['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 df_total['HasGarage'] = df_total['GarageCars'].apply(lambda x: 1 if x > 0 else 0)
 df_total['HasBsmt'] = df_total['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 df_total['HasFireplace'] = df_total['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 df_total["HasShed"] =(df_total["MiscFeature"] == "Shed" ).apply(lambda x: 1 if x > 0 else 0 )
class FeatureChecker(TransformerMixin):
def fit(self, X):
self.std = X.std(axis=0)
 self.mean = X.mean(axis=0)
 def transform(self, X):
new_mean = X.mean(axis=0)
 abs_diff = np.absolute(new_mean - self.mean)
 if any(abs_diff > self.std):
cols = np.arange(X.shape[1])
 print('Following columns are over 1 std dev out form training: {}' 
.format(cols[abs_diff > self.std]))
return X<sort_values>
neighborhood_map = {"MeadowV" : 1, "IDOTRR" : 1, "BrDale" : 1, "OldTown" : 2, "Edwards" : 2, "BrkSide" : 2, "Sawyer" : 2,
"Blueste" : 2, "SWISU" : 2, "NAmes" : 2, "NPkVill" : 2, "Mitchel" : 2, "SawyerW" : 3, "Gilbert" : 3,
"NWAmes" : 3, "Blmngtn" : 3, "CollgCr" : 3, "ClearCr" : 3, "Crawfor" : 3, "Veenker" : 4,
"Somerst" : 4, "Timber" : 4, "StoneBr" : 5, "NoRidge" : 5, "NridgHt" : 5,}
df_total["NeighborhoodCluster"] = df_total["Neighborhood"].map(neighborhood_map )
train.isnull().sum().sort_values(ascending=False)[:5]<categorify>
df_total["HighSeason"] = df_total["MoSold"].replace(
 {'1': 0, '2': 0, '3': 0, '4': 1, '5': 1, '6': 1, '7': 1, '8': 0, '9': 0, '10': 0, '11': 0, '12': 0} )
yards = train.pop('Yards')
 season = train.pop('Season')
 season_weights = season.map({2017:.8, 2018: 1.2, 2019: 1.2} ).fillna(1 ).values<feature_engineering>
df_total["Remodeled"] =(df_total["YearRemodAdd"] != df_total["YearBuilt"] ).apply(lambda x: 1 if x > 0 else 0)
 df_total["RecentRemodel"] =(df_total["YearRemodAdd"] == df_total["YrSold"] ).apply(lambda x: 1 if x > 0 else 0 )
fill_val = train.mean()<prepare_x_and_y>
df_total["Unusual_sale"] = df_total.SaleCondition.replace(
 {'Abnorml': 1, 'Alloca': 1, 'AdjLand': 1, 'Family': 1, 'Normal': 0, 'Partial': 0})
 df_total["BoughtOffPlan"] = df_total.SaleCondition.replace(
 {"Abnorml" : 0, "Alloca" : 0, "AdjLand" : 0, "Family" : 0, "Normal" : 0, "Partial" : 1} )
X = train.copy()
y = np.zeros(( yards.shape[0], 199))
for idx, target in enumerate(list(yards)) :
y[idx][99 + target] = 1<normalization>
df_total.drop(['GarageArea', 'GarageCond', 'TotRmsAbvGrd', 'AboveSF', '2ndFlrSF', 'GarageYrBlt', 'BsmtFinSF2'],
axis = 1, inplace = True)
 df_total = df_total.drop(['Utilities', 'Street', 'PoolQC', 'PoolArea'], axis=1 )
col_order = sorted(train.columns)
 X = X[col_order]
scaler = StandardScaler()
X = scaler.fit_transform(X.fillna(fill_val))<train_model>
numerical_features = df_total.select_dtypes(exclude = ["object"] ).columns
numerical_features = numerical_features.drop("SalePrice" )
class CRPSCallback(Callback):
def __init__(self,validation, predict_batch_size=20, include_on_batch=False):
super(CRPSCallback, self ).__init__()
self.validation = validation
self.predict_batch_size = predict_batch_size
self.include_on_batch = include_on_batch
print('validation shape',len(self.validation))
def on_batch_begin(self, batch, logs={}):
pass
def on_train_begin(self, logs={}):
if not('CRPS_score_val' in self.params['metrics']):
self.params['metrics'].append('CRPS_score_val')
 def on_batch_end(self, batch, logs={}):
if(self.include_on_batch):
logs['CRPS_score_val'] = float('-inf')
 def on_epoch_end(self, epoch, logs={}):
logs['CRPS_score_val'] = float('-inf')
 if(self.validation):
X_valid, y_valid = self.validation[0], self.validation[1]
y_pred = self.model.predict(X_valid)
 y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * X_valid.shape[0])
 val_s = np.round(val_s, 6)
 logs['CRPS_score_val'] = val_s<choose_model_class>
print("Pre - dummy number of features : " + str(df_total.shape[1]))
N_TRAIN = X.shape[0]
BATCH_SIZE = 1024
STEPS_PER_EPOCH = N_TRAIN//BATCH_SIZE
lr_schedule = tf.keras.optimizers.schedules.InverseTimeDecay(0.001,
decay_steps=STEPS_PER_EPOCH*100,
decay_rate=1,
staircase=False)
 def get_optimizer() :
return tf.keras.optimizers.Adam(lr_schedule)
 optimizer = get_optimizer()
def get_model(x_tr,y_tr,x_val,y_val,w_tr,dropouts):
inp = Input(shape =(x_tr.shape[1],))
x = Dropout(0.8 )(inp)
 x = Dense(64, input_dim=X.shape[1], activation='relu',
kernel_regularizer=regularizers.l1(0.01))(inp)
 x = BatchNormalization()(x)
 x = Dense(64, activation='relu',
kernel_regularizer=regularizers.l1(0.01))(x)
 x = BatchNormalization()(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inp, out)
 model.compile(
 optimizer=optimizer,
loss='categorical_crossentropy',
metrics=[] 
)
 es = EarlyStopping(
 monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
verbose=1,
patience=10 
)
 mc = ModelCheckpoint(
 'best_model.h5',
monitor='CRPS_score_val',
mode='min',
save_best_only=True,
verbose=1,
save_weights_only=True 
)
 bsz = 1024
steps = x_tr.shape[0] / bsz
history = model.fit(
 x_tr,
y_tr,
callbacks=[CRPSCallback(validation =(x_val,y_val)) ,es,mc],
epochs=100,
batch_size=bsz,
sample_weight=w_tr,
verbose=1 
)
 model.load_weights("best_model.h5")
 y_pred = model.predict(x_val)
 y_valid = y_val
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * x_val.shape[0])
 crps = np.round(val_s, 6)
 return {'loss': crps, 'status': STATUS_OK, 'model': model}, history<split>
df_total = pd.get_dummies(df_total)
 print("New number of features : " + str(df_total.shape[1]))
def get_data(X, y, seed):
x_tr, y_tr, x_val, y_val = train_test_split(X, y, test_size=0.15, random_state=seed)
 return x_tr, y_tr, x_val, y_val<train_on_grid>
df_total = df_total.drop(['SalePrice', "MSZoning_C(all)"], axis = 1)
 train = df_total.iloc[:df_train_ends,:]
test = df_total.iloc[df_train_ends:,:]
print(train.shape )
losses = []
models = []
crps_csv = []
s_time = time.time()
for k in range(2):
kfold = KFold(5, random_state=SEED + k, shuffle=True)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(yards)) :
print("-----------")
 print("-----------")
 tr_x, tr_y = X[tr_inds],y[tr_inds]
val_x, val_y = X[val_inds],y[val_inds]
w_tr = season_weights[tr_inds]
results, history = get_model(tr_x,tr_y,val_x,val_y,w_tr,dropouts=[.4,.5,.5,.5])
 models.append(results['model'])
 print("the %d fold crps is %f"%(( k_fold+1), results['loss']))
print("mean crps is %f"%np.mean(crps_csv))
crps_csv.append(results['loss'])
 print("mean crps is %f"%np.mean(crps_csv))
def predict(x_te):
model_num = len(models)
 for k,m in enumerate(models):
if k==0:
y_pred = m.predict(x_te, batch_size=1024)
 else:
y_pred += m.predict(x_te, batch_size=1024)
 y_pred = y_pred / model_num
return y_pred<compute_test_metric>
X_train, X_test, y_train, y_test = train_test_split(train, y, test_size = 0.3, random_state = 0)
 print("X_train : " + str(X_train.shape))
print("X_test : " + str(X_test.shape))
print("y_train : " + str(y_train.shape))
print("y_test : " + str(y_test.shape))
print("mean crps is\t%f"%np.mean(crps_csv))
print("std crps is\t%f"%np.std(crps_csv))<predict_on_test>
rbstSc = RobustScaler()
X_train.loc[:, numerical_features] = rbstSc.fit_transform(X_train.loc[:, numerical_features])
 X_test.loc[:, numerical_features] = rbstSc.transform(X_test.loc[:, numerical_features] )
names = dict(zip(range(199), ['Yards%d' % i for i in range(-99, 100)]))
env = nflrush.make_env()
for i,(df_test, sample_pred)in enumerate(env.iter_test()):
test = processor.initial_cleaning(df_test)
 test = processor.transform(test)
 for col in col_order:
if col not in test.columns:
test[col] = 0
test = test[col_order]
scaled_test = scaler.transform(test.fillna(fill_val))
y_pred = predict(scaled_test)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1 ).tolist() [0]
preds_df = pd.DataFrame(data=[y_pred], columns=sample_pred.columns)
 if i == 0:
all_preds = preds_df
all_test_rows = test
else:
all_preds = pd.concat([all_preds, preds_df], ignore_index=True, sort=False)
 all_test_rows = pd.concat([all_test_rows, test], ignore_index=True, sort=False)
 env.predict(preds_df)
 all_test_rows.to_csv('X_test.csv')
 env.write_submission_file()<set_options>
train.loc[:, numerical_features] = rbstSc.fit_transform(train.loc[:, numerical_features])
 test.loc[:, numerical_features] = rbstSc.transform(test.loc[:, numerical_features] )
TRAIN_ABLE_FALSE=True
if TRAIN_ABLE_FALSE:
os.environ['CUDA_VISIBLE_DEVICES'] = "1"
TRAIN_OFFLINE = False
pd.set_option('display.max_columns', 50)
 pd.set_option('display.max_rows', 30 )<load_from_csv>
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
train = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'})
 outcomes = train[['GameId','PlayId','Yards']].drop_duplicates() 
<categorify>
y_rf_train = randomforest.predict(X_train)
 y_rf_test = randomforest.predict(X_test)
 print("Random Forest RMSE on Training set :", rmsle(y_train, y_rf_train ).mean())
 print("Random Forest RMSE on Validation set :", rmsle(y_test, y_rf_test ).mean() )
def strtoseconds(txt):
txt = txt.split(':')
 ans = int(txt[0])*60 + int(txt[1])+ int(txt[2])/60
return ans
def strtofloat(x):
try:
return float(x)
 except:
return -1
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
def OffensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0, 'QB' : 0, 'RB' : 0, 'TE' : 0, 'WR' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def DefensePersonnelSplit(x):
dic = {'DB' : 0, 'DL' : 0, 'LB' : 0, 'OL' : 0}
for xx in x.split(","):
xxs = xx.split(" ")
 dic[xxs[-1]] = int(xxs[-2])
 return dic
def orientation_to_cat(x):
x = np.clip(x, 0, 360 - 1)
 try:
return str(int(x/15))
except:
return "nan"
def preprocess(train):
train['GameClock_sec'] = train['GameClock'].apply(strtoseconds)
 train["GameClock_minute"] = train["GameClock"].apply(lambda x : x.split(":")[0] ).astype("object")
 train['PlayerHeight_dense'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 train["PlayerAge_ob"] = train['PlayerAge'].astype(np.int ).astype("object")
 train['WindSpeed_ob'] = train['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 train['WindSpeed_ob'] = train['WindSpeed_ob'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 train['WindSpeed_ob'] = train['WindSpeed_ob'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 train['WindSpeed_dense'] = train['WindSpeed_ob'].apply(strtofloat)
 train['GameWeather_process'] = train['GameWeather'].str.lower()
train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: "indoor" if not pd.isna(x)and "indoor" in x else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train['GameWeather_process'] = train['GameWeather_process'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 train['GameWeather_dense'] = train['GameWeather_process'].apply(map_weather)
 train['IsRusher'] =(train['NflId'] == train['NflIdRusher'])
 train['IsRusher_ob'] =(train['NflId'] == train['NflIdRusher'] ).astype("object")
 temp = train[train["IsRusher"]][["Team", "PlayId"]].rename(columns={"Team":"RusherTeam"})
 train = train.merge(temp, on = "PlayId")
 train["IsRusherTeam"] = train["Team"] == train["RusherTeam"]
train["Quarter_ob"] = train["Quarter"].astype("object")
 train["Down_ob"] = train["Down"].astype("object")
 train["JerseyNumber_ob"] = train["JerseyNumber"].astype("object")
 train["YardLine_ob"] = train["YardLine"].astype("object")
 train["Orientation_ob"] = train["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Dir_ob"] = train["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 train["Orientation_sin"] = train["Orientation"].apply(lambda x : np.sin(2 * np.pi * x/360.0))
train["Orientation_cos"] = train["Orientation"].apply(lambda x : np.cos(2 * np.pi * x/360.0))
train["Dir_sin"] = train["Dir"].apply(lambda x : np.sin(2 * np.pi * x/360))
train["Dir_cos"] = train["Dir"].apply(lambda x : np.cos(2 * np.pi * x/360))
train["diffScoreBeforePlay"] = train["HomeScoreBeforePlay"] - train["VisitorScoreBeforePlay"]
train["diffScoreBeforePlay_binary_ob"] =(train["HomeScoreBeforePlay"] > train["VisitorScoreBeforePlay"] ).astype("object")
 Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural', 'UBU Sports Speed S5-M':'Artificial', 'Artificial':'Artificial', 'DD GrassMaster':'Artificial', 'Natural Grass':'Natural', 'UBU Speed Series-S5-M':'Artificial', 'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural', 'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural', 'Field turf':'Artificial', 'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial', 'natural grass':'Natural'}
train['Turf'] = train['Turf'].map(Turf)
 temp = train["OffensePersonnel"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(OffensePersonnelSplit(x)))
 temp.columns = ["Offense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 22)]
train = train.merge(temp, on = "PlayId")
 temp = train["DefensePersonnel"].iloc[np.arange(0, len(train), 22)].apply(lambda x : pd.Series(DefensePersonnelSplit(x)))
 temp.columns = ["Defense" + c for c in temp.columns]
temp["PlayId"] = train["PlayId"].iloc[np.arange(0, len(train), 22)]
train = train.merge(temp, on = "PlayId")
 train = train.sort_values(by = ['X'] ).sort_values(by = ['Dis'] ).sort_values(by=['PlayId', 'IsRusherTeam', 'IsRusher'] ).reset_index(drop = True)
 return train<prepare_x_and_y>
randomforest = RandomForestRegressor(n_estimators=1000, random_state=42)
 randomforest.fit(train, y)
 y_rf = randomforest.predict(train)
 print("Random Forest RMSE on Full Training set :", rmsle(y, y_rf ).mean())
 randforest_submission_price = randomforest.predict(test )
def create_features(df, deploy=False):
def new_X(x_coordinate, play_direction):
if play_direction == 'left':
return 120.0 - x_coordinate
else:
return x_coordinate
def new_line(rush_team, field_position, yardline):
if rush_team == field_position:
return 10.0 + yardline
else:
return 60.0 +(50 - yardline)
 def new_orientation(angle, play_direction):
if play_direction == 'left':
new_angle = 360.0 - angle
if new_angle == 360.0:
new_angle = 0.0
return new_angle
else:
return angle
def euclidean_distance(x1,y1,x2,y2):
return np.sqrt(( x1-x2)**2 +(y1-y2)**2)
 def back_direction(orientation):
if orientation > 180.0:
return 1
else:
return 0
def update_yardline(df):
new_yardline = df[df['NflId'] == df['NflIdRusher']]
new_yardline['YardLine'] = new_yardline[['PossessionTeam','FieldPosition','YardLine']].apply(lambda x: new_line(x[0],x[1],x[2]), axis=1)
 new_yardline = new_yardline[['GameId','PlayId','YardLine']]
return new_yardline
def update_orientation(df, yardline):
df['X'] = df[['X','PlayDirection']].apply(lambda x: new_X(x[0],x[1]), axis=1)
 df['Orientation'] = df[['Orientation','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df['Dir'] = df[['Dir','PlayDirection']].apply(lambda x: new_orientation(x[0],x[1]), axis=1)
 df = df.drop('YardLine', axis=1)
 df = pd.merge(df, yardline, on=['GameId','PlayId'], how='inner')
 return df
def back_features(df):
carriers = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','NflIdRusher','X','Y','Orientation','Dir','YardLine']]
carriers['back_from_scrimmage'] = carriers['YardLine'] - carriers['X']
carriers['back_oriented_down_field'] = carriers['Orientation'].apply(lambda x: back_direction(x))
carriers['back_moving_down_field'] = carriers['Dir'].apply(lambda x: back_direction(x))
carriers = carriers.rename(columns={'X':'back_X',
'Y':'back_Y'})
 carriers = carriers[['GameId','PlayId','NflIdRusher','back_X','back_Y','back_from_scrimmage','back_oriented_down_field','back_moving_down_field']]
return carriers
def features_relative_to_back(df, carriers):
player_distance = df[['GameId','PlayId','NflId','X','Y']]
player_distance = pd.merge(player_distance, carriers, on=['GameId','PlayId'], how='inner')
 player_distance = player_distance[player_distance['NflId'] != player_distance['NflIdRusher']]
player_distance['dist_to_back'] = player_distance[['X','Y','back_X','back_Y']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 player_distance = player_distance.groupby(['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field'])\ 
.agg({'dist_to_back':['min','max','mean','std']})\ 
.reset_index()
player_distance.columns = ['GameId','PlayId','back_from_scrimmage','back_oriented_down_field','back_moving_down_field',
'min_dist','max_dist','mean_dist','std_dist']
return player_distance
def defense_features(df):
rusher = df[df['NflId'] == df['NflIdRusher']][['GameId','PlayId','Team','X','Y']]
rusher.columns = ['GameId','PlayId','RusherTeam','RusherX','RusherY']
defense = pd.merge(df,rusher,on=['GameId','PlayId'],how='inner')
 defense = defense[defense['Team'] != defense['RusherTeam']][['GameId','PlayId','X','Y','RusherX','RusherY']]
defense['def_dist_to_back'] = defense[['X','Y','RusherX','RusherY']].apply(lambda x: euclidean_distance(x[0],x[1],x[2],x[3]), axis=1)
 defense = defense.groupby(['GameId','PlayId'])\ 
.agg({'def_dist_to_back':['min','max','mean','std']})\ 
.reset_index()
defense.columns = ['GameId','PlayId','def_min_dist','def_max_dist','def_mean_dist','def_std_dist']
return defense
def static_features(df):
add_new_feas = []
df['PlayerHeight_dense'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
add_new_feas.append('PlayerHeight_dense')
 df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df['PlayerBirthDate'] =df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 add_new_feas.append('PlayerAge')
 df['WindSpeed_ob'] = df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 df['WindSpeed_ob'] = df['WindSpeed_ob'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 df['WindSpeed_ob'] = df['WindSpeed_ob'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 df['WindSpeed_dense'] = df['WindSpeed_ob'].apply(strtofloat)
 add_new_feas.append('WindSpeed_dense')
 df['GameWeather_process'] = df['GameWeather'].str.lower()
df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: "indoor" if not pd.isna(x)and "indoor" in x else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 df['GameWeather_process'] = df['GameWeather_process'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 df['GameWeather_dense'] = df['GameWeather_process'].apply(map_weather)
 add_new_feas.append('GameWeather_dense')
 df["Orientation_ob"] = df["Orientation"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Dir_ob"] = df["Dir"].apply(lambda x : orientation_to_cat(x)).astype("object")
 df["Orientation_sin"] = df["Orientation"].apply(lambda x : np.sin(2 * np.pi * x/360.0))
df["Orientation_cos"] = df["Orientation"].apply(lambda x : np.cos(2 * np.pi * x/360.0))
add_new_feas.append("Orientation_sin")
 add_new_feas.append("Orientation_cos")
 df["Dir_sin"] = df["Dir"].apply(lambda x : np.sin(2 * np.pi * x/360.0))
df["Dir_cos"] = df["Dir"].apply(lambda x : np.cos(2 * np.pi * x/360.0))
add_new_feas.append("Dir_sin")
 add_new_feas.append("Dir_cos")
 df["diffScoreBeforePlay"] = df["HomeScoreBeforePlay"] - df["VisitorScoreBeforePlay"]
add_new_feas.append("diffScoreBeforePlay")
 static_features = df[df['NflId'] == df['NflIdRusher']][add_new_feas+['GameId','PlayId','X','Y','S','A','Dis','Orientation','Dir',
'YardLine','Quarter','Down','Distance','DefendersInTheBox']].drop_duplicates()
static_features['DefendersInTheBox'] = static_features['DefendersInTheBox'].fillna(np.mean(static_features['DefendersInTheBox']))
static_features.fillna(-999,inplace=True)
 return static_features
def combine_features(relative_to_back, defense, static, deploy=deploy):
df = pd.merge(relative_to_back,defense,on=['GameId','PlayId'],how='inner')
 df = pd.merge(df,static,on=['GameId','PlayId'],how='inner')
 if not deploy:
df = pd.merge(df, outcomes, on=['GameId','PlayId'], how='inner')
 return df
yardline = update_yardline(df)
 df = update_orientation(df, yardline)
 back_feats = back_features(df)
 rel_back = features_relative_to_back(df, back_feats)
 def_feats = defense_features(df)
 static_feats = static_features(df)
 basetable = combine_features(rel_back, def_feats, static_feats, deploy=deploy)
 return basetable<feature_engineering>
y_train_pred = reg.predict(X_train)
 y_test_pred = reg.predict(X_test)
 print("RMSE on Training set :", rmsle(y_train.values, y_train_pred ).mean())
 print("RMSE on Validation set :", rmsle(y_test.values, y_test_pred ).mean() )
%time train_basetable = create_features(train, False )<prepare_x_and_y>
reg = LinearRegression()
reg.fit(train, y)
 y_train_OLS = reg.predict(train)
 OLS_predictions = reg.predict(test)
 print("RMSE on Full Training set :", rmsle(y, y_train_OLS ).mean() )
X = train_basetable.copy()
yards = X.Yards
y = np.zeros(( yards.shape[0], 199))
for idx, target in enumerate(list(yards)) :
y[idx][99 + target] = 1
X.drop(['GameId','PlayId','Yards'], axis=1, inplace=True )<normalization>
alphas = np.linspace(1,50,50)
 ridge = RidgeCV(alphas = alphas, cv=10)
 ridge.fit(X_train, y_train)
 alpha = ridge.alpha_
print("Best alpha :", alpha)
 print("Try again for more precision with alphas centered around " + str(alpha))
ridge = RidgeCV(alphas = [alpha *.6, alpha *.65, alpha *.7, alpha *.75, alpha *.8, alpha *.85,
alpha *.9, alpha *.95, alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15,
alpha * 1.25, alpha * 1.3, alpha * 1.35, alpha * 1.4],
cv = 10)
 ridge.fit(X_train, y_train)
 alpha = ridge.alpha_
print("Best alpha :", alpha)
 print("Ridge picked " + str(sum(ridge.coef_ != 0)) + " features and eliminated the other " + \
str(sum(ridge.coef_ == 0)) + " features" )
scaler = StandardScaler()
X = scaler.fit_transform(X )<split>
y_train_rdg = ridge.predict(X_train)
 y_test_rdg = ridge.predict(X_test)
 print("RMSE on Training set :", rmsle(y_train.values, y_train_rdg ).mean())
 print("RMSE on Validation set :", rmsle(y_test.values, y_test_rdg ).mean() )
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.15, random_state=12345 )<set_options>
alphas = np.linspace(1,50,50)
 ridge = RidgeCV(alphas = alphas)
 ridge.fit(train, y)
 alpha = ridge.alpha_
print("Best alpha :", alpha)
 print("Try again for more precision with alphas centered around " + str(alpha))
ridge = RidgeCV(alphas = [alpha *.6, alpha *.65, alpha *.7, alpha *.75, alpha *.8, alpha *.85,
alpha *.9, alpha *.95, alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15,
alpha * 1.25, alpha * 1.3, alpha * 1.35, alpha * 1.4],
cv = 10)
 ridge.fit(train, y)
 alpha = ridge.alpha_
print("Best alpha :", alpha)
 print("Ridge picked " + str(sum(ridge.coef_ != 0)) + " features and eliminated the other " + \
str(sum(ridge.coef_ == 0)) + " features")
 y_ridge = ridge.predict(train)
 print("Ridge RMSE on Full Training set :", rmsle(y, y_ridge ).mean())
 ridge_submission_price = ridge.predict(test )
class CRPSCallback(Callback):
def __init__(self,validation, predict_batch_size=20, include_on_batch=False):
super(CRPSCallback, self ).__init__()
self.validation = validation
self.predict_batch_size = predict_batch_size
self.include_on_batch = include_on_batch
print('validation shape',len(self.validation))
def on_batch_begin(self, batch, logs={}):
pass
def on_train_begin(self, logs={}):
if not('CRPS_score_val' in self.params['metrics']):
self.params['metrics'].append('CRPS_score_val')
 def on_batch_end(self, batch, logs={}):
if(self.include_on_batch):
logs['CRPS_score_val'] = float('-inf')
 def on_epoch_end(self, epoch, logs={}):
logs['CRPS_score_val'] = float('-inf')
 if(self.validation):
X_valid, y_valid = self.validation[0], self.validation[1]
y_pred = self.model.predict(X_valid)
 y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * X_valid.shape[0])
 val_s = np.round(val_s, 6)
 logs['CRPS_score_val'] = val_s 
<choose_model_class>
alphas = np.linspace(0.00001,1,1000)
 lasso = LassoCV(alphas = alphas, max_iter = 50000, cv = 10)
 lasso.fit(X_train, y_train)
 alpha = lasso.alpha_
print("Best alpha :", alpha)
 print("Try again for more precision with alphas centered around " + str(alpha))
lasso = LassoCV(alphas = [alpha *.6, alpha *.65, alpha *.7, alpha *.75, alpha *.8,
alpha *.85, alpha *.9, alpha *.95, alpha, alpha * 1.05,
alpha * 1.1, alpha * 1.15, alpha * 1.25, alpha * 1.3, alpha * 1.35,
alpha * 1.4],
max_iter = 50000, cv = 10)
 lasso.fit(X_train, y_train)
 alpha = lasso.alpha_
print("Best alpha :", alpha)
 print("LASSO picked " + str(sum(lasso.coef_ != 0)) + " features and eliminated the other " + \
str(sum(lasso.coef_ == 0)) + " features" )
def get_model(x_tr,y_tr,x_val,y_val):
inp = Input(shape =(x_tr.shape[1],))
x = Dense(1024, input_dim=X.shape[1], activation='relu' )(inp)
 x = Dropout(0.60 )(x)
 x = BatchNormalization()(x)
 x = Dense(512, activation='relu' )(x)
 x = Dropout(0.55 )(x)
 x = BatchNormalization()(x)
 x = Dense(256, activation='relu' )(x)
 x = Dropout(0.55 )(x)
 x = BatchNormalization()(x)
 out = Dense(199, activation='softmax' )(x)
 model = Model(inp,out)
 model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=[])
 es = EarlyStopping(monitor='CRPS_score_val',
mode='min',
restore_best_weights=True,
verbose=1,
patience=81)
 mc = ModelCheckpoint('best_model.h5',monitor='CRPS_score_val',mode='min',
save_best_only=True, verbose=1, save_weights_only=True)
 bsz = 1024
steps = x_tr.shape[0]/bsz
model.fit(x_tr, y_tr,callbacks=[CRPSCallback(validation =(x_val,y_val)) ,es,mc], epochs=250, batch_size=bsz,verbose=1)
 model.load_weights("best_model.h5")
 y_pred = model.predict(x_val)
 y_valid = y_val
y_true = np.clip(np.cumsum(y_valid, axis=1), 0, 1)
 y_pred = np.clip(np.cumsum(y_pred, axis=1), 0, 1)
 val_s =(( y_true - y_pred)** 2 ).sum(axis=1 ).sum(axis=0)/(199 * x_val.shape[0])
 crps = np.round(val_s, 6)
 return model,crps<set_options>
y_train_las = lasso.predict(X_train)
 y_test_las = lasso.predict(X_test)
 print("Lasso RMSE on Training set :", rmsle(y_train, y_train_las ).mean())
 print("Lasso RMSE on Validation set :", rmsle(y_test, y_test_las ).mean() )
warnings.filterwarnings("ignore" )<split>
alphas = np.linspace(0.00001,1,1000)
 lasso = LassoCV(alphas = alphas, max_iter = 50000, cv = 10)
 lasso.fit(train, y)
 alpha = lasso.alpha_
print("Best alpha :", alpha)
 print("Try again for more precision with alphas centered around " + str(alpha))
lasso = LassoCV(alphas = [alpha *.6, alpha *.65, alpha *.7, alpha *.75, alpha *.8,
alpha *.85, alpha *.9, alpha *.95, alpha, alpha * 1.05,
alpha * 1.1, alpha * 1.15, alpha * 1.25, alpha * 1.3, alpha * 1.35,
alpha * 1.4],
max_iter = 50000, cv = 10)
 lasso.fit(train, y)
 alpha = lasso.alpha_
print("Best alpha :", alpha)
 print("LASSO picked " + str(sum(lasso.coef_ != 0)) + " features and eliminated the other " + \
str(sum(lasso.coef_ == 0)) + " features")
 y_lasso = lasso.predict(train)
 print("Lasso RMSE on Training set :", rmsle(y, y_lasso ).mean())
 lasso_submission_price = lasso.predict(test )
losses = []
models = []
crps_csv = []
s_time = time.time()
for k in range(2):
kfold = KFold(5, random_state = 42 + k, shuffle = True)
 for k_fold,(tr_inds, val_inds)in enumerate(kfold.split(yards)) :
print("-----------")
 print("-----------")
 tr_x,tr_y = X[tr_inds],y[tr_inds]
val_x,val_y = X[val_inds],y[val_inds]
model,crps = get_model(tr_x,tr_y,val_x,val_y)
 models.append(model)
 print("the %d fold crps is %f"%(( k_fold+1),crps))
crps_csv.append(crps)
 print("mean crps is %f"%np.mean(crps_csv))
def predict(x_te):
model_num = len(models)
 for k,m in enumerate(models):
if k==0:
y_pred = m.predict(x_te,batch_size=1024)
 else:
y_pred+=m.predict(x_te,batch_size=1024)
 y_pred = y_pred / model_num
return y_pred 
<compute_test_metric>
ridge_submission_price = np.expm1(ridge_submission_price )
<predict_on_test><EOS> 
solution = pd.DataFrame({"Id":test_ID, "SalePrice":ridge_submission_price})
 solution.to_csv("ridge_sol.csv", index = False )