Kaggle competition: House Prices - Advanced Regression Techniques
env.write_submission_file()<set_options>
test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 sample_submission = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/sample_submission.csv' )
torch.manual_seed(0)
 np.random.seed(0)
 Rand_matrix = scipy.special.logit(np.random.rand(22,22))
class Data(object):
def __init__(self, players: torch.Tensor, rusher: torch.Tensor, meta: torch.Tensor, y: Optional[torch.Tensor],
yardLine: np.ndarray, year: np.ndarray, player_cols: List[str], rusher_cols: List[str],
meta_cols: List[str]):
self.players = players
self.rusher = rusher
self.meta = meta
self.y = y
self.yardLine = yardLine
self.year = year
self.player_cols = player_cols
self.rusher_cols = rusher_cols
self.meta_cols = meta_cols
assert self.players.size(0)== self.rusher.size(0)
 if yardLine is not None:
assert len(yardLine)== self.players.size(0)
 def len(self):
return self.players.size(0)
 def y_soft(self, sigma: float = 1.0):
return torch.from_numpy(gaussian_filter1d(self.y.numpy() , sigma=sigma))
def slice(self, begin: int, end: int)-> 'Data':
p = self.players[begin:end] if self.players is not None else None
r = self.rusher[begin:end] if self.rusher is not None else None
m = self.meta[begin:end] if self.meta is not None else None
y = self.y[begin:end] if self.y is not None else None
yd = self.yardLine[begin:end].copy() if self.yardLine is not None else None
yr = self.year[begin:end].copy() if self.year is not None else None
return Data(p, r, m, y, yd, yr, self.player_cols, self.rusher_cols, self.meta_cols)
 def _sample_by_mask(self, mask):
mask_tensor = torch.from_numpy(mask)
 p = self.players[mask_tensor] if self.players is not None else None
r = self.rusher[mask_tensor] if self.rusher is not None else None
m = self.meta[mask_tensor] if self.meta is not None else None
y = self.y[mask_tensor] if self.y is not None else None
yd = self.yardLine[mask].copy() if self.yardLine is not None else None
yr = self.year[mask].copy() if self.year is not None else None
return Data(p, r, m, y, yd, yr, self.player_cols, self.rusher_cols, self.meta_cols)
 def downsample_2017(self, dropout_rate: float):
assert 0 <= dropout_rate <= 1.0
dropout = np.random.choice([True, False], size=len(self.year), p=[1 - dropout_rate, dropout_rate])
 mask =(self.year != 2017)| dropout
return self._sample_by_mask(mask)
 def shuffled(self):
indices = np.random.permutation(self.players.shape[0])
 p = np.take(self.players, indices, axis=0)
 r = np.take(self.rusher, indices, axis=0)if self.rusher is not None else None
m = np.take(self.meta, indices, axis=0)if self.meta is not None else None
y = np.take(self.y, indices, axis=0)if self.y is not None else None
yd = np.take(self.yardLine, indices, axis=0 ).copy() if self.yardLine is not None else None
yr = np.take(self.year, indices, axis=0 ).copy() if self.year is not None else None
return Data(p, r, m, y, yd, yr, self.player_cols, self.rusher_cols, self.meta_cols)
 def random_split(self, p: float):
mask = np.random.choice([True, False], p=[p, 1 - p], size=self.meta.size(0))
d1 = self._sample_by_mask(mask)
 d2 = self._sample_by_mask(~mask)
 return d1, d2
def concat_dataset(l: Data, r: Data):
p = torch.cat(( l.players, r.players))
rs = torch.cat(( l.rusher, r.rusher))
m = torch.cat(( l.meta, r.meta))
y = torch.cat(( l.y, r.y)) if l.y is not None else None
yd = np.concatenate([l.yardLine, r.yardLine])if l.yardLine is not None else None
yr = np.concatenate([l.year, r.year])if l.year is not None else None
print(p.size())
 print(yd.shape)
 return Data(p, rs, m, y, yd, yr, l.player_cols, l.rusher_cols, l.meta_cols)
 def _get_clones(module, N):
return ModuleList([copy.deepcopy(module)for i in range(N)])
 class MultiheadAttention(Module):
__constants__ = ['q_proj_weight', 'k_proj_weight', 'v_proj_weight', 'in_proj_weight', 'R_proj_weight']
def __init__(self, embed_dim, num_heads, dropout=0., bias=True, add_bias_kv=False, add_zero_attn=False, kdim=None,
vdim=None):
super(MultiheadAttention, self ).__init__()
self.embed_dim = embed_dim
self.kdim = kdim if kdim is not None else embed_dim
self.vdim = vdim if vdim is not None else embed_dim
self._qkv_same_embed_dim = self.kdim == embed_dim and self.vdim == embed_dim
self.num_heads = num_heads
self.dropout = dropout
self.head_dim = embed_dim // num_heads
assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads"
if self._qkv_same_embed_dim is False:
self.q_proj_weight = Parameter(torch.Tensor(embed_dim, embed_dim))
self.k_proj_weight = Parameter(torch.Tensor(embed_dim, self.kdim))
self.v_proj_weight = Parameter(torch.Tensor(embed_dim, self.vdim))
self.register_parameter('in_proj_weight', None)
 else:
self.in_proj_weight = Parameter(torch.empty(3 * embed_dim, embed_dim))
self.register_parameter('q_proj_weight', None)
 self.register_parameter('k_proj_weight', None)
 self.register_parameter('v_proj_weight', None)
 self.R_proj_weight = Parameter(torch.Tensor(22, 22))
self.alpha_1 = Parameter(torch.Tensor(1))
torch.nn.init.uniform_(self.alpha_1, a=0.49, b=0.51)
 if bias:
self.in_proj_bias = Parameter(torch.empty(3 * embed_dim))
else:
self.register_parameter('in_proj_bias', None)
 self.out_proj = Linear(embed_dim, embed_dim, bias=bias)
 if add_bias_kv:
self.bias_k = Parameter(torch.empty(1, 1, embed_dim))
self.bias_v = Parameter(torch.empty(1, 1, embed_dim))
else:
self.bias_k = self.bias_v = None
self.add_zero_attn = add_zero_attn
self._reset_parameters()
def _reset_parameters(self):
if self._qkv_same_embed_dim:
xavier_uniform_(self.in_proj_weight)
 xavier_normal_(self.R_proj_weight)
 else:
xavier_uniform_(self.q_proj_weight)
 xavier_uniform_(self.k_proj_weight)
 xavier_uniform_(self.v_proj_weight)
 if self.in_proj_bias is not None:
constant_(self.in_proj_bias, 0.)
constant_(self.out_proj.bias, 0.)
if self.bias_k is not None:
xavier_normal_(self.bias_k)
 if self.bias_v is not None:
xavier_normal_(self.bias_v)
 def forward(self, query, key, value, key_padding_mask=None,
need_weights=True, attn_mask=None):
if not self._qkv_same_embed_dim:
return multi_head_attention_forward(
 query, key, value, self.embed_dim, self.num_heads,
self.in_proj_weight, self.in_proj_bias,
self.bias_k, self.bias_v, self.add_zero_attn,
self.dropout, self.out_proj.weight, self.out_proj.bias,
training=self.training,
key_padding_mask=key_padding_mask, need_weights=need_weights,
attn_mask=attn_mask, use_separate_proj_weight=True,
q_proj_weight=self.q_proj_weight, k_proj_weight=self.k_proj_weight,
v_proj_weight=self.v_proj_weight, R_proj_weight=self.R_proj_weight,alpha_1=self.alpha_1)
 else:
return multi_head_attention_forward(
 query, key, value, self.embed_dim, self.num_heads,
self.in_proj_weight, self.in_proj_bias,
self.bias_k, self.bias_v, self.add_zero_attn,
self.dropout, self.out_proj.weight, self.out_proj.bias,
training=self.training,
key_padding_mask=key_padding_mask, need_weights=need_weights,
attn_mask=attn_mask, R_proj_weight=self.R_proj_weight,alpha_1=self.alpha_1)
 class TransformerEncoder(Module):
def __init__(self, encoder_layer, num_layers, norm=None):
super(TransformerEncoder, self ).__init__()
self.layers = _get_clones(encoder_layer, num_layers)
 self.num_layers = num_layers
self.norm = norm
def forward(self, src, mask=None, src_key_padding_mask=None):
r
output = src
for i in range(self.num_layers):
output = self.layers[i](output, src_mask=mask,
src_key_padding_mask=src_key_padding_mask)
 if self.norm:
output = self.norm(output)
 return output
class TransformerEncoderLayer(Module):
def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, dropout_attn=0.1, pre_LN=False):
super(TransformerEncoderLayer, self ).__init__()
self.self_attn = MultiheadAttention(d_model, nhead, dropout=dropout_attn)
 self.linear1 = Linear(d_model, dim_feedforward)
 self.dropout = Dropout(dropout)
 self.linear2 = Linear(dim_feedforward, d_model)
 self.norm1 = LayerNorm(d_model)
 self.norm2 = LayerNorm(d_model)
 self.dropout1 = Dropout(dropout)
 self.dropout2 = Dropout(dropout)
 self.activation = F.relu
self.pre_LN = pre_LN
def forward(self, src, src_mask=None, src_key_padding_mask=None):
r
if self.pre_LN:
src2 = self.norm1(src)
 src2 = self.self_attn(src2, src2, src2, attn_mask=src_mask,
key_padding_mask=src_key_padding_mask)[0]
src = src + self.dropout1(src2)
 else:
src2 = self.self_attn(src, src, src, attn_mask=src_mask,
key_padding_mask=src_key_padding_mask)[0]
src = src + self.dropout1(src2)
 src = self.norm1(src)
 if hasattr(self, "activation"):
src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))
else:
src2 = self.linear2(self.dropout(F.relu(self.linear1(src))))
src = src + self.dropout2(src2)
 src = self.norm2(src)
 return src
def multi_head_attention_forward(query,
key,
value,
embed_dim_to_check,
num_heads,
in_proj_weight,
in_proj_bias,
bias_k,
bias_v,
add_zero_attn,
dropout_p,
out_proj_weight,
out_proj_bias,
training=True,
key_padding_mask=None,
need_weights=True,
attn_mask=None,
use_separate_proj_weight=False,
q_proj_weight=None,
k_proj_weight=None,
v_proj_weight=None,
R_proj_weight=None,
static_k=None,
static_v=None,
alpha_1=None 
):
r
qkv_same = torch.equal(query, key)and torch.equal(key, value)
 kv_same = torch.equal(key, value)
 tgt_len, bsz, embed_dim = query.size()
assert embed_dim == embed_dim_to_check
assert list(query.size())== [tgt_len, bsz, embed_dim]
assert key.size() == value.size()
head_dim = embed_dim // num_heads
assert head_dim * num_heads == embed_dim, "embed_dim must be divisible by num_heads"
scaling = float(head_dim)** -0.5
if use_separate_proj_weight is not True:
if qkv_same:
q, k, v = F.linear(query, in_proj_weight, in_proj_bias ).chunk(3, dim=-1)
 elif kv_same:
_b = in_proj_bias
_start = 0
_end = embed_dim
_w = in_proj_weight[_start:_end, :]
if _b is not None:
_b = _b[_start:_end]
q = F.linear(query, _w, _b)
 if key is None:
assert value is None
k = None
v = None
else:
_b = in_proj_bias
_start = embed_dim
_end = None
_w = in_proj_weight[_start:, :]
if _b is not None:
_b = _b[_start:]
k, v = F.linear(key, _w, _b ).chunk(2, dim=-1)
 else:
_b = in_proj_bias
_start = 0
_end = embed_dim
_w = in_proj_weight[_start:_end, :]
if _b is not None:
_b = _b[_start:_end]
q = F.linear(query, _w, _b)
 _b = in_proj_bias
_start = embed_dim
_end = embed_dim * 2
_w = in_proj_weight[_start:_end, :]
if _b is not None:
_b = _b[_start:_end]
k = F.linear(key, _w, _b)
 _b = in_proj_bias
_start = embed_dim * 2
_end = None
_w = in_proj_weight[_start:, :]
if _b is not None:
_b = _b[_start:]
v = F.linear(value, _w, _b)
 else:
q_proj_weight_non_opt = torch.jit._unwrap_optional(q_proj_weight)
 len1, len2 = q_proj_weight_non_opt.size()
assert len1 == embed_dim and len2 == query.size(-1)
 k_proj_weight_non_opt = torch.jit._unwrap_optional(k_proj_weight)
 len1, len2 = k_proj_weight_non_opt.size()
assert len1 == embed_dim and len2 == key.size(-1)
 v_proj_weight_non_opt = torch.jit._unwrap_optional(v_proj_weight)
 len1, len2 = v_proj_weight_non_opt.size()
assert len1 == embed_dim and len2 == value.size(-1)
 if in_proj_bias is not None:
q = F.linear(query, q_proj_weight_non_opt, in_proj_bias[0:embed_dim])
 k = F.linear(key, k_proj_weight_non_opt, in_proj_bias[embed_dim:(embed_dim * 2)])
 v = F.linear(value, v_proj_weight_non_opt, in_proj_bias[(embed_dim * 2):])
 else:
q = F.linear(query, q_proj_weight_non_opt, in_proj_bias)
 k = F.linear(key, k_proj_weight_non_opt, in_proj_bias)
 v = F.linear(value, v_proj_weight_non_opt, in_proj_bias)
 q = q * scaling
R_proj_weight_non_opt = torch.jit._unwrap_optional(R_proj_weight)
 len1, len2 = R_proj_weight_non_opt.size()
if bias_k is not None and bias_v is not None:
if static_k is None and static_v is None:
k = torch.cat([k, bias_k.repeat(1, bsz, 1)])
 v = torch.cat([v, bias_v.repeat(1, bsz, 1)])
 if attn_mask is not None:
attn_mask = torch.cat([attn_mask,
torch.zeros(( attn_mask.size(0), 1),
dtype=attn_mask.dtype,
device=attn_mask.device)], dim=1)
 if key_padding_mask is not None:
key_padding_mask = torch.cat(
 [key_padding_mask, torch.zeros(( key_padding_mask.size(0), 1),
dtype=key_padding_mask.dtype,
device=key_padding_mask.device)], dim=1)
 else:
assert static_k is None, "bias cannot be added to static key."
assert static_v is None, "bias cannot be added to static value."
else:
assert bias_k is None
assert bias_v is None
q = q.contiguous().view(tgt_len, bsz * num_heads, head_dim ).transpose(0, 1)
 if k is not None:
k = k.contiguous().view(-1, bsz * num_heads, head_dim ).transpose(0, 1)
 if v is not None:
v = v.contiguous().view(-1, bsz * num_heads, head_dim ).transpose(0, 1)
 if static_k is not None:
assert static_k.size(0)== bsz * num_heads
assert static_k.size(2)== head_dim
k = static_k
if static_v is not None:
assert static_v.size(0)== bsz * num_heads
assert static_v.size(2)== head_dim
v = static_v
src_len = k.size(1)
 if key_padding_mask is not None:
assert key_padding_mask.size(0)== bsz
assert key_padding_mask.size(1)== src_len
if add_zero_attn:
src_len += 1
k = torch.cat([k, torch.zeros(( k.size(0), 1)+ k.size() [2:], dtype=k.dtype, device=k.device)], dim=1)
 v = torch.cat([v, torch.zeros(( v.size(0), 1)+ v.size() [2:], dtype=v.dtype, device=v.device)], dim=1)
 if attn_mask is not None:
attn_mask = torch.cat([attn_mask, torch.zeros(( attn_mask.size(0), 1),
dtype=attn_mask.dtype,
device=attn_mask.device)], dim=1)
 if key_padding_mask is not None:
key_padding_mask = torch.cat(
 [key_padding_mask, torch.zeros(( key_padding_mask.size(0), 1),
dtype=key_padding_mask.dtype,
device=key_padding_mask.device)], dim=1)
 q_k_product = torch.bmm(q, k.transpose(1, 2))
alpha_1 = np.clip(alpha_1.detach().numpy() ,0,1)
 temp_matrix = np.tile(R_proj_weight_non_opt.detach().numpy() ,(bsz * num_heads,1,1))
attn_output_weights = 0.75* torch.Tensor(temp_matrix)+ 0.25 * q_k_product
assert list(attn_output_weights.size())== [bsz * num_heads, tgt_len, src_len]
if attn_mask is not None:
attn_mask = attn_mask.unsqueeze(0)
 attn_output_weights += attn_mask
if key_padding_mask is not None:
attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
 attn_output_weights = attn_output_weights.masked_fill(
 key_padding_mask.unsqueeze(1 ).unsqueeze(2),
float('-inf'), 
)
 attn_output_weights = attn_output_weights.view(bsz * num_heads, tgt_len, src_len)
 attn_output_weights = F.softmax(
 attn_output_weights, dim=-1)
 attn_output_weights = F.dropout(attn_output_weights, p=dropout_p, training=training)
 attn_output = torch.bmm(attn_output_weights, v)
 assert list(attn_output.size())== [bsz * num_heads, tgt_len, head_dim]
attn_output = attn_output.transpose(0, 1 ).contiguous().view(tgt_len, bsz, embed_dim)
 attn_output = F.linear(attn_output, out_proj_weight, out_proj_bias)
 if need_weights:
attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
 return attn_output, attn_output_weights.sum(dim=1)/ num_heads
else:
return attn_output, None
class TransformerModel(nn.Module):
def __init__(self, ninp: int, nemb: int = 1, nhead: int = 1, nhid: int = 32, nlayers: int = 4, nfinal: int = 1024,
dropout_encoder: float = 0.1, dropout_embed: float = 0.0, dropout_classifier: float = 0.0,
n_class: int = 199, ninp_rusher: int = 16, pre_LN: bool = False, rusher_emb: int = 32,
n_emb_layers: int = 2,
ninp_meta: int = 8, meta_emb: int = 32, gauss_noise: float = 0.0,
gauss_xy_noise: float = 0.0,
n_fin_layers: int = 3, dropout_attn: float = 0):

 super(TransformerModel, self ).__init__()
self.model_type = 'Transformer'
encoder_layers = TransformerEncoderLayer(nemb, nhead, nhid, dropout_encoder, pre_LN=pre_LN, dropout_attn=dropout_attn)
 self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
 self.n_emb_layers = n_emb_layers
self.conv1 = nn.Conv1d(in_channels=ninp, out_channels=nemb, kernel_size=1)
 self.conv2 = nn.Conv1d(in_channels=nemb, out_channels=nemb, kernel_size=1)
 self.conv3 = nn.Conv1d(in_channels=nemb, out_channels=nemb, kernel_size=1)
 self.relu1 = nn.PReLU()
self.relu2 = nn.PReLU()
self.relu3 = nn.PReLU()
self.relu4 = nn.PReLU()
self.relu5 = nn.ReLU()
if dropout_embed > 0:
self.dropout1 = nn.Dropout(dropout_embed)
 else:
self.dropout1 = None
self.avgpool = nn.AvgPool1d(kernel_size=22)
 assert n_fin_layers == 3
self.linear = nn.Sequential(
 nn.Dropout(dropout_classifier),
nn.Linear(nemb + rusher_emb + meta_emb, nfinal),
nn.ReLU() ,
nn.Dropout(dropout_classifier),
nn.Linear(nfinal, nfinal),
nn.ReLU() ,
nn.Dropout(dropout_classifier),
nn.Linear(nfinal, n_class)
)
 self.activation = nn.Softmax(dim=1)
 self.linear2 = nn.Linear(ninp_rusher, rusher_emb)
 self.gauss_noise = gauss_noise
self.gauss_xy_noise = gauss_xy_noise
if meta_emb > 0:
self.linear3 = nn.Linear(ninp_meta, meta_emb)
 else:
self.linear3 = None
def forward(self, x: Data, with_clip: bool = True):
src = x.players
src_rusher = x.rusher
src = src.permute([0, 2, 1])
 if self.training:
if self.gauss_noise > 0.0:
noise = torch.randn_like(src)* self.gauss_noise
src = src + noise
if self.gauss_xy_noise > 0.0:
dy = torch.randn(src.size(0)) * self.gauss_xy_noise
src[:, 1, :] += dy.reshape(src.size(0), 1 ).expand(src.size(0), src.size(2))
src_rusher[:, 1] += dy
src = self.relu1(self.conv1(src))
src = self.dropout1(src)
 src = self.relu2(self.conv2(src))
src = self.dropout1(src)
 src = self.relu3(self.conv3(src))
src = src.permute([2, 0, 1])
 output = self.transformer_encoder(src)
 output = output.permute([1, 2, 0])
 output = torch.squeeze(self.avgpool(output), dim=2)
 if self.linear3 is not None:
output = torch.cat(( output, self.relu4(self.linear2(src_rusher)) , self.relu5(self.linear3(x.meta))), dim=1)
 else:
output = torch.cat(( output, self.relu4(self.linear2(src_rusher))), dim=1)
 output = self.linear(output)
 output = self.activation(output)
 if not self.training and x.yardLine is not None:
output = torch.cumsum(output, dim=1 ).numpy()
output = np.clip(output, 0.0, 1.0)
 if with_clip:
left = 99 - x.yardLine
right = 199 - x.yardLine
for k in range(len(output)) :
output[k, :left[k] + 1] = 0.0
output[k, right[k]:] = 1.0
return output
def prep(df: pd.DataFrame)-> Tuple[pd.DataFrame, pd.DataFrame]:
df['ToLeft'] = df.PlayDirection == "left"
df['IsBallCarrier'] = df.NflId == df.NflIdRusher
df.loc[df.VisitorTeamAbbr == "ARI", 'VisitorTeamAbbr'] = "ARZ"
df.loc[df.HomeTeamAbbr == "ARI", 'HomeTeamAbbr'] = "ARZ"
df.loc[df.VisitorTeamAbbr == "BAL", 'VisitorTeamAbbr'] = "BLT"
df.loc[df.HomeTeamAbbr == "BAL", 'HomeTeamAbbr'] = "BLT"
df.loc[df.VisitorTeamAbbr == "CLE", 'VisitorTeamAbbr'] = "CLV"
df.loc[df.HomeTeamAbbr == "CLE", 'HomeTeamAbbr'] = "CLV"
df.loc[df.VisitorTeamAbbr == "HOU", 'VisitorTeamAbbr'] = "HST"
df.loc[df.HomeTeamAbbr == "HOU", 'HomeTeamAbbr'] = "HST"
df['TeamOnOffense'] = "home"
df.loc[df.PossessionTeam != df.HomeTeamAbbr, 'TeamOnOffense'] = "away"
df['IsOnOffense'] = df.Team == df.TeamOnOffense
df['YardLine_std'] = 100 - df.YardLine
df.loc[df.FieldPosition.fillna('')== df.PossessionTeam, 'YardLine_std'] = df.loc[
df.FieldPosition.fillna('')== df.PossessionTeam, 'YardLine']
df['X_std'] = df.X
df.loc[df.ToLeft, 'X_std'] = 120 - df.loc[df.ToLeft, 'X']
df['Y_std'] = df.Y
df.loc[df.ToLeft, 'Y_std'] = 160 / 3 - df.loc[df.ToLeft, 'Y']
df['Orientation_std'] = df.Orientation
df.loc[df.ToLeft, 'Orientation_std'] = np.mod(180 + df.loc[df.ToLeft, 'Orientation_std'], 360)
 df['Dir_std'] = df.Dir
df.loc[df.ToLeft, 'Dir_std'] = np.mod(180 + df.loc[df.ToLeft, 'Dir_std'], 360)
 df['IsOffence'] = df['Team'] == df['TeamOnOffense']
df['OffenceScoreBeforePlay'] = df['HomeScoreBeforePlay']
df.loc[df.TeamOnOffense == "away", 'OffenceScoreBeforePlay'] = df.loc[
df.TeamOnOffense == "away", 'VisitorScoreBeforePlay']
df['DefenseScoreBeforePlay'] = df['VisitorScoreBeforePlay']
df.loc[df.TeamOnOffense == "away", 'DefenseScoreBeforePlay'] = df.loc[
df.TeamOnOffense == "away", 'HomeScoreBeforePlay']
df['OffenceTeamAbbr'] = df['HomeTeamAbbr']
df.loc[df.TeamOnOffense == "away", 'OffenceTeamAbbr'] = df.loc[df.TeamOnOffense == "away", 'VisitorTeamAbbr']
df['DefenseTeamAbbr'] = df['VisitorTeamAbbr']
df.loc[df.TeamOnOffense == "away", 'DefenseTeamAbbr'] = df.loc[df.TeamOnOffense == "away", 'HomeTeamAbbr']
df['Year'] = pd.to_datetime(df.TimeSnap ).dt.year
df.loc[df['Year'] == 2017, 'Orientation_std'] = np.mod(90 + df.loc[df['Year'] == 2017, 'Orientation_std'], 360)
 player_features = ['X_std', 'Y_std', 'S', 'A', 'Dis', 'Orientation_std', 'Dir_std', 'NflId', 'JerseyNumber',
'PlayerHeight',
'PlayerWeight', 'PlayerBirthDate', 'PlayerCollegeName', 'Position', 'IsBallCarrier', 'IsOffence']
play_features = ['YardLine_std', 'Quarter', 'GameClock', 'PossessionTeam', 'Down', 'Distance', 'FieldPosition',
'OffenceScoreBeforePlay', 'DefenseScoreBeforePlay', 'OffenseFormation', 'OffensePersonnel',
'DefendersInTheBox', 'DefensePersonnel', 'TimeHandoff', 'TimeSnap',
'OffenceTeamAbbr', 'DefenseTeamAbbr', 'Week', 'Stadium', 'Location',
'Turf', 'GameWeather', 'Temperature', 'Humidity', 'WindSpeed', 'WindDirection',
'TeamOnOffense', 'HomeTeamAbbr', 'Year']
if 'Yards' in df:
play_features.append('Yards')
 players = df[['PlayId', 'GameId'] + player_features].copy()
play = df[['GameId', 'PlayId'] + play_features].copy().drop_duplicates(subset=['PlayId'])
 return play, players
def prep_players_nn(play, players, scaler=None, scaler_meta=None):
p = players.drop(['NflId', 'JerseyNumber', 'PlayerHeight', 'PlayerBirthDate', 'PlayerCollegeName', 'GameId'],
axis=1 ).copy()
if scaler is None:
is_training = True
else:
assert scaler_meta is not None
is_training = False
if 'YardLine_std' not in p:
p = pd.merge(p, play[['YardLine_std', 'PlayId']], on='PlayId', how='left')
 p['IsBallCarrier'] = p['IsBallCarrier'].astype(int)
 p['IsOffence'] = p['IsOffence'].astype(int)
 p['X_std'] -= p['YardLine_std']
p['Dir_cos'] = np.cos(np.deg2rad(90 - p['Dir_std']))
p['Dir_sin'] = np.sin(np.deg2rad(90 - p['Dir_std']))
p = p.fillna(0)
 rb = p[p['IsBallCarrier'] == 1][['X_std', 'Y_std', 'PlayId', 'S', 'Dir_sin', 'Dir_cos', 'Dir_std']]
rb.columns = ['X_', 'Y_', 'PlayId', 'S_', 'Dir_sin_', 'Dir_cos_', 'Dir_std_']
p = pd.merge(p, rb, how='left')
 p['DX'] = p['X_std'] - p['X_']
p['DY'] = p['Y_std'] - p['Y_']
angles = 90.0 - np.rad2deg(np.arctan2(p['DY'], p['DX']))
p['AngleFromRB'] = angles - p['Dir_std_']
p['AngleFromRB'] = np.mod(p['AngleFromRB'] + 360, 360)
 p.loc[p['AngleFromRB'] > 180, 'AngleFromRB'] -= 360
dt = 0.001
p['DX2'] =(p['X_std'] + dt * p['Dir_cos'] * p['S'])-(p['X_'] + dt * p['Dir_cos_'] * p['S_'])
 p['DY2'] =(p['Y_std'] + dt * p['Dir_sin'] * p['S'])-(p['Y_'] + dt * p['Dir_sin_'] * p['S_'])
 angles = 90.0 - np.rad2deg(np.arctan2(p['DY2'], p['DX2']))
p['AngleFromRB2'] = angles - p['Dir_std_']
p['AngleFromRB2'] = np.mod(p['AngleFromRB2'] + 360, 360)
 p.loc[p['AngleFromRB2'] > 180, 'AngleFromRB2'] -= 360
p['AngleFromRB2'] = p['AngleFromRB2'] - p['AngleFromRB']
p.loc[p['AngleFromRB2'] > 180, 'AngleFromRB2'] -= 360
p.loc[p['AngleFromRB2'] < -180, 'AngleFromRB2'] += 360
p.loc[p['IsBallCarrier'] == 1, 'AngleFromRB'] = 0
p.loc[p['IsBallCarrier'] == 1, 'AngleFromRB2'] = 0
p.drop(['DX2', 'DY2'], axis=1, inplace=True)
 p['AngleTan'] = np.arctan2(p['DY'], p['DX'])
 p = p.replace([np.inf, -np.inf], np.nan)
 p = p.fillna(0)
 p.drop(['X_', 'Y_', 'Dir_sin_', 'Dir_cos_', 'S_', 'Dir_std_', 'Orientation_std', 'Dir_std'], axis=1, inplace=True)
 concat = p.drop(['Position', 'PlayId', 'YardLine_std'], axis=1)
 if is_training:
scaler = StandardScaler()
scaled = scaler.fit_transform(concat.values)
 else:
scaled = scaler.transform(concat.values)
 df = pd.DataFrame(scaled, columns=concat.columns)
 df_rusher = df[p['IsBallCarrier'] == 1].reset_index(drop=True)
 drop_cols = ['IsBallCarrier', 'IsOffence', 'OffenseDist0', 'PlayerWeight', 'DX', 'DY', 'AngleFromRB', 'AngleToRB',
'DistDelta', 'AngleTan', 'AngleFromRB2', 'delaunay_adj']
drop_cols = [c for c in drop_cols if c in df_rusher.columns]
df_rusher.drop(drop_cols, axis=1, inplace=True)
 df_rusher = df_rusher.set_index(players['PlayId'].iloc[np.arange(0, len(players), 22)])
 meta = play[['YardLine_std', 'Distance']].copy()
meta = meta.replace([np.inf, -np.inf], np.nan)
 meta = meta.fillna(0)
 if scaler_meta is not None:
scaled_meta = scaler_meta.transform(meta.values)
 else:
scaler_meta = StandardScaler()
scaled_meta = scaler_meta.fit_transform(meta.values)
 scaled_meta = pd.DataFrame(scaled_meta, columns=meta.columns)
 return df.set_index(players['PlayId']), df_rusher, scaled_meta, scaler, scaler_meta
class EnsembleModel(object):
def __init__(self):
self.models = []
def add_model(self, model):
self.models.append(model)
 def train(self):
for m in self.models:
m.train()
def eval(self):
for m in self.models:
m.eval()
def __call__(self, *input, **kwargs):
assert len(self.models)>= 1
base = self.models[0](*input, **kwargs)
 for m in self.models[1:]:
base += m(*input, **kwargs)
 return base / len(self.models)
 class SnapShot(object):
def __init__(self, model: nn.Module, epoch: int, loss: float):
self.state = copy.deepcopy(model.state_dict())
 self.epoch = epoch
self.loss = loss
torch.save(self.state, f'snapshot_{epoch}_{loss}')
 class SnapShots(object):
def __init__(self, interval: int = 3, torelance: float = 1.01, verbose: bool = True):
self.best_model = None
self.snap_shots = []
self.best_val_loss = 1.0
self.interval = interval
self.torelance = torelance
self.verbose = verbose
def add(self, model: nn.Module, epoch: int, loss: float):
if loss < self.best_val_loss:
if self.verbose:
print(f'best model is updated.epoch{epoch}, loss={loss:.7f}')
 self.best_model = SnapShot(model, epoch, loss)
 self.best_val_loss = loss
if epoch % self.interval == 0:
if self.verbose:
print(f'Add snapshot.epoch{epoch}, loss={loss:.7f}')
 self.snap_shots.append(SnapShot(model, epoch, loss))
def load_best_single_model(self, model: nn.Module):
if self.best_model is not None:
model.load_state_dict(self.best_model.state)
 return model
def load_ensemble_model(self, cls: Type, params: Dict, max_models: int = 5):
model = EnsembleModel()
best = cls(**params)
 self.load_best_single_model(best)
 model.add_model(best)
 candidates = []
for i, s in enumerate(self.snap_shots):
if s.loss > self.torelance * self.best_model.loss:
continue
if s.epoch == self.best_model.epoch:
continue
candidates.append(( i, s.loss))
candidates = sorted(candidates, key = lambda x: x[1])
 if len(candidates)> max_models - 1:
candidates = candidates[:max_models - 1]
for i, _ in candidates:
s = self.snap_shots[i]
sub = cls(**params)
 sub.load_state_dict(s.state)
 model.add_model(sub)
 if self.verbose:
print(f'add {s.epoch}-th epoch to ensemble(loss:{s.loss:.7f})')
 return model
def train_model(model: nn.Module, scheduler, batch_size: int,
train_data: Data, valid_data: Data, writer, epochs: int,
downsample_2017: float,
calc_train_loss: bool = True,
params: Dict = None):
np.random.seed(0)
 std_batch_length = 12000 // batch_size + 1
n_total_batch = 0
epoch = 1
epoch_start_time = time.time()
ensemble = None
model.eval()
snapshots = SnapShots(torelance=1.007, interval=3)
 snapshots.add(model, 0, evaluate(model, valid_data))
model.train()
while epoch < epochs:
data = train_data.shuffled()
if downsample_2017 > 0.0:
data = data.downsample_2017(downsample_2017)
 for i in range(0, data.len() - 1, batch_size):
iend = min(i + batch_size, data.len())
 batch_data = data.slice(i, iend)
 optimizer.zero_grad()
output = model(batch_data)
 loss = crps(output, batch_data.y)
 loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 0.5)
 optimizer.step()
n_total_batch += 1
if n_total_batch > std_batch_length:
model.eval()
if calc_train_loss and epoch % 10 == 0:
train_loss = evaluate(model, data)
 else:
train_loss = -1
if epoch % 4 == 0 or epoch > 30:
val_loss = evaluate(model, valid_data)
 snapshots.add(model, epoch, val_loss)
 else:
val_loss = -1
print('| end of epoch {:3d} | lr: {:2.5f} | time: {:5.2f}s | train loss {:5.6f} | '
'valid loss {:5.9f}'.format(epoch, scheduler.get_lr() [0],(time.time() - epoch_start_time),
train_loss, val_loss))
for name, param in model.named_parameters() :
if param.requires_grad and name == 'transformer_encoder.layers.0.self_attn.alpha_1':
print(name, param.data)
 scheduler.step()
if writer is not None:
writer.writerow([epoch, scheduler.get_lr() [0], train_loss, val_loss])
 f.flush()
n_total_batch = 0
epoch += 1
epoch_start_time = time.time()
model.train()
if epoch in [40]:
batch_size *= 2
std_batch_length = 12000 // batch_size + 1
model = snapshots.load_best_single_model(model)
 eval_single = evaluate(model, valid_data)
 try:
ensemble = snapshots.load_ensemble_model(TransformerModel, params)
 eval_ensemble = evaluate(ensemble, valid_data)
 print(f'final loss: {eval_single:.8f}(single)/ {eval_ensemble:.8f}({len(ensemble.models)} models)')
 except:
pass
return ensemble if ensemble is not None else model
def evaluate(eval_model: nn.Module, data: Data, with_clip: bool = True):
eval_model.eval()
assert data.y is not None
with torch.no_grad() :
y_actual = data.y.numpy()
y_predicted = eval_model(data, with_clip)
 loss = np.sum(( y_predicted - y_actual)** 2)/(199 * len(y_predicted))
return loss
def return_delta(x):
temp = np.zeros(199)
 temp[x + 99:] = 1
return temp
def NFL_validation_split(df: pd.DataFrame, game_set=None):
games = df[['GameId', 'PossessionTeam']].drop_duplicates()
games = games.sort_values(['PossessionTeam', 'GameId'], ascending=[True, False])
 games['row_number'] = games.groupby(['PossessionTeam'] ).cumcount() + 1
game_set = game_set or {1, 2, 3, 4, 5}
game_ids = set(games[games['row_number'].isin(game_set)]['GameId'].unique().tolist())
 return game_ids
def NFL_group_split(df: pd.DataFrame, nfolds: int, nidx: int):
kf = GroupKFold(nfolds)
 train_idx, valid_idx = list(kf.split(df, groups=df['GameId'])) [nidx]
return set(df['GameId'].iloc[valid_idx].unique())
 def load_data_nn_test(test_df: pd.DataFrame, scaler: StandardScaler, scaler_meta: StandardScaler)-> Data:
play, players = prep(test_df)
 players_nn, rusher_nn, meta_nn, _, _ = prep_players_nn(play, players, scaler, scaler_meta)
 X = players_nn.values.astype(np.float32 ).reshape(( -1, 22, len(players_nn.columns)))
 Xr = rusher_nn.values.astype(np.float32)
 Xm = meta_nn.values.astype(np.float32)
 year = play['Year'].values.copy()
return Data(torch.from_numpy(X), torch.from_numpy(Xr), torch.from_numpy(Xm),
None, play['YardLine_std'].values.copy() , year,
list(players_nn.columns), list(rusher_nn.columns), list(meta_nn.columns))
def load_data_nn(n_plays_sample=None,
nfolds=None, nidx=None,
skiprows=None,
game_set=None)-> Tuple[Data, Data, StandardScaler, StandardScaler]:
train = pd.read_csv('.. /input/nfl-big-data-bowl-2020/train.csv', nrows=n_plays_sample, skiprows=skiprows)
 play, players = prep(train)
 players_nn, rusher_nn, meta_nn, scaler, scaler_meta = prep_players_nn(play, players, None, None)
 assert len(players_nn)== len(rusher_nn)* 22
assert len(rusher_nn)== len(meta_nn)
 assert len(rusher_nn)== len(play)
 play_ids = np.array(players_nn.index[np.arange(0, len(players_nn), 22)])
 game_ids = np.array(players['GameId'].values[np.arange(0, len(players_nn), 22)])
 assert len(play_ids)== len(game_ids)
 if nfolds is not None:
assert nidx is not None
game_ids_valid = NFL_group_split(train, nfolds, nidx)
 else:
game_ids_valid = NFL_validation_split(train, game_set)
 X = players_nn.values.astype(np.float32 ).reshape(( -1, 22, len(players_nn.columns)))
 Xr = rusher_nn.values.astype(np.float32)
 Xm = meta_nn.values.astype(np.float32)
 y = np.vstack(play['Yards'].apply(return_delta ).values ).astype(np.float32)
 yardLine = play['YardLine_std'].values
year = play['Year'].values
valid_mask = np.isin(game_ids, np.array(list(game_ids_valid)))
 X_valid = torch.from_numpy(X[valid_mask])
 Xr_valid = torch.from_numpy(Xr[valid_mask])
 Xm_valid = torch.from_numpy(Xm[valid_mask])
 y_valid = torch.from_numpy(y[valid_mask])
 yd_valid = yardLine[valid_mask].copy()
yr_valid = year[valid_mask].copy()
X_train = torch.from_numpy(X[~valid_mask])
 Xr_train = torch.from_numpy(Xr[~valid_mask])
 Xm_train = torch.from_numpy(Xm[~valid_mask])
 y_train = torch.from_numpy(y[~valid_mask])
 yd_train = yardLine[~valid_mask].copy()
yr_train = year[~valid_mask].copy()
print(f'X_train: {X_train.shape}, {Xr_train.shape}, {Xm_train.shape}')
 print(f'X_valid: {X_valid.shape}, {Xr_valid.shape}, {Xm_valid.shape}')
 print(f'y_train: {y_train.shape}, y_valid: {y_valid.shape}')
 print(f'players: {list(players_nn.columns)}')
 print(f'rusher: {list(rusher_nn.columns)}')
 print(f'meta: {list(meta_nn.columns)}')
 dtrain = Data(X_train, Xr_train, Xm_train, y_train, yd_train, yr_train,
list(players_nn.columns), list(rusher_nn.columns), list(meta_nn.columns))
dvalid = Data(X_valid, Xr_valid, Xm_valid, y_valid, yd_valid, yr_valid,
list(players_nn.columns), list(rusher_nn.columns), list(meta_nn.columns))
assert len(X)== len(y)
 return dtrain, dvalid, scaler, scaler_meta
def crps(y_pred, y_true):
loss = torch.mean(( torch.cumsum(y_pred, dim=1)- y_true)** 2)
 return loss
n_holdout = 0
n_train = None
train_data, valid_data, scaler, scaler_meta = load_data_nn(None)
 n_inp = train_data.players.shape[2]
nemb = 128
nhead = 1
nhid = 96
nlayers = 4
nfinal = 512
lr = 0.0001
gamma = 0.976
dropout_encoder = 0.0
n_emb_layers = 3
dropout_classifier = 0.3
dropout_embed = 0.15
dropout_attn = 0.35
batch_size = 16
epochs = 100
meta_emb = 8
downsample_2017 = 0.4
gauss_noise = 0.15
gauss_xy_noise = 0.1
n_fin_layers = 3
mode = 'ensemble'
log_filename = 'kernel_v92_encoder_0_attn_02'
params = {
'ninp': n_inp,
'nemb': nemb,
'nhead': nhead,
'nhid': nhid,
'nlayers': nlayers,
'nfinal': nfinal,
'ninp_rusher': train_data.rusher.shape[1],
'pre_LN': True,
'dropout_encoder': dropout_encoder,
'dropout_embed': dropout_embed,
'dropout_classifier': dropout_classifier,
'n_emb_layers': n_emb_layers,
'ninp_meta': train_data.meta.shape[1],
'meta_emb': meta_emb,
'gauss_noise': gauss_noise,
'gauss_xy_noise': gauss_xy_noise,
'n_fin_layers': n_fin_layers,
'dropout_attn': dropout_attn
}
no_decay = ['bias', '.norm']
if mode == 'grid':
for dropout_embed in [0.05, 0.1, 0.15]:
for dropout_encoder in [0.2, 0.3, 0.4]:
log_f = f'{log_filename}_embed{dropout_embed}_encoder{dropout_encoder}.csv'
params['dropout_encoder'] = dropout_encoder
params['dropout_embed'] = dropout_embed
model = TransformerModel(**params)
 f = open(log_f, 'w+', newline='')
 writer = csv.writer(f)
 writer.writerow([dropout_embed, dropout_classifier])
 writer.writerow(['epoch', 'lr', 'train_loss', 'valid_loss'])
 optimizer = torch.optim.Adam(model.parameters() , lr=lr)
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=gamma)
 model = train_model(model, scheduler, batch_size, train_data, valid_data, writer, epochs, downsample_2017, params=params)
 elif mode == 'ensemble':
models = EnsembleModel()
game_sets = [
{1, 3, 5, 7, 9},
{2, 4, 6, 8, 10}
]
for i in range(len(game_sets)) :
model = TransformerModel(**params)
 train_data, valid_data, scaler, scaler_meta = load_data_nn(None, nfolds=8, nidx=i)
 optimizer = torch.optim.Adam(model.parameters() , lr=lr)
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=gamma)
 model = train_model(model, scheduler, batch_size, train_data, valid_data, None, epochs, downsample_2017,
calc_train_loss=False, params=params)
 models.add_model(model)
 model = models
else:
assert mode == 'single'
model = TransformerModel(**params)
 f = open(log_filename + '.csv', 'w+', newline='')
 writer = csv.writer(f)
 writer.writerow(train_data.player_cols)
 writer.writerow(train_data.rusher_cols)
 writer.writerow(['epoch', 'lr', 'train_loss', 'valid_loss'])
 optimizer = torch.optim.Adam(model.parameters() , lr=lr)
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=gamma)
 model = train_model(model, scheduler, batch_size, train_data, valid_data, writer, epochs, downsample_2017, params=params )<feature_engineering>
pd.set_option('display.max_columns', 100)
 train.head()
def tta(test_df, sigma_dir=1.0, sigma_y=1.0):
n_aug = 10
test_df_aug = pd.concat([test_df]*n_aug)
 test_df_aug['Dir'] += np.random.normal(0, sigma_dir, size=len(test_df_aug))
test_df_aug['Y'] += np.repeat(np.random.normal(0, sigma_y, size=n_aug), 22)
 test_df_aug['PlayId'] += np.repeat(np.arange(0, n_aug), 22)
 return test_df_aug<set_options>
f = open('.. /input/house-prices-advanced-regression-techniques/data_description.txt','r')
 message = f.read()
print(message )
env = nflrush.make_env()<predict_on_test>
NA_col = pd.DataFrame(train.isna().sum() , columns = ['NA_Count'])
 NA_col['% of NA'] =(NA_col.NA_Count/len(train)) *100
NA_col.sort_values(by = ['% of NA'], ascending = False, na_position = 'first')
model.eval()
n_prev = 0
original_data = concat_dataset(train_data, valid_data)
 for(test_df, sample_prediction_df)in env.iter_test() :
try:
test_df = tta(test_df)
 X_test = load_data_nn_test(test_df, scaler, scaler_meta)
 except Exception as e:
print(f'
env.predict(sample_prediction_df)
 continue
with torch.no_grad() :
predicted = model(X_test ).mean(axis=0)
 sample_prediction_df.iloc[0,:] = np.squeeze(predicted)
 env.predict(sample_prediction_df)
<save_to_csv>
pd.set_option('display.max_rows', 500)
 NA_row = pd.DataFrame(train.isnull().sum(axis = 1), columns = ['NA_Row_Count'])
 NA_row['% of NA'] =(NA_row.NA_Row_Count/len(train.columns)) *100
NA_row.sort_values(by = ['% of NA'], ascending = False, na_position = 'first')
env.write_submission_file()<set_options>
NA_col = pd.DataFrame(test.isna().sum() , columns = ['NA_Count'])
 NA_col['% of NA'] =(NA_col.NA_Count/len(test)) *100
NA_col.sort_values(by = ['% of NA'], ascending = False, na_position = 'first')
torch.manual_seed(0)
 np.random.seed(0)
 Rand_matrix = scipy.special.logit(np.random.rand(22,22))
class Data(object):
def __init__(self, players: torch.Tensor, rusher: torch.Tensor, meta: torch.Tensor, y: Optional[torch.Tensor],
yardLine: np.ndarray, year: np.ndarray, player_cols: List[str], rusher_cols: List[str],
meta_cols: List[str]):
self.players = players
self.rusher = rusher
self.meta = meta
self.y = y
self.yardLine = yardLine
self.year = year
self.player_cols = player_cols
self.rusher_cols = rusher_cols
self.meta_cols = meta_cols
assert self.players.size(0)== self.rusher.size(0)
 if yardLine is not None:
assert len(yardLine)== self.players.size(0)
 def len(self):
return self.players.size(0)
 def y_soft(self, sigma: float = 1.0):
return torch.from_numpy(gaussian_filter1d(self.y.numpy() , sigma=sigma))
def slice(self, begin: int, end: int)-> 'Data':
p = self.players[begin:end] if self.players is not None else None
r = self.rusher[begin:end] if self.rusher is not None else None
m = self.meta[begin:end] if self.meta is not None else None
y = self.y[begin:end] if self.y is not None else None
yd = self.yardLine[begin:end].copy() if self.yardLine is not None else None
yr = self.year[begin:end].copy() if self.year is not None else None
return Data(p, r, m, y, yd, yr, self.player_cols, self.rusher_cols, self.meta_cols)
 def _sample_by_mask(self, mask):
mask_tensor = torch.from_numpy(mask)
 p = self.players[mask_tensor] if self.players is not None else None
r = self.rusher[mask_tensor] if self.rusher is not None else None
m = self.meta[mask_tensor] if self.meta is not None else None
y = self.y[mask_tensor] if self.y is not None else None
yd = self.yardLine[mask].copy() if self.yardLine is not None else None
yr = self.year[mask].copy() if self.year is not None else None
return Data(p, r, m, y, yd, yr, self.player_cols, self.rusher_cols, self.meta_cols)
 def downsample_2017(self, dropout_rate: float):
assert 0 <= dropout_rate <= 1.0
dropout = np.random.choice([True, False], size=len(self.year), p=[1 - dropout_rate, dropout_rate])
 mask =(self.year != 2017)| dropout
return self._sample_by_mask(mask)
 def shuffled(self):
indices = np.random.permutation(self.players.shape[0])
 p = np.take(self.players, indices, axis=0)
 r = np.take(self.rusher, indices, axis=0)if self.rusher is not None else None
m = np.take(self.meta, indices, axis=0)if self.meta is not None else None
y = np.take(self.y, indices, axis=0)if self.y is not None else None
yd = np.take(self.yardLine, indices, axis=0 ).copy() if self.yardLine is not None else None
yr = np.take(self.year, indices, axis=0 ).copy() if self.year is not None else None
return Data(p, r, m, y, yd, yr, self.player_cols, self.rusher_cols, self.meta_cols)
 def random_split(self, p: float):
mask = np.random.choice([True, False], p=[p, 1 - p], size=self.meta.size(0))
d1 = self._sample_by_mask(mask)
 d2 = self._sample_by_mask(~mask)
 return d1, d2
def concat_dataset(l: Data, r: Data):
p = torch.cat(( l.players, r.players))
rs = torch.cat(( l.rusher, r.rusher))
m = torch.cat(( l.meta, r.meta))
y = torch.cat(( l.y, r.y)) if l.y is not None else None
yd = np.concatenate([l.yardLine, r.yardLine])if l.yardLine is not None else None
yr = np.concatenate([l.year, r.year])if l.year is not None else None
print(p.size())
 print(yd.shape)
 return Data(p, rs, m, y, yd, yr, l.player_cols, l.rusher_cols, l.meta_cols)
 def _get_clones(module, N):
return ModuleList([copy.deepcopy(module)for i in range(N)])
 class MultiheadAttention(Module):
__constants__ = ['q_proj_weight', 'k_proj_weight', 'v_proj_weight', 'in_proj_weight', 'R_proj_weight']
def __init__(self, embed_dim, num_heads, dropout=0., bias=True, add_bias_kv=False, add_zero_attn=False, kdim=None,
vdim=None):
super(MultiheadAttention, self ).__init__()
self.embed_dim = embed_dim
self.kdim = kdim if kdim is not None else embed_dim
self.vdim = vdim if vdim is not None else embed_dim
self._qkv_same_embed_dim = self.kdim == embed_dim and self.vdim == embed_dim
self.num_heads = num_heads
self.dropout = dropout
self.head_dim = embed_dim // num_heads
assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads"
if self._qkv_same_embed_dim is False:
self.q_proj_weight = Parameter(torch.Tensor(embed_dim, embed_dim))
self.k_proj_weight = Parameter(torch.Tensor(embed_dim, self.kdim))
self.v_proj_weight = Parameter(torch.Tensor(embed_dim, self.vdim))
self.register_parameter('in_proj_weight', None)
 else:
self.in_proj_weight = Parameter(torch.empty(3 * embed_dim, embed_dim))
self.register_parameter('q_proj_weight', None)
 self.register_parameter('k_proj_weight', None)
 self.register_parameter('v_proj_weight', None)
 self.R_proj_weight = Parameter(torch.Tensor(22, 22))
self.alpha_1 = Parameter(torch.Tensor(1))
if bias:
self.in_proj_bias = Parameter(torch.empty(3 * embed_dim))
else:
self.register_parameter('in_proj_bias', None)
 self.out_proj = Linear(embed_dim, embed_dim, bias=bias)
 if add_bias_kv:
self.bias_k = Parameter(torch.empty(1, 1, embed_dim))
self.bias_v = Parameter(torch.empty(1, 1, embed_dim))
else:
self.bias_k = self.bias_v = None
self.add_zero_attn = add_zero_attn
self._reset_parameters()
def _reset_parameters(self):
if self._qkv_same_embed_dim:
xavier_uniform_(self.in_proj_weight)
 xavier_normal_(self.R_proj_weight)
 else:
xavier_uniform_(self.q_proj_weight)
 xavier_uniform_(self.k_proj_weight)
 xavier_uniform_(self.v_proj_weight)
 if self.in_proj_bias is not None:
constant_(self.in_proj_bias, 0.)
constant_(self.out_proj.bias, 0.)
if self.bias_k is not None:
xavier_normal_(self.bias_k)
 if self.bias_v is not None:
xavier_normal_(self.bias_v)
 def forward(self, query, key, value, key_padding_mask=None,
need_weights=True, attn_mask=None):
if not self._qkv_same_embed_dim:
return multi_head_attention_forward(
 query, key, value, self.embed_dim, self.num_heads,
self.in_proj_weight, self.in_proj_bias,
self.bias_k, self.bias_v, self.add_zero_attn,
self.dropout, self.out_proj.weight, self.out_proj.bias,
training=self.training,
key_padding_mask=key_padding_mask, need_weights=need_weights,
attn_mask=attn_mask, use_separate_proj_weight=True,
q_proj_weight=self.q_proj_weight, k_proj_weight=self.k_proj_weight,
v_proj_weight=self.v_proj_weight, R_proj_weight=self.R_proj_weight,alpha_1=self.alpha_1)
 else:
return multi_head_attention_forward(
 query, key, value, self.embed_dim, self.num_heads,
self.in_proj_weight, self.in_proj_bias,
self.bias_k, self.bias_v, self.add_zero_attn,
self.dropout, self.out_proj.weight, self.out_proj.bias,
training=self.training,
key_padding_mask=key_padding_mask, need_weights=need_weights,
attn_mask=attn_mask, R_proj_weight=self.R_proj_weight,alpha_1=self.alpha_1)
 class TransformerEncoder(Module):
def __init__(self, encoder_layer, num_layers, norm=None):
super(TransformerEncoder, self ).__init__()
self.layers = _get_clones(encoder_layer, num_layers)
 self.num_layers = num_layers
self.norm = norm
def forward(self, src, mask=None, src_key_padding_mask=None):
r
output = src
for i in range(self.num_layers):
output = self.layers[i](output, src_mask=mask,
src_key_padding_mask=src_key_padding_mask)
 if self.norm:
output = self.norm(output)
 return output
class TransformerEncoderLayer(Module):
def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, dropout_attn=0.1, pre_LN=False):
super(TransformerEncoderLayer, self ).__init__()
self.self_attn = MultiheadAttention(d_model, nhead, dropout=dropout_attn)
 self.linear1 = Linear(d_model, dim_feedforward)
 self.dropout = Dropout(dropout)
 self.linear2 = Linear(dim_feedforward, d_model)
 self.norm1 = LayerNorm(d_model)
 self.norm2 = LayerNorm(d_model)
 self.dropout1 = Dropout(dropout)
 self.dropout2 = Dropout(dropout)
 self.activation = F.relu
self.pre_LN = pre_LN
def forward(self, src, src_mask=None, src_key_padding_mask=None):
r
if self.pre_LN:
src2 = self.norm1(src)
 src2 = self.self_attn(src2, src2, src2, attn_mask=src_mask,
key_padding_mask=src_key_padding_mask)[0]
src = src + self.dropout1(src2)
 else:
src2 = self.self_attn(src, src, src, attn_mask=src_mask,
key_padding_mask=src_key_padding_mask)[0]
src = src + self.dropout1(src2)
 src = self.norm1(src)
 if hasattr(self, "activation"):
src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))
else:
src2 = self.linear2(self.dropout(F.relu(self.linear1(src))))
src = src + self.dropout2(src2)
 src = self.norm2(src)
 return src
def multi_head_attention_forward(query,
key,
value,
embed_dim_to_check,
num_heads,
in_proj_weight,
in_proj_bias,
bias_k,
bias_v,
add_zero_attn,
dropout_p,
out_proj_weight,
out_proj_bias,
training=True,
key_padding_mask=None,
need_weights=True,
attn_mask=None,
use_separate_proj_weight=False,
q_proj_weight=None,
k_proj_weight=None,
v_proj_weight=None,
R_proj_weight=None,
static_k=None,
static_v=None,
alpha_1=None 
):
r
qkv_same = torch.equal(query, key)and torch.equal(key, value)
 kv_same = torch.equal(key, value)
 tgt_len, bsz, embed_dim = query.size()
assert embed_dim == embed_dim_to_check
assert list(query.size())== [tgt_len, bsz, embed_dim]
assert key.size() == value.size()
head_dim = embed_dim // num_heads
assert head_dim * num_heads == embed_dim, "embed_dim must be divisible by num_heads"
scaling = float(head_dim)** -0.5
if use_separate_proj_weight is not True:
if qkv_same:
q, k, v = F.linear(query, in_proj_weight, in_proj_bias ).chunk(3, dim=-1)
 elif kv_same:
_b = in_proj_bias
_start = 0
_end = embed_dim
_w = in_proj_weight[_start:_end, :]
if _b is not None:
_b = _b[_start:_end]
q = F.linear(query, _w, _b)
 if key is None:
assert value is None
k = None
v = None
else:
_b = in_proj_bias
_start = embed_dim
_end = None
_w = in_proj_weight[_start:, :]
if _b is not None:
_b = _b[_start:]
k, v = F.linear(key, _w, _b ).chunk(2, dim=-1)
 else:
_b = in_proj_bias
_start = 0
_end = embed_dim
_w = in_proj_weight[_start:_end, :]
if _b is not None:
_b = _b[_start:_end]
q = F.linear(query, _w, _b)
 _b = in_proj_bias
_start = embed_dim
_end = embed_dim * 2
_w = in_proj_weight[_start:_end, :]
if _b is not None:
_b = _b[_start:_end]
k = F.linear(key, _w, _b)
 _b = in_proj_bias
_start = embed_dim * 2
_end = None
_w = in_proj_weight[_start:, :]
if _b is not None:
_b = _b[_start:]
v = F.linear(value, _w, _b)
 else:
q_proj_weight_non_opt = torch.jit._unwrap_optional(q_proj_weight)
 len1, len2 = q_proj_weight_non_opt.size()
assert len1 == embed_dim and len2 == query.size(-1)
 k_proj_weight_non_opt = torch.jit._unwrap_optional(k_proj_weight)
 len1, len2 = k_proj_weight_non_opt.size()
assert len1 == embed_dim and len2 == key.size(-1)
 v_proj_weight_non_opt = torch.jit._unwrap_optional(v_proj_weight)
 len1, len2 = v_proj_weight_non_opt.size()
assert len1 == embed_dim and len2 == value.size(-1)
 if in_proj_bias is not None:
q = F.linear(query, q_proj_weight_non_opt, in_proj_bias[0:embed_dim])
 k = F.linear(key, k_proj_weight_non_opt, in_proj_bias[embed_dim:(embed_dim * 2)])
 v = F.linear(value, v_proj_weight_non_opt, in_proj_bias[(embed_dim * 2):])
 else:
q = F.linear(query, q_proj_weight_non_opt, in_proj_bias)
 k = F.linear(key, k_proj_weight_non_opt, in_proj_bias)
 v = F.linear(value, v_proj_weight_non_opt, in_proj_bias)
 q = q * scaling
R_proj_weight_non_opt = torch.jit._unwrap_optional(R_proj_weight)
 len1, len2 = R_proj_weight_non_opt.size()
if bias_k is not None and bias_v is not None:
if static_k is None and static_v is None:
k = torch.cat([k, bias_k.repeat(1, bsz, 1)])
 v = torch.cat([v, bias_v.repeat(1, bsz, 1)])
 if attn_mask is not None:
attn_mask = torch.cat([attn_mask,
torch.zeros(( attn_mask.size(0), 1),
dtype=attn_mask.dtype,
device=attn_mask.device)], dim=1)
 if key_padding_mask is not None:
key_padding_mask = torch.cat(
 [key_padding_mask, torch.zeros(( key_padding_mask.size(0), 1),
dtype=key_padding_mask.dtype,
device=key_padding_mask.device)], dim=1)
 else:
assert static_k is None, "bias cannot be added to static key."
assert static_v is None, "bias cannot be added to static value."
else:
assert bias_k is None
assert bias_v is None
q = q.contiguous().view(tgt_len, bsz * num_heads, head_dim ).transpose(0, 1)
 if k is not None:
k = k.contiguous().view(-1, bsz * num_heads, head_dim ).transpose(0, 1)
 if v is not None:
v = v.contiguous().view(-1, bsz * num_heads, head_dim ).transpose(0, 1)
 if static_k is not None:
assert static_k.size(0)== bsz * num_heads
assert static_k.size(2)== head_dim
k = static_k
if static_v is not None:
assert static_v.size(0)== bsz * num_heads
assert static_v.size(2)== head_dim
v = static_v
src_len = k.size(1)
 if key_padding_mask is not None:
assert key_padding_mask.size(0)== bsz
assert key_padding_mask.size(1)== src_len
if add_zero_attn:
src_len += 1
k = torch.cat([k, torch.zeros(( k.size(0), 1)+ k.size() [2:], dtype=k.dtype, device=k.device)], dim=1)
 v = torch.cat([v, torch.zeros(( v.size(0), 1)+ v.size() [2:], dtype=v.dtype, device=v.device)], dim=1)
 if attn_mask is not None:
attn_mask = torch.cat([attn_mask, torch.zeros(( attn_mask.size(0), 1),
dtype=attn_mask.dtype,
device=attn_mask.device)], dim=1)
 if key_padding_mask is not None:
key_padding_mask = torch.cat(
 [key_padding_mask, torch.zeros(( key_padding_mask.size(0), 1),
dtype=key_padding_mask.dtype,
device=key_padding_mask.device)], dim=1)
 temp_matrix = np.tile(R_proj_weight_non_opt.detach().numpy() ,(bsz * num_heads,1,1))
attn_output_weights = torch.Tensor(temp_matrix)
 assert list(attn_output_weights.size())== [bsz * num_heads, tgt_len, src_len]
if attn_mask is not None:
attn_mask = attn_mask.unsqueeze(0)
 attn_output_weights += attn_mask
if key_padding_mask is not None:
attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
 attn_output_weights = attn_output_weights.masked_fill(
 key_padding_mask.unsqueeze(1 ).unsqueeze(2),
float('-inf'), 
)
 attn_output_weights = attn_output_weights.view(bsz * num_heads, tgt_len, src_len)
 attn_output_weights = F.softmax(
 attn_output_weights, dim=-1)
 attn_output_weights = F.dropout(attn_output_weights, p=dropout_p, training=training)
 attn_output = torch.bmm(attn_output_weights, v)
 assert list(attn_output.size())== [bsz * num_heads, tgt_len, head_dim]
attn_output = attn_output.transpose(0, 1 ).contiguous().view(tgt_len, bsz, embed_dim)
 attn_output = F.linear(attn_output, out_proj_weight, out_proj_bias)
 if need_weights:
attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
 return attn_output, attn_output_weights.sum(dim=1)/ num_heads
else:
return attn_output, None
class TransformerModel(nn.Module):
def __init__(self, ninp: int, nemb: int = 1, nhead: int = 1, nhid: int = 32, nlayers: int = 4, nfinal: int = 1024,
dropout_encoder: float = 0.1, dropout_embed: float = 0.0, dropout_classifier: float = 0.0,
n_class: int = 199, ninp_rusher: int = 16, pre_LN: bool = False, rusher_emb: int = 32,
n_emb_layers: int = 2,
ninp_meta: int = 8, meta_emb: int = 32, gauss_noise: float = 0.0,
gauss_xy_noise: float = 0.0,
n_fin_layers: int = 3, dropout_attn: float = 0):

 super(TransformerModel, self ).__init__()
self.model_type = 'Transformer'
encoder_layers = TransformerEncoderLayer(nemb, nhead, nhid, dropout_encoder, pre_LN=pre_LN, dropout_attn=dropout_attn)
 self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)
 self.n_emb_layers = n_emb_layers
self.conv1 = nn.Conv1d(in_channels=ninp, out_channels=nemb, kernel_size=1)
 self.conv2 = nn.Conv1d(in_channels=nemb, out_channels=nemb, kernel_size=1)
 self.conv3 = nn.Conv1d(in_channels=nemb, out_channels=nemb, kernel_size=1)
 self.relu1 = nn.PReLU()
self.relu2 = nn.PReLU()
self.relu3 = nn.PReLU()
self.relu4 = nn.PReLU()
self.relu5 = nn.ReLU()
if dropout_embed > 0:
self.dropout1 = nn.Dropout(dropout_embed)
 else:
self.dropout1 = None
self.avgpool = nn.AvgPool1d(kernel_size=22)
 assert n_fin_layers == 3
self.linear = nn.Sequential(
 nn.Dropout(dropout_classifier),
nn.Linear(nemb + rusher_emb + meta_emb, nfinal),
nn.ReLU() ,
nn.Dropout(dropout_classifier),
nn.Linear(nfinal, nfinal),
nn.ReLU() ,
nn.Dropout(dropout_classifier),
nn.Linear(nfinal, n_class)
)
 self.activation = nn.Softmax(dim=1)
 self.linear2 = nn.Linear(ninp_rusher, rusher_emb)
 self.gauss_noise = gauss_noise
self.gauss_xy_noise = gauss_xy_noise
if meta_emb > 0:
self.linear3 = nn.Linear(ninp_meta, meta_emb)
 else:
self.linear3 = None
def forward(self, x: Data, with_clip: bool = True):
src = x.players
src_rusher = x.rusher
src = src.permute([0, 2, 1])
 if self.training:
if self.gauss_noise > 0.0:
noise = torch.randn_like(src)* self.gauss_noise
src = src + noise
if self.gauss_xy_noise > 0.0:
dy = torch.randn(src.size(0)) * self.gauss_xy_noise
src[:, 1, :] += dy.reshape(src.size(0), 1 ).expand(src.size(0), src.size(2))
src_rusher[:, 1] += dy
src = self.relu1(self.conv1(src))
src = self.dropout1(src)
 src = self.relu2(self.conv2(src))
src = self.dropout1(src)
 src = self.relu3(self.conv3(src))
src = src.permute([2, 0, 1])
 output = self.transformer_encoder(src)
 output = output.permute([1, 2, 0])
 output = torch.squeeze(self.avgpool(output), dim=2)
 if self.linear3 is not None:
output = torch.cat(( output, self.relu4(self.linear2(src_rusher)) , self.relu5(self.linear3(x.meta))), dim=1)
 else:
output = torch.cat(( output, self.relu4(self.linear2(src_rusher))), dim=1)
 output = self.linear(output)
 output = self.activation(output)
 if not self.training and x.yardLine is not None:
output = torch.cumsum(output, dim=1 ).numpy()
output = np.clip(output, 0.0, 1.0)
 if with_clip:
left = 99 - x.yardLine
right = 199 - x.yardLine
for k in range(len(output)) :
output[k, :left[k] + 1] = 0.0
output[k, right[k]:] = 1.0
return output
def prep(df: pd.DataFrame)-> Tuple[pd.DataFrame, pd.DataFrame]:
df['ToLeft'] = df.PlayDirection == "left"
df['IsBallCarrier'] = df.NflId == df.NflIdRusher
df.loc[df.VisitorTeamAbbr == "ARI", 'VisitorTeamAbbr'] = "ARZ"
df.loc[df.HomeTeamAbbr == "ARI", 'HomeTeamAbbr'] = "ARZ"
df.loc[df.VisitorTeamAbbr == "BAL", 'VisitorTeamAbbr'] = "BLT"
df.loc[df.HomeTeamAbbr == "BAL", 'HomeTeamAbbr'] = "BLT"
df.loc[df.VisitorTeamAbbr == "CLE", 'VisitorTeamAbbr'] = "CLV"
df.loc[df.HomeTeamAbbr == "CLE", 'HomeTeamAbbr'] = "CLV"
df.loc[df.VisitorTeamAbbr == "HOU", 'VisitorTeamAbbr'] = "HST"
df.loc[df.HomeTeamAbbr == "HOU", 'HomeTeamAbbr'] = "HST"
df['TeamOnOffense'] = "home"
df.loc[df.PossessionTeam != df.HomeTeamAbbr, 'TeamOnOffense'] = "away"
df['IsOnOffense'] = df.Team == df.TeamOnOffense
df['YardLine_std'] = 100 - df.YardLine
df.loc[df.FieldPosition.fillna('')== df.PossessionTeam, 'YardLine_std'] = df.loc[
df.FieldPosition.fillna('')== df.PossessionTeam, 'YardLine']
df['X_std'] = df.X
df.loc[df.ToLeft, 'X_std'] = 120 - df.loc[df.ToLeft, 'X']
df['Y_std'] = df.Y
df.loc[df.ToLeft, 'Y_std'] = 160 / 3 - df.loc[df.ToLeft, 'Y']
df['Orientation_std'] = df.Orientation
df.loc[df.ToLeft, 'Orientation_std'] = np.mod(180 + df.loc[df.ToLeft, 'Orientation_std'], 360)
 df['Dir_std'] = df.Dir
df.loc[df.ToLeft, 'Dir_std'] = np.mod(180 + df.loc[df.ToLeft, 'Dir_std'], 360)
 df['IsOffence'] = df['Team'] == df['TeamOnOffense']
df['OffenceScoreBeforePlay'] = df['HomeScoreBeforePlay']
df.loc[df.TeamOnOffense == "away", 'OffenceScoreBeforePlay'] = df.loc[
df.TeamOnOffense == "away", 'VisitorScoreBeforePlay']
df['DefenseScoreBeforePlay'] = df['VisitorScoreBeforePlay']
df.loc[df.TeamOnOffense == "away", 'DefenseScoreBeforePlay'] = df.loc[
df.TeamOnOffense == "away", 'HomeScoreBeforePlay']
df['OffenceTeamAbbr'] = df['HomeTeamAbbr']
df.loc[df.TeamOnOffense == "away", 'OffenceTeamAbbr'] = df.loc[df.TeamOnOffense == "away", 'VisitorTeamAbbr']
df['DefenseTeamAbbr'] = df['VisitorTeamAbbr']
df.loc[df.TeamOnOffense == "away", 'DefenseTeamAbbr'] = df.loc[df.TeamOnOffense == "away", 'HomeTeamAbbr']
df['Year'] = pd.to_datetime(df.TimeSnap ).dt.year
df.loc[df['Year'] == 2017, 'Orientation_std'] = np.mod(90 + df.loc[df['Year'] == 2017, 'Orientation_std'], 360)
 player_features = ['X_std', 'Y_std', 'S', 'A', 'Dis', 'Orientation_std', 'Dir_std', 'NflId', 'JerseyNumber',
'PlayerHeight',
'PlayerWeight', 'PlayerBirthDate', 'PlayerCollegeName', 'Position', 'IsBallCarrier', 'IsOffence']
play_features = ['YardLine_std', 'Quarter', 'GameClock', 'PossessionTeam', 'Down', 'Distance', 'FieldPosition',
'OffenceScoreBeforePlay', 'DefenseScoreBeforePlay', 'OffenseFormation', 'OffensePersonnel',
'DefendersInTheBox', 'DefensePersonnel', 'TimeHandoff', 'TimeSnap',
'OffenceTeamAbbr', 'DefenseTeamAbbr', 'Week', 'Stadium', 'Location',
'Turf', 'GameWeather', 'Temperature', 'Humidity', 'WindSpeed', 'WindDirection',
'TeamOnOffense', 'HomeTeamAbbr', 'Year']
if 'Yards' in df:
play_features.append('Yards')
 players = df[['PlayId', 'GameId'] + player_features].copy()
play = df[['GameId', 'PlayId'] + play_features].copy().drop_duplicates(subset=['PlayId'])
 return play, players
def prep_players_nn(play, players, scaler=None, scaler_meta=None):
p = players.drop(['NflId', 'JerseyNumber', 'PlayerHeight', 'PlayerBirthDate', 'PlayerCollegeName', 'GameId'],
axis=1 ).copy()
if scaler is None:
is_training = True
else:
assert scaler_meta is not None
is_training = False
if 'YardLine_std' not in p:
p = pd.merge(p, play[['YardLine_std', 'PlayId']], on='PlayId', how='left')
 p['IsBallCarrier'] = p['IsBallCarrier'].astype(int)
 p['IsOffence'] = p['IsOffence'].astype(int)
 p['X_std'] -= p['YardLine_std']
p['Dir_cos'] = np.cos(np.deg2rad(90 - p['Dir_std']))
p['Dir_sin'] = np.sin(np.deg2rad(90 - p['Dir_std']))
p = p.fillna(0)
 rb = p[p['IsBallCarrier'] == 1][['X_std', 'Y_std', 'PlayId', 'S', 'Dir_sin', 'Dir_cos', 'Dir_std']]
rb.columns = ['X_', 'Y_', 'PlayId', 'S_', 'Dir_sin_', 'Dir_cos_', 'Dir_std_']
p = pd.merge(p, rb, how='left')
 p['DX'] = p['X_std'] - p['X_']
p['DY'] = p['Y_std'] - p['Y_']
angles = 90.0 - np.rad2deg(np.arctan2(p['DY'], p['DX']))
p['AngleFromRB'] = angles - p['Dir_std_']
p['AngleFromRB'] = np.mod(p['AngleFromRB'] + 360, 360)
 p.loc[p['AngleFromRB'] > 180, 'AngleFromRB'] -= 360
dt = 0.001
p['DX2'] =(p['X_std'] + dt * p['Dir_cos'] * p['S'])-(p['X_'] + dt * p['Dir_cos_'] * p['S_'])
 p['DY2'] =(p['Y_std'] + dt * p['Dir_sin'] * p['S'])-(p['Y_'] + dt * p['Dir_sin_'] * p['S_'])
 angles = 90.0 - np.rad2deg(np.arctan2(p['DY2'], p['DX2']))
p['AngleFromRB2'] = angles - p['Dir_std_']
p['AngleFromRB2'] = np.mod(p['AngleFromRB2'] + 360, 360)
 p.loc[p['AngleFromRB2'] > 180, 'AngleFromRB2'] -= 360
p['AngleFromRB2'] = p['AngleFromRB2'] - p['AngleFromRB']
p.loc[p['AngleFromRB2'] > 180, 'AngleFromRB2'] -= 360
p.loc[p['AngleFromRB2'] < -180, 'AngleFromRB2'] += 360
p.loc[p['IsBallCarrier'] == 1, 'AngleFromRB'] = 0
p.loc[p['IsBallCarrier'] == 1, 'AngleFromRB2'] = 0
p.drop(['DX2', 'DY2'], axis=1, inplace=True)
 p['AngleTan'] = np.arctan2(p['DY'], p['DX'])
 p = p.replace([np.inf, -np.inf], np.nan)
 p = p.fillna(0)
 p.drop(['X_', 'Y_', 'Dir_sin_', 'Dir_cos_', 'S_', 'Dir_std_', 'Orientation_std', 'Dir_std'], axis=1, inplace=True)
 concat = p.drop(['Position', 'PlayId', 'YardLine_std'], axis=1)
 if is_training:
scaler = StandardScaler()
scaled = scaler.fit_transform(concat.values)
 else:
scaled = scaler.transform(concat.values)
 df = pd.DataFrame(scaled, columns=concat.columns)
 df_rusher = df[p['IsBallCarrier'] == 1].reset_index(drop=True)
 drop_cols = ['IsBallCarrier', 'IsOffence', 'OffenseDist0', 'PlayerWeight', 'DX', 'DY', 'AngleFromRB', 'AngleToRB',
'DistDelta', 'AngleTan', 'AngleFromRB2', 'delaunay_adj']
drop_cols = [c for c in drop_cols if c in df_rusher.columns]
df_rusher.drop(drop_cols, axis=1, inplace=True)
 df_rusher = df_rusher.set_index(players['PlayId'].iloc[np.arange(0, len(players), 22)])
 meta = play[['YardLine_std', 'Distance']].copy()
meta = meta.replace([np.inf, -np.inf], np.nan)
 meta = meta.fillna(0)
 if scaler_meta is not None:
scaled_meta = scaler_meta.transform(meta.values)
 else:
scaler_meta = StandardScaler()
scaled_meta = scaler_meta.fit_transform(meta.values)
 scaled_meta = pd.DataFrame(scaled_meta, columns=meta.columns)
 return df.set_index(players['PlayId']), df_rusher, scaled_meta, scaler, scaler_meta
class EnsembleModel(object):
def __init__(self):
self.models = []
def add_model(self, model):
self.models.append(model)
 def train(self):
for m in self.models:
m.train()
def eval(self):
for m in self.models:
m.eval()
def __call__(self, *input, **kwargs):
assert len(self.models)>= 1
base = self.models[0](*input, **kwargs)
 for m in self.models[1:]:
base += m(*input, **kwargs)
 return base / len(self.models)
 class SnapShot(object):
def __init__(self, model: nn.Module, epoch: int, loss: float):
self.state = copy.deepcopy(model.state_dict())
 self.epoch = epoch
self.loss = loss
torch.save(self.state, f'snapshot_{epoch}_{loss}')
 class SnapShots(object):
def __init__(self, interval: int = 3, torelance: float = 1.01, verbose: bool = True):
self.best_model = None
self.snap_shots = []
self.best_val_loss = 1.0
self.interval = interval
self.torelance = torelance
self.verbose = verbose
def add(self, model: nn.Module, epoch: int, loss: float):
if loss < self.best_val_loss:
if self.verbose:
print(f'best model is updated.epoch{epoch}, loss={loss:.7f}')
 self.best_model = SnapShot(model, epoch, loss)
 self.best_val_loss = loss
if epoch % self.interval == 0:
if self.verbose:
print(f'Add snapshot.epoch{epoch}, loss={loss:.7f}')
 self.snap_shots.append(SnapShot(model, epoch, loss))
def load_best_single_model(self, model: nn.Module):
if self.best_model is not None:
model.load_state_dict(self.best_model.state)
 return model
def load_ensemble_model(self, cls: Type, params: Dict, max_models: int = 5):
model = EnsembleModel()
best = cls(**params)
 self.load_best_single_model(best)
 model.add_model(best)
 candidates = []
for i, s in enumerate(self.snap_shots):
if s.loss > self.torelance * self.best_model.loss:
continue
if s.epoch == self.best_model.epoch:
continue
candidates.append(( i, s.loss))
candidates = sorted(candidates, key = lambda x: x[1])
 if len(candidates)> max_models - 1:
candidates = candidates[:max_models - 1]
for i, _ in candidates:
s = self.snap_shots[i]
sub = cls(**params)
 sub.load_state_dict(s.state)
 model.add_model(sub)
 if self.verbose:
print(f'add {s.epoch}-th epoch to ensemble(loss:{s.loss:.7f})')
 return model
def train_model(model: nn.Module, scheduler, batch_size: int,
train_data: Data, valid_data: Data, writer, epochs: int,
downsample_2017: float,
calc_train_loss: bool = True,
params: Dict = None):
np.random.seed(0)
 std_batch_length = 12000 // batch_size + 1
n_total_batch = 0
epoch = 1
epoch_start_time = time.time()
ensemble = None
model.eval()
snapshots = SnapShots(torelance=1.007, interval=3)
 snapshots.add(model, 0, evaluate(model, valid_data))
model.train()
while epoch < epochs:
data = train_data.shuffled()
if downsample_2017 > 0.0:
data = data.downsample_2017(downsample_2017)
 for i in range(0, data.len() - 1, batch_size):
iend = min(i + batch_size, data.len())
 batch_data = data.slice(i, iend)
 optimizer.zero_grad()
output = model(batch_data)
 loss = crps(output, batch_data.y)
 loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 0.5)
 optimizer.step()
n_total_batch += 1
if n_total_batch > std_batch_length:
model.eval()
if calc_train_loss and epoch % 10 == 0:
train_loss = evaluate(model, data)
 else:
train_loss = -1
if epoch % 4 == 0 or epoch > 30:
val_loss = evaluate(model, valid_data)
 snapshots.add(model, epoch, val_loss)
 else:
val_loss = -1
print('| end of epoch {:3d} | lr: {:2.5f} | time: {:5.2f}s | train loss {:5.6f} | '
'valid loss {:5.9f}'.format(epoch, scheduler.get_lr() [0],(time.time() - epoch_start_time),
train_loss, val_loss))
scheduler.step()
if writer is not None:
writer.writerow([epoch, scheduler.get_lr() [0], train_loss, val_loss])
 f.flush()
n_total_batch = 0
epoch += 1
epoch_start_time = time.time()
model.train()
if epoch in [40]:
batch_size *= 2
std_batch_length = 12000 // batch_size + 1
model = snapshots.load_best_single_model(model)
 eval_single = evaluate(model, valid_data)
 try:
ensemble = snapshots.load_ensemble_model(TransformerModel, params)
 eval_ensemble = evaluate(ensemble, valid_data)
 print(f'final loss: {eval_single:.8f}(single)/ {eval_ensemble:.8f}({len(ensemble.models)} models)')
 except:
pass
return ensemble if ensemble is not None else model
def evaluate(eval_model: nn.Module, data: Data, with_clip: bool = True):
eval_model.eval()
assert data.y is not None
with torch.no_grad() :
y_actual = data.y.numpy()
y_predicted = eval_model(data, with_clip)
 loss = np.sum(( y_predicted - y_actual)** 2)/(199 * len(y_predicted))
return loss
def return_delta(x):
temp = np.zeros(199)
 temp[x + 99:] = 1
return temp
def NFL_validation_split(df: pd.DataFrame, game_set=None):
games = df[['GameId', 'PossessionTeam']].drop_duplicates()
games = games.sort_values(['PossessionTeam', 'GameId'], ascending=[True, False])
 games['row_number'] = games.groupby(['PossessionTeam'] ).cumcount() + 1
game_set = game_set or {1, 2, 3, 4, 5}
game_ids = set(games[games['row_number'].isin(game_set)]['GameId'].unique().tolist())
 return game_ids
def NFL_group_split(df: pd.DataFrame, nfolds: int, nidx: int):
kf = GroupKFold(nfolds)
 train_idx, valid_idx = list(kf.split(df, groups=df['GameId'])) [nidx]
return set(df['GameId'].iloc[valid_idx].unique())
 def load_data_nn_test(test_df: pd.DataFrame, scaler: StandardScaler, scaler_meta: StandardScaler)-> Data:
play, players = prep(test_df)
 players_nn, rusher_nn, meta_nn, _, _ = prep_players_nn(play, players, scaler, scaler_meta)
 X = players_nn.values.astype(np.float32 ).reshape(( -1, 22, len(players_nn.columns)))
 Xr = rusher_nn.values.astype(np.float32)
 Xm = meta_nn.values.astype(np.float32)
 year = play['Year'].values.copy()
return Data(torch.from_numpy(X), torch.from_numpy(Xr), torch.from_numpy(Xm),
None, play['YardLine_std'].values.copy() , year,
list(players_nn.columns), list(rusher_nn.columns), list(meta_nn.columns))
def load_data_nn(n_plays_sample=None,
nfolds=None, nidx=None,
skiprows=None,
game_set=None)-> Tuple[Data, Data, StandardScaler, StandardScaler]:
train = pd.read_csv('.. /input/nfl-big-data-bowl-2020/train.csv', nrows=n_plays_sample, skiprows=skiprows)
 play, players = prep(train)
 players_nn, rusher_nn, meta_nn, scaler, scaler_meta = prep_players_nn(play, players, None, None)
 assert len(players_nn)== len(rusher_nn)* 22
assert len(rusher_nn)== len(meta_nn)
 assert len(rusher_nn)== len(play)
 play_ids = np.array(players_nn.index[np.arange(0, len(players_nn), 22)])
 game_ids = np.array(players['GameId'].values[np.arange(0, len(players_nn), 22)])
 assert len(play_ids)== len(game_ids)
 if nfolds is not None:
assert nidx is not None
game_ids_valid = NFL_group_split(train, nfolds, nidx)
 else:
game_ids_valid = NFL_validation_split(train, game_set)
 X = players_nn.values.astype(np.float32 ).reshape(( -1, 22, len(players_nn.columns)))
 Xr = rusher_nn.values.astype(np.float32)
 Xm = meta_nn.values.astype(np.float32)
 y = np.vstack(play['Yards'].apply(return_delta ).values ).astype(np.float32)
 yardLine = play['YardLine_std'].values
year = play['Year'].values
valid_mask = np.isin(game_ids, np.array(list(game_ids_valid)))
 X_valid = torch.from_numpy(X[valid_mask])
 Xr_valid = torch.from_numpy(Xr[valid_mask])
 Xm_valid = torch.from_numpy(Xm[valid_mask])
 y_valid = torch.from_numpy(y[valid_mask])
 yd_valid = yardLine[valid_mask].copy()
yr_valid = year[valid_mask].copy()
X_train = torch.from_numpy(X[~valid_mask])
 Xr_train = torch.from_numpy(Xr[~valid_mask])
 Xm_train = torch.from_numpy(Xm[~valid_mask])
 y_train = torch.from_numpy(y[~valid_mask])
 yd_train = yardLine[~valid_mask].copy()
yr_train = year[~valid_mask].copy()
print(f'X_train: {X_train.shape}, {Xr_train.shape}, {Xm_train.shape}')
 print(f'X_valid: {X_valid.shape}, {Xr_valid.shape}, {Xm_valid.shape}')
 print(f'y_train: {y_train.shape}, y_valid: {y_valid.shape}')
 print(f'players: {list(players_nn.columns)}')
 print(f'rusher: {list(rusher_nn.columns)}')
 print(f'meta: {list(meta_nn.columns)}')
 dtrain = Data(X_train, Xr_train, Xm_train, y_train, yd_train, yr_train,
list(players_nn.columns), list(rusher_nn.columns), list(meta_nn.columns))
dvalid = Data(X_valid, Xr_valid, Xm_valid, y_valid, yd_valid, yr_valid,
list(players_nn.columns), list(rusher_nn.columns), list(meta_nn.columns))
assert len(X)== len(y)
 return dtrain, dvalid, scaler, scaler_meta
def crps(y_pred, y_true):
loss = torch.mean(( torch.cumsum(y_pred, dim=1)- y_true)** 2)
 return loss
n_holdout = 0
n_train = None
train_data, valid_data, scaler, scaler_meta = load_data_nn(None)
 n_inp = train_data.players.shape[2]
nemb = 128
nhead = 1
nhid = 96
nlayers = 4
nfinal = 512
lr = 0.0001
gamma = 0.976
dropout_encoder = 0.0
n_emb_layers = 3
dropout_classifier = 0.3
dropout_embed = 0.15
dropout_attn = 0.35
batch_size = 16
epochs = 100
meta_emb = 8
downsample_2017 = 0.4
gauss_noise = 0.15
gauss_xy_noise = 0.1
n_fin_layers = 3
mode = 'ensemble'
log_filename = 'kernel_v92_encoder_0_attn_02'
params = {
'ninp': n_inp,
'nemb': nemb,
'nhead': nhead,
'nhid': nhid,
'nlayers': nlayers,
'nfinal': nfinal,
'ninp_rusher': train_data.rusher.shape[1],
'pre_LN': True,
'dropout_encoder': dropout_encoder,
'dropout_embed': dropout_embed,
'dropout_classifier': dropout_classifier,
'n_emb_layers': n_emb_layers,
'ninp_meta': train_data.meta.shape[1],
'meta_emb': meta_emb,
'gauss_noise': gauss_noise,
'gauss_xy_noise': gauss_xy_noise,
'n_fin_layers': n_fin_layers,
'dropout_attn': dropout_attn
}
no_decay = ['bias', '.norm']
if mode == 'grid':
for dropout_embed in [0.05, 0.1, 0.15]:
for dropout_encoder in [0.2, 0.3, 0.4]:
log_f = f'{log_filename}_embed{dropout_embed}_encoder{dropout_encoder}.csv'
params['dropout_encoder'] = dropout_encoder
params['dropout_embed'] = dropout_embed
model = TransformerModel(**params)
 f = open(log_f, 'w+', newline='')
 writer = csv.writer(f)
 writer.writerow([dropout_embed, dropout_classifier])
 writer.writerow(['epoch', 'lr', 'train_loss', 'valid_loss'])
 optimizer = torch.optim.Adam(model.parameters() , lr=lr)
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=gamma)
 model = train_model(model, scheduler, batch_size, train_data, valid_data, writer, epochs, downsample_2017, params=params)
 elif mode == 'ensemble':
models = EnsembleModel()
game_sets = [
{1, 3, 5, 7, 9},
{2, 4, 6, 8, 10}
]
for i in range(len(game_sets)) :
model = TransformerModel(**params)
 train_data, valid_data, scaler, scaler_meta = load_data_nn(None, nfolds=8, nidx=i)
 optimizer = torch.optim.Adam(model.parameters() , lr=lr)
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=gamma)
 model = train_model(model, scheduler, batch_size, train_data, valid_data, None, epochs, downsample_2017,
calc_train_loss=False, params=params)
 models.add_model(model)
 model = models
else:
assert mode == 'single'
model = TransformerModel(**params)
 f = open(log_filename + '.csv', 'w+', newline='')
 writer = csv.writer(f)
 writer.writerow(train_data.player_cols)
 writer.writerow(train_data.rusher_cols)
 writer.writerow(['epoch', 'lr', 'train_loss', 'valid_loss'])
 optimizer = torch.optim.Adam(model.parameters() , lr=lr)
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=gamma)
 model = train_model(model, scheduler, batch_size, train_data, valid_data, writer, epochs, downsample_2017, params=params )<feature_engineering>
pd.set_option('display.max_rows', 500)
 NA_row = pd.DataFrame(test.isnull().sum(axis = 1), columns = ['NA_Row_Count'])
 NA_row['% of NA'] =(NA_row.NA_Row_Count/len(test.columns)) *100
NA_row.sort_values(by = ['% of NA'], ascending = False, na_position = 'first')
def tta(test_df, sigma_dir=1.0, sigma_y=1.0):
n_aug = 10
test_df_aug = pd.concat([test_df]*n_aug)
 test_df_aug['Dir'] += np.random.normal(0, sigma_dir, size=len(test_df_aug))
test_df_aug['Y'] += np.repeat(np.random.normal(0, sigma_y, size=n_aug), 22)
 test_df_aug['PlayId'] += np.repeat(np.arange(0, n_aug), 22)
 return test_df_aug<set_options>
train_data = train.drop(['PoolQC', 'MiscFeature', 'Alley', 'Fence'], axis=1)
 test_data = test.drop(['PoolQC', 'MiscFeature', 'Alley', 'Fence'], axis=1 )
env = nflrush.make_env()<predict_on_test>
train_wo_target = train_data.drop(['Id', 'SalePrice'], axis=1 )
model.eval()
n_prev = 0
original_data = concat_dataset(train_data, valid_data)
 for(test_df, sample_prediction_df)in env.iter_test() :
try:
test_df = tta(test_df)
 X_test = load_data_nn_test(test_df, scaler, scaler_meta)
 except Exception as e:
print(f'
env.predict(sample_prediction_df)
 continue
with torch.no_grad() :
predicted = model(X_test ).mean(axis=0)
 sample_prediction_df.iloc[0,:] = np.squeeze(predicted)
 env.predict(sample_prediction_df)
<save_to_csv>
cols = train_wo_target.columns
num_cols = train_wo_target._get_numeric_data().columns
cat_cols = list(set(cols)- set(num_cols))
env.write_submission_file()<set_options>
X_train = train_data.copy().drop('SalePrice', axis = 1)
 y_train = train_data[['Id','SalePrice']]
sns.set_style('whitegrid')
 sns.set_context('talk')
 pd.set_option("display.max_columns", 100 )<load_from_csv>
from sklearn.preprocessing import Imputer
from sklearn.impute import SimpleImputer
env = nflrush.make_env()
train = pd.read_csv('.. /input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'})
 len(train )<feature_engineering>
mean_imputer = Imputer(missing_values="NaN", strategy="mean", axis=0)
 mean_imputer.fit(X_train[num_cols])
 X_train[num_cols] = pd.DataFrame(mean_imputer.fit_transform(X_train[num_cols]))
test_data[num_cols] = pd.DataFrame(mean_imputer.fit_transform(test_data[num_cols]))
def split_play_and_player_cols(df,predicting=False):
df['IsRusher'] = df['NflId'] == df['NflIdRusher']
play_ids = df["PlayId"].unique()
df['PlayId'] = df['PlayId'].astype(str)
 player_cols = [
'PlayId',
'Season',
'Team',
'X',
'Y',
'S',
'A',
'Dis',
'Dir',
'NflId',
'IsRusher',
]
df_players = df[player_cols]
play_cols = [
'PlayId',
'Season',
'PossessionTeam',
'HomeTeamAbbr',
'VisitorTeamAbbr',
'PlayDirection',
'FieldPosition',
'YardLine',
]
if not predicting:
play_cols.append('Yards')
 df_play = df[play_cols].copy()
df_play = df_play.groupby('PlayId' ).first().reset_index()
assert df_play.PlayId.nunique() == df.PlayId.nunique() , "Play/player split failed?"
return df_play, df_players<split>
train_objs_num = len(X_train)
 dataset = pd.concat(objs=[X_train, test_data], axis=0)
 dataset_preprocessed = pd.get_dummies(dataset)
 X_train = dataset_preprocessed[:train_objs_num]
test_data = dataset_preprocessed[train_objs_num:]
play_ids = train["PlayId"].unique()
df_play, df_players = split_play_and_player_cols(train )<categorify>
X_train.isna().sum()
def process_team_abbr(df):
map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}
for abb in df['PossessionTeam'].unique() :
map_abbr[abb] = abb
df['PossessionTeam'] = df['PossessionTeam'].map(map_abbr)
 df['HomeTeamAbbr'] = df['HomeTeamAbbr'].map(map_abbr)
 df['VisitorTeamAbbr'] = df['VisitorTeamAbbr'].map(map_abbr)
 df['HomePossession'] = df['PossessionTeam'] == df['HomeTeamAbbr']
return
process_team_abbr(df_play )<feature_engineering>
test_data.isna().sum()
def process_play_direction(df):
df['IsPlayLeftToRight'] = df['PlayDirection'].apply(lambda val: True if val.strip() == 'right' else False)
 return
process_play_direction(df_play )<categorify>
X_train.set_index('Id', inplace = True)
 y_train.set_index('Id', inplace = True)
 test_data.set_index('Id', inplace = True )
def process_yard_til_end_zone(df):
def convert_to_yardline100(row):
return(100 - row['YardLine'])if(row['PossessionTeam'] == row['FieldPosition'])else row['YardLine']
df['Yardline100'] = df.apply(convert_to_yardline100, axis=1)
 return
process_yard_til_end_zone(df_play )<merge>
%%time
X_train = reduce_mem_usage(X_train)
 y_train = reduce_mem_usage(y_train)
 test_data = reduce_mem_usage(test_data )
df_players = df_players.merge(
 df_play[['PlayId', 'PossessionTeam', 'HomeTeamAbbr', 'PlayDirection', 'Yardline100']],
how='left', on='PlayId')
 df_players.loc[df_players.Season == 2017, 'S'] = 10*df_players.loc[df_players.Season == 2017,'Dis']<feature_engineering>
from sklearn import metrics, tree
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score, f1_score, mean_squared_error, r2_score
from xgboost import XGBRegressor
def standarize_direction(df):
df['HomePossesion'] = df['PossessionTeam'] == df['HomeTeamAbbr']
df['Dir_rad'] = np.mod(90 - df.Dir, 360)* math.pi/180.0
df['ToLeft'] = df.PlayDirection == "left"
df['TeamOnOffense'] = "home"
df.loc[df.PossessionTeam != df.HomeTeamAbbr, 'TeamOnOffense'] = "away"
df['IsOnOffense'] = df.Team == df.TeamOnOffense
df['X_std'] = df.X
df.loc[df.ToLeft, 'X_std'] = 120 - df.loc[df.ToLeft, 'X']
df['Y_std'] = df.Y
df.loc[df.ToLeft, 'Y_std'] = 160/3 - df.loc[df.ToLeft, 'Y']
df['Dir_std'] = df.Dir_rad
df.loc[df.ToLeft, 'Dir_std'] = np.mod(np.pi + df.loc[df.ToLeft, 'Dir_rad'], 2*np.pi)
 df.loc[(df.IsOnOffense)& df['Dir_std'].isna() ,'Dir_std'] = 0.0
df.loc[~(df.IsOnOffense)& df['Dir_std'].isna() ,'Dir_std'] = np.pi
standarize_direction(df_players )<feature_engineering>
XGB = XGBRegressor(n_jobs=-1)
 param_grid = {
'colsample_bytree': np.linspace(0.3, 0.8, 1.0),
'n_estimators':[40 ,80, 150, 200,240, 300],
'max_depth': [1,2, 3,5, 8]
}
CV_XGB = GridSearchCV(estimator=XGB, param_grid=param_grid, cv= 10 )
def data_augmentation(df, sample_ids):
df_sample = df.loc[df.PlayId.isin(sample_ids)].copy()
df_sample['Y_std'] = 160/3 - df_sample['Y_std']
df_sample['Dir_std'] = df_sample['Dir_std'].apply(lambda x: 2*np.pi - x)
 df_sample['PlayId'] = df_sample['PlayId'].apply(lambda x: x+'_aug')
 return df_sample
def process_tracking_data(df):
df['Sx'] = df['S']*df['Dir_std'].apply(math.cos)
 df['Sy'] = df['S']*df['Dir_std'].apply(math.sin)
 rushers = df[df['IsRusher']].copy()
rushers.set_index('PlayId', inplace=True, drop=True)
 playId_rusher_map = rushers[['X_std', 'Y_std', 'Sx', 'Sy']].to_dict(orient='index')
 rusher_x = df['PlayId'].apply(lambda val: playId_rusher_map[val]['X_std'])
 rusher_y = df['PlayId'].apply(lambda val: playId_rusher_map[val]['Y_std'])
 rusher_Sx = df['PlayId'].apply(lambda val: playId_rusher_map[val]['Sx'])
 rusher_Sy = df['PlayId'].apply(lambda val: playId_rusher_map[val]['Sy'])
 df['player_minus_rusher_x'] = rusher_x - df['X_std']
df['player_minus_rusher_y'] = rusher_y - df['Y_std']
df['player_minus_rusher_Sx'] = rusher_Sx - df['Sx']
df['player_minus_rusher_Sy'] = rusher_Sy - df['Sy']
return
sample_ids = np.random.choice(df_play.PlayId.unique() , int(0.5*len(df_play.PlayId.unique())))
 df_players_aug = data_augmentation(df_players, sample_ids)
 df_players = pd.concat([df_players, df_players_aug])
 df_players.reset_index()
df_play_aug = df_play.loc[df_play.PlayId.isin(sample_ids)].copy()
df_play_aug['PlayId'] = df_play_aug['PlayId'].apply(lambda x: x+'_aug')
 df_play = pd.concat([df_play, df_play_aug])
 df_play.reset_index()
df_players.sort_values(by=['PlayId'],inplace=True)
 df_play.sort_values(by=['PlayId'],inplace=True)
 process_tracking_data(df_players )<count_missing_values>
%time CV_XGB.fit(X_train, y_train )
tracking_level_features = [
'PlayId',
'IsOnOffense',
'X_std',
'Y_std',
'Sx',
'Sy',
'player_minus_rusher_x',
'player_minus_rusher_y',
'player_minus_rusher_Sx',
'player_minus_rusher_Sy',
'IsRusher'
]
df_all_feats = df_players[tracking_level_features]
print('Any null values: ', df_all_feats.isnull().sum().sum() )<groupby>
best_xgb_model = CV_XGB.best_estimator_
print(CV_XGB.best_score_, CV_XGB.best_params_ )
%%time
grouped = df_all_feats.groupby('PlayId')
 train_x = np.zeros([len(grouped.size()),11,10,10])
 i = 0
play_ids = df_play.PlayId.values
for name, group in grouped:
if name!=play_ids[i]:
print("Error")
 [[rusher_x, rusher_y, rusher_Sx, rusher_Sy]] = group.loc[group.IsRusher==1,['X_std', 'Y_std','Sx','Sy']].values
offense_ids = group[group.IsOnOffense & ~group.IsRusher].index
defense_ids = group[~group.IsOnOffense].index
for j, defense_id in enumerate(defense_ids):
[def_x, def_y, def_Sx, def_Sy] = group.loc[defense_id,['X_std', 'Y_std','Sx','Sy']].values
[def_rusher_x, def_rusher_y] = group.loc[defense_id,['player_minus_rusher_x', 'player_minus_rusher_y']].values
[def_rusher_Sx, def_rusher_Sy] = group.loc[defense_id,['player_minus_rusher_Sx', 'player_minus_rusher_Sy']].values
train_x[i,j,:,:4] = group.loc[offense_ids,['Sx','Sy','X_std', 'Y_std']].values - np.array([def_Sx, def_Sy, def_x,def_y])
 train_x[i,j,:,-6:] = [def_rusher_Sx, def_rusher_Sy, def_rusher_x, def_rusher_y, def_Sx, def_Sy]
i+=1<feature_engineering>
pred_train_xgb = best_xgb_model.predict(X_train)
 pred_test_xgb = best_xgb_model.predict(test_data )
train_y = df_play[['PlayId', 'Yards']].copy()
train_y['YardIndex'] = train_y['Yards'].apply(lambda val: val + 99)
 min_idx_y = 71
max_idx_y = 150
train_y['YardIndexClipped'] = train_y['YardIndex'].apply(
 lambda val: min_idx_y if val < min_idx_y else max_idx_y if val > max_idx_y else val)
 print('max yardIndex: ', train_y.YardIndex.max())
 print('max yardIndexClipped: ', train_y.YardIndexClipped.max())
 print('min yardIndex: ', train_y.YardIndex.min())
 print('min yardIndexClipped: ', train_y.YardIndexClipped.min() )<create_dataframe>
print(metrics.mean_squared_log_error(y_train, pred_train_xgb ).round(5))
df_season = df_play[['PlayId', 'Season']].copy()<compute_train_metric>
y_pred_test_xgb = pd.DataFrame(pred_test_xgb, columns = ['SalePrice'])
 y_pred_test_xgb['Id'] = test['Id']
<define_variables><EOS> 
filename = 'House_Pricing_XGB.csv'
submission.to_csv(filename,index=False)
 print('Saved file: ' + filename )