Kaggle competition: NFL Big Data Bowl
<SOS> metric: CRPS Kaggle data source: nfl-big-data-bowl<normalization>
start_time = time.time()
time_limit = 9000
warnings.filterwarnings("ignore")
 env = nflrush.make_env()
loss_fn=nn.BCEWithLogitsLoss(reduction='sum')
 test_array = np.zeros(len(x_test))
kf = KFold(n_splits=n_splits, shuffle=True)
 for i_seed in range(n_seeds):
print(f'start seed {i_seed}')
 for i_fold,(dev_index, val_index)in enumerate(kf.split(x_test)) :
if i_fold >= TRAIN_ON_N_SPLITS:
break
print(f'start fold {i_fold}')
 scaler = joblib.load(os.path.join(MODEL_PATH, f'scaler-seed{0}-fold{0}.joblib'))
test_sentence_feature_mat = scaler.transform(sentence_feature_mat)
 test_loader = DynamicBucketIterator(x_test,
torch.tensor(np.concatenate([np.zeros(( len(x_test), 1+len(aux_col_list))),
test_sentence_feature_mat], axis=1),dtype=torch.float32 ).cuda() ,
capacity=MAX_LEN*batch_size, pad_token=0, shuffle=False, length_quantile=1, max_batch_size=2048, for_bert=True)
 print(torch.cuda.memory_allocated())
 mb = master_bar(range(n_epochs))
i_epoch = 1
model = NeuralNet(len(aux_col_list), sentence_feature_mat.shape[-1])
 load_path = os.path.join(MODEL_PATH, f'seed{0}-fold{0}-epoch{i_epoch*4+3}.torchModelState')
 model.load_state_dict(torch.load(load_path))
model.cuda()
print(i_epoch)
 start_time = time.time()
model.eval()
epoch_test_pred = np.zeros(len(x_test))
for batch in progress_bar(test_loader):
x_batch = batch[0]
segment_id_batch = batch[1]
input_mask_batch = batch[2]
y_batch = batch[3]
index_batch = batch[4]
y_true_batch = y_batch[:, :1+len(aux_col_list)]
sample_weight_batch = y_batch[:, 1+len(aux_col_list)]
sentence_feature_batch = y_batch[:, -sentence_feature_mat.shape[-1]:]
x_features = [torch.tensor(feature, dtype=torch.long ).cuda() for feature in [x_batch, segment_id_batch, input_mask_batch]]
y_pred = model(x_features, sentence_feature_batch)
 epoch_test_pred[index_batch] = sigmoid(y_pred[:, 0].detach().cpu().numpy())
 del x_features, y_pred
torch.cuda.empty_cache()
test_array = epoch_test_pred
elapsed_time = time.time() - start_time
del model, epoch_test_pred
gc.collect()
torch.cuda.empty_cache() 
<define_variables>
random.seed(721)
 os.environ['PYTHONHASHSEED'] = str(831)
 np.random.seed(101)
 torch.manual_seed(1111)
 torch.cuda.manual_seed(1117)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
test_pred_large_CASED = test_array<save_to_csv>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 if device == "cuda:0":
torch.cuda.set_device(0)
 debug = True
else:
debug = False
debug = False
device
df_submit = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')
 df_submit.prediction = test_pred
df_submit.to_csv('submission.csv', index=False )<load_from_csv>
class Mish(nn.Module):
def __init__(self):
super().__init__()
def forward(self, x):
return x *(torch.tanh(F.softplus(x)))
 def gelu(x):
return 0.5 * x *(1 + torch.tanh(math.sqrt(2 / math.pi)*(x + 0.044715 * torch.pow(x, 3))))
def _get_activation_fn(activation):
if activation == "relu":
return F.relu
elif activation == "gelu":
return gelu
else:
raise RuntimeError("activation should be relu/gelu, not %s." % activation)
 def multi_head_attention_forward(query,
key,
value,
embed_dim_to_check,
num_heads,
in_proj_weight,
in_proj_bias,
bias_k,
bias_v,
add_zero_attn,
dropout_p,
out_proj_weight,
out_proj_bias,
training=True,
key_padding_mask=None,
need_weights=True,
attn_mask=None,
use_separate_proj_weight=False,
q_proj_weight=None,
k_proj_weight=None,
v_proj_weight=None,
static_k=None,
static_v=None 
):
r
qkv_same = torch.equal(query, key)and torch.equal(key, value)
 kv_same = torch.equal(key, value)
 tgt_len, bsz, embed_dim = query.size()
assert embed_dim == embed_dim_to_check
assert list(query.size())== [tgt_len, bsz, embed_dim]
assert key.size() == value.size()
head_dim = embed_dim // num_heads
assert head_dim * num_heads == embed_dim, "embed_dim must be divisible by num_heads"
scaling = float(head_dim)** -0.5
if use_separate_proj_weight is not True:
if qkv_same:
q, k, v = F.linear(query, in_proj_weight, in_proj_bias ).chunk(3, dim=-1)
 elif kv_same:
_b = in_proj_bias
_start = 0
_end = embed_dim
_w = in_proj_weight[_start:_end, :]
if _b is not None:
_b = _b[_start:_end]
q = F.linear(query, _w, _b)
 if key is None:
assert value is None
k = None
v = None
else:
_b = in_proj_bias
_start = embed_dim
_end = None
_w = in_proj_weight[_start:, :]
if _b is not None:
_b = _b[_start:]
k, v = F.linear(key, _w, _b ).chunk(2, dim=-1)
 else:
_b = in_proj_bias
_start = 0
_end = embed_dim
_w = in_proj_weight[_start:_end, :]
if _b is not None:
_b = _b[_start:_end]
q = F.linear(query, _w, _b)
 _b = in_proj_bias
_start = embed_dim
_end = embed_dim * 2
_w = in_proj_weight[_start:_end, :]
if _b is not None:
_b = _b[_start:_end]
k = F.linear(key, _w, _b)
 _b = in_proj_bias
_start = embed_dim * 2
_end = None
_w = in_proj_weight[_start:, :]
if _b is not None:
_b = _b[_start:]
v = F.linear(value, _w, _b)
 else:
q_proj_weight_non_opt = torch.jit._unwrap_optional(q_proj_weight)
 len1, len2 = q_proj_weight_non_opt.size()
assert len1 == embed_dim and len2 == query.size(-1)
 k_proj_weight_non_opt = torch.jit._unwrap_optional(k_proj_weight)
 len1, len2 = k_proj_weight_non_opt.size()
assert len1 == embed_dim and len2 == key.size(-1)
 v_proj_weight_non_opt = torch.jit._unwrap_optional(v_proj_weight)
 len1, len2 = v_proj_weight_non_opt.size()
assert len1 == embed_dim and len2 == value.size(-1)
 if in_proj_bias is not None:
q = F.linear(query, q_proj_weight_non_opt, in_proj_bias[0:embed_dim])
 k = F.linear(key, k_proj_weight_non_opt, in_proj_bias[embed_dim:(embed_dim * 2)])
 v = F.linear(value, v_proj_weight_non_opt, in_proj_bias[(embed_dim * 2):])
 else:
q = F.linear(query, q_proj_weight_non_opt, in_proj_bias)
 k = F.linear(key, k_proj_weight_non_opt, in_proj_bias)
 v = F.linear(value, v_proj_weight_non_opt, in_proj_bias)
 q = q * scaling
if bias_k is not None and bias_v is not None:
if static_k is None and static_v is None:
k = torch.cat([k, bias_k.repeat(1, bsz, 1)])
 v = torch.cat([v, bias_v.repeat(1, bsz, 1)])
 if attn_mask is not None:
attn_mask = torch.cat([attn_mask,
torch.zeros(( attn_mask.size(0), 1),
dtype=attn_mask.dtype,
device=attn_mask.device)], dim=1)
 if key_padding_mask is not None:
key_padding_mask = torch.cat(
 [key_padding_mask, torch.zeros(( key_padding_mask.size(0), 1),
dtype=key_padding_mask.dtype,
device=key_padding_mask.device)], dim=1)
 else:
assert static_k is None, "bias cannot be added to static key."
assert static_v is None, "bias cannot be added to static value."
else:
assert bias_k is None
assert bias_v is None
q = q.contiguous().view(tgt_len, bsz * num_heads, head_dim ).transpose(0, 1)
 if k is not None:
k = k.contiguous().view(-1, bsz * num_heads, head_dim ).transpose(0, 1)
 if v is not None:
v = v.contiguous().view(-1, bsz * num_heads, head_dim ).transpose(0, 1)
 if static_k is not None:
assert static_k.size(0)== bsz * num_heads
assert static_k.size(2)== head_dim
k = static_k
if static_v is not None:
assert static_v.size(0)== bsz * num_heads
assert static_v.size(2)== head_dim
v = static_v
src_len = k.size(1)
 if key_padding_mask is not None:
assert key_padding_mask.size(0)== bsz
assert key_padding_mask.size(1)== src_len
if add_zero_attn:
src_len += 1
k = torch.cat([k, torch.zeros(( k.size(0), 1)+ k.size() [2:], dtype=k.dtype, device=k.device)], dim=1)
 v = torch.cat([v, torch.zeros(( v.size(0), 1)+ v.size() [2:], dtype=v.dtype, device=v.device)], dim=1)
 if attn_mask is not None:
attn_mask = torch.cat([attn_mask, torch.zeros(( attn_mask.size(0), 1),
dtype=attn_mask.dtype,
device=attn_mask.device)], dim=1)
 if key_padding_mask is not None:
key_padding_mask = torch.cat(
 [key_padding_mask, torch.zeros(( key_padding_mask.size(0), 1),
dtype=key_padding_mask.dtype,
device=key_padding_mask.device)], dim=1)
 attn_output_weights = torch.bmm(q, k.transpose(1, 2))
assert list(attn_output_weights.size())== [bsz * num_heads, tgt_len, src_len]
if attn_mask is not None:
attn_mask = attn_mask.unsqueeze(0)
 attn_output_weights += attn_mask
if key_padding_mask is not None:
attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
 attn_output_weights = attn_output_weights.masked_fill(
 key_padding_mask.unsqueeze(1 ).unsqueeze(2),
float('-inf'), 
)
 attn_output_weights = attn_output_weights.view(bsz * num_heads, tgt_len, src_len)
 attn_output_weights = F.softmax(
 attn_output_weights, dim=-1)
 attn_output_weights = F.dropout(attn_output_weights, p=dropout_p, training=training)
 attn_output = torch.bmm(attn_output_weights, v)
 assert list(attn_output.size())== [bsz * num_heads, tgt_len, head_dim]
attn_output = attn_output.transpose(0, 1 ).contiguous().view(tgt_len, bsz, embed_dim)
 attn_output = F.linear(attn_output, out_proj_weight, out_proj_bias)
 if need_weights:
attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
 return attn_output, attn_output_weights.sum(dim=1)/ num_heads
else:
return attn_output, None
class MultiheadAttention(nn.Module):
r
def __init__(self, embed_dim, num_heads, dropout=0., bias=True, add_bias_kv=False, add_zero_attn=False, kdim=None, vdim=None):
super(MultiheadAttention, self ).__init__()
self.embed_dim = embed_dim
self.kdim = kdim if kdim is not None else embed_dim
self.vdim = vdim if vdim is not None else embed_dim
self._qkv_same_embed_dim = self.kdim == embed_dim and self.vdim == embed_dim
self.num_heads = num_heads
self.dropout = dropout
self.head_dim = embed_dim // num_heads
assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads"
self.in_proj_weight = Parameter(torch.empty(3 * embed_dim, embed_dim))
if self._qkv_same_embed_dim is False:
self.q_proj_weight = Parameter(torch.Tensor(embed_dim, embed_dim))
self.k_proj_weight = Parameter(torch.Tensor(embed_dim, self.kdim))
self.v_proj_weight = Parameter(torch.Tensor(embed_dim, self.vdim))
if bias:
self.in_proj_bias = Parameter(torch.empty(3 * embed_dim))
else:
self.register_parameter('in_proj_bias', None)
 self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
 if add_bias_kv:
self.bias_k = Parameter(torch.empty(1, 1, embed_dim))
self.bias_v = Parameter(torch.empty(1, 1, embed_dim))
else:
self.bias_k = self.bias_v = None
self.add_zero_attn = add_zero_attn
self._reset_parameters()
def _reset_parameters(self):
if self._qkv_same_embed_dim:
xavier_uniform_(self.in_proj_weight)
 else:
xavier_uniform_(self.q_proj_weight)
 xavier_uniform_(self.k_proj_weight)
 xavier_uniform_(self.v_proj_weight)
 if self.in_proj_bias is not None:
constant_(self.in_proj_bias, 0.)
constant_(self.out_proj.bias, 0.)
if self.bias_k is not None:
xavier_normal_(self.bias_k)
 if self.bias_v is not None:
xavier_normal_(self.bias_v)
 def forward(self, query, key, value, key_padding_mask=None,
need_weights=True, attn_mask=None):
r
if hasattr(self, '_qkv_same_embed_dim')and self._qkv_same_embed_dim is False:
return multi_head_attention_forward(
 query, key, value, self.embed_dim, self.num_heads,
self.in_proj_weight, self.in_proj_bias,
self.bias_k, self.bias_v, self.add_zero_attn,
self.dropout, self.out_proj.weight, self.out_proj.bias,
training=self.training,
key_padding_mask=key_padding_mask, need_weights=need_weights,
attn_mask=attn_mask, use_separate_proj_weight=True,
q_proj_weight=self.q_proj_weight, k_proj_weight=self.k_proj_weight,
v_proj_weight=self.v_proj_weight)
 else:
if not hasattr(self, '_qkv_same_embed_dim'):
warnings.warn('A new version of MultiheadAttention module has been implemented.\
Please re-train your model with the new module',
UserWarning)
 return multi_head_attention_forward(
 query, key, value, self.embed_dim, self.num_heads,
self.in_proj_weight, self.in_proj_bias,
self.bias_k, self.bias_v, self.add_zero_attn,
self.dropout, self.out_proj.weight, self.out_proj.bias,
training=self.training,
key_padding_mask=key_padding_mask, need_weights=need_weights,
attn_mask=attn_mask)
 class TransformerEncoderLayer(nn.Module):
r
def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation="relu"):
super(TransformerEncoderLayer, self ).__init__()
self.self_attn = MultiheadAttention(d_model, nhead, dropout=dropout)
 self.linear1 = nn.Linear(d_model, dim_feedforward)
 self.dropout = nn.Dropout(dropout)
 self.linear2 = nn.Linear(dim_feedforward, d_model)
 self.norm1 = nn.LayerNorm(d_model)
 self.norm2 = nn.LayerNorm(d_model)
 self.dropout1 = nn.Dropout(dropout)
 self.dropout2 = nn.Dropout(dropout)
 self.activation = _get_activation_fn(activation)
 def forward(self, src, src_mask=None, src_key_padding_mask=None):
r
src2 = self.self_attn(src, src, src, attn_mask=src_mask,
key_padding_mask=src_key_padding_mask)[0]
src = src + self.dropout1(src2)
 src = self.norm1(src)
 if hasattr(self, "activation"):
src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))
else:
src2 = self.linear2(self.dropout(F.relu(self.linear1(src))))
src = src + self.dropout2(src2)
 src = self.norm2(src)
 return src
class TransformerDecoderLayer(nn.Module):
r
def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation="relu"):
super(TransformerDecoderLayer, self ).__init__()
self.self_attn = MultiheadAttention(d_model, nhead, dropout=dropout)
 self.multihead_attn = MultiheadAttention(d_model, nhead, dropout=dropout)
 self.linear1 = nn.Linear(d_model, dim_feedforward)
 self.dropout = nn.Dropout(dropout)
 self.linear2 = nn.Linear(dim_feedforward, d_model)
 self.norm1 = nn.LayerNorm(d_model)
 self.norm2 = nn.LayerNorm(d_model)
 self.norm3 = nn.LayerNorm(d_model)
 self.dropout1 = nn.Dropout(dropout)
 self.dropout2 = nn.Dropout(dropout)
 self.dropout3 = nn.Dropout(dropout)
 self.activation = _get_activation_fn(activation)
 def forward(self, tgt, memory, tgt_mask=None, memory_mask=None,
tgt_key_padding_mask=None, memory_key_padding_mask=None):
r
tgt2 = self.self_attn(tgt, tgt, tgt, attn_mask=tgt_mask,
key_padding_mask=tgt_key_padding_mask)[0]
tgt = tgt + self.dropout1(tgt2)
 tgt = self.norm1(tgt)
 tgt2 = self.multihead_attn(tgt, memory, memory, attn_mask=memory_mask,
key_padding_mask=memory_key_padding_mask)[0]
tgt = tgt + self.dropout2(tgt2)
 tgt = self.norm2(tgt)
 if hasattr(self, "activation"):
tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
else:
tgt2 = self.linear2(self.dropout(F.relu(self.linear1(tgt))))
tgt = tgt + self.dropout3(tgt2)
 tgt = self.norm3(tgt)
 return tgt
class RAdam(Optimizer):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):
defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 self.buffer = [[None, None, None] for ind in range(10)]
super(RAdam, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(RAdam, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
buffered = self.buffer[int(state['step'] % 10)]
if state['step'] == buffered[0]:
N_sma, step_size = buffered[1], buffered[2]
else:
buffered[0] = state['step']
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 buffered[1] = N_sma
if N_sma >= 5:
step_size = group['lr'] * math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 else:
step_size = group['lr'] /(1 - beta1 ** state['step'])
 buffered[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma >= 5:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 else:
p_data_fp32.add_(-step_size, exp_avg)
 p.data.copy_(p_data_fp32)
 return loss
class Lookahead(Optimizer):
def __init__(self, optimizer, k=5, alpha=0.5):
self.optimizer = optimizer
self.k = k
self.alpha = alpha
self.param_groups = self.optimizer.param_groups
self.state = defaultdict(dict)
 self.fast_state = self.optimizer.state
for group in self.param_groups:
group["counter"] = 0
def update(self, group):
for fast in group["params"]:
param_state = self.state[fast]
if "slow_param" not in param_state:
param_state["slow_param"] = torch.zeros_like(fast.data)
 param_state["slow_param"].copy_(fast.data)
 slow = param_state["slow_param"]
slow +=(fast.data - slow)* self.alpha
fast.data.copy_(slow)
 def update_lookahead(self):
for group in self.param_groups:
self.update(group)
 def step(self, closure=None):
loss = self.optimizer.step(closure)
 for group in self.param_groups:
if group["counter"] == 0:
self.update(group)
 group["counter"] += 1
if group["counter"] >= self.k:
group["counter"] = 0
return loss
def state_dict(self):
fast_state_dict = self.optimizer.state_dict()
slow_state = { 
(id(k)if isinstance(k, torch.Tensor)else k): v
for k, v in self.state.items()
}
fast_state = fast_state_dict["state"]
param_groups = fast_state_dict["param_groups"]
return {
"fast_state": fast_state,
"slow_state": slow_state,
"param_groups": param_groups,
}
def load_state_dict(self, state_dict):
slow_state_dict = {
"state": state_dict["slow_state"],
"param_groups": state_dict["param_groups"],
}
fast_state_dict = {
"state": state_dict["fast_state"],
"param_groups": state_dict["param_groups"],
}
super(Lookahead, self ).load_state_dict(slow_state_dict)
 self.optimizer.load_state_dict(fast_state_dict)
 self.fast_state = self.optimizer.state
def add_param_group(self, param_group):
param_group["counter"] = 0
self.optimizer.add_param_group(param_group )
items = pd.read_csv('.. /input/items.csv')
 shops = pd.read_csv('.. /input/shops.csv')
 cats = pd.read_csv('.. /input/item_categories.csv')
 train = pd.read_csv('.. /input/sales_train.csv')
 test = pd.read_csv('.. /input/test.csv' ).set_index('ID' )<filter>
train_df = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', low_memory=False )
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]<feature_engineering>
num_cols = ['IsRusher',
'IsRusherTeam',
'X','Y',
'Dir_X','Dir_Y',
'Orientation_X','Orientation_Y',
'S',
'DistanceToBall',
'BallDistanceX','BallDistanceY',
'BallAngleX','BallAngleY',
'related_horizontal_v','related_vertical_v',
'related_horizontal_A','related_vertical_A',
'TeamDistance','EnermyTeamDistance',
'TeamXstd','EnermyXstd',
'EnermyYstd','TeamYstd',
'DistanceToBallRank',
'DistanceToBallRank_AttTeam','DistanceToBallRank_DefTeam',
'YardLine',
'NextX','NextY',
'NextDistanceToBall',
'BallNextAngleX','BallNextAngleY',
'BallNextDistanceX','BallNextDistanceY',
'A',
]
map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}
for abb in train_df['PossessionTeam'].unique() :
map_abbr[abb] = abb
def Create_group_features(df,train=True):
df['PossessionTeam'] = df['PossessionTeam'].map(map_abbr)
 df['HomeTeamAbbr'] = df['HomeTeamAbbr'].map(map_abbr)
 df['Season'] =(df['Season'] == 2017)
 df.loc[df['Season'],'A'] = np.nan
df['S'] = 10*df['Dis']
df.loc[df['Season'],'Orientation'] = np.mod(90+df.loc[df['Season'],'Orientation'],360)
 mask = df['PlayDirection'] == 'left'
df.loc[mask, 'X'] = 120 - df.loc[mask, 'X']
df.loc[mask, 'Y'] =(160 / 3)- df.loc[mask, 'Y']
df.loc[mask, 'Orientation'] = np.mod(180 + df.loc[mask, 'Orientation'], 360)
 df.loc[mask, 'Dir'] = np.mod(180 + df.loc[mask, 'Dir'], 360)
 df['FieldPosition'].fillna('', inplace=True)
 mask = df['PossessionTeam'] != df['FieldPosition']
df.loc[mask, 'YardLine'] = 100 - df.loc[mask, 'YardLine']
df['PlayDirection'] = df['PlayDirection'].map({'left':1,'right':0})
 df['Team'] = df['Team'].map({'away':1,'home':0})
 df['Dir'] = np.mod(90+df['Dir'],360)
 df['Orientation'] = np.mod(90+df['Orientation'],360)
 df['Orientation_X'] = np.cos(df['Orientation']*2*np.pi/360)
 df['Orientation_Y'] = np.sin(df['Orientation']*2*np.pi/360)
 df['Dir_X'] = np.cos(df['Dir']*2*np.pi/360)
 df['Dir_Y'] = np.sin(df['Dir']*2*np.pi/360)
 df['IsRusher'] =(df['NflId'] == df['NflIdRusher'])
 rushers = df[df['IsRusher']][['PlayId', 'Team', 'X', 'Y', 'A', 'S', 'Dir_X','Dir_Y']]
rushers.columns = ['PlayId', 'RusherTeam', 'BallX', 'BallY', 'BallA', 'BallS', 'BallDir_X', 'BallDir_Y']
df = df.join(rushers.set_index('PlayId'), on='PlayId')
 df['IsRusherTeam'] =(df['Team'] == df['RusherTeam'])
 df['DistanceToBall'] =(( df['X'] - df['BallX'])**2 +(df['Y'] - df['BallY'])**2)**0.5
dx = df['BallX']-df['X']
dy = df['BallY']-df['Y']
angle = np.arctan(dy/(1e-8+dx)) *360/(2*np.pi)
 angle[dx<0] += 180
angle[(dx>0)&(dy<0)] += 360
angle = np.mod(angle,360)
 df['BallAngleX'] = np.sin(angle*2*np.pi/360)
 df['BallAngleY'] = np.cos(angle*2*np.pi/360)
 df['BallDistanceX'] = -dx
df['BallDistanceY'] = -dy
df['related_horizontal_v'] = df['S']*df['Dir_X'] - df['BallS']*df['BallDir_X']
df['related_vertical_v'] = df['S']*df['Dir_Y'] - df['BallS']*df['BallDir_Y']
df['related_horizontal_A'] = df['A']*df['Dir_X'] - df['BallA']*df['BallDir_X']
df['related_vertical_A'] = df['A']*df['Dir_Y'] - df['BallA']*df['BallDir_Y']
df['DistanceToBallRank'] = df.groupby('PlayId')['DistanceToBall'].rank()
Distance_Ball_Team = df.groupby(['PlayId','Team'])['DistanceToBall'].rank()
df['DistanceToBallRank_AttTeam'] = np.nan
df['DistanceToBallRank_DefTeam'] = np.nan
df.loc[df['IsRusherTeam'],'DistanceToBallRank_AttTeam'] = Distance_Ball_Team.loc[df['IsRusherTeam']]
df.loc[~df['IsRusherTeam'],'DistanceToBallRank_DefTeam'] = Distance_Ball_Team.loc[~df['IsRusherTeam']]
TeamXY = df.groupby(['PlayId','Team'])[['X','Y']].transform(np.mean)
 df['TeamDistance'] =(( df['X'] - TeamXY['X'])**2 +(df['Y'] - TeamXY['Y'])**2)**0.5
df['TeamXstd'] = df['X'] - TeamXY['X']
df['TeamYstd'] = df['Y'] - TeamXY['Y']
EnermyTeamXY = TeamXY.groupby(df['PlayId'] ).apply(lambda x: x.iloc[::-1] ).values
EnermyTeamXY = pd.DataFrame(EnermyTeamXY,index = TeamXY.index,columns = TeamXY.columns)
 df['EnermyTeamDistance'] =(( df['X'] - EnermyTeamXY['X'])**2 +(df['Y'] - EnermyTeamXY['Y'])**2)**0.5
df['EnermyXstd'] = df['X'] - EnermyTeamXY['X']
df['EnermyYstd'] = df['Y'] - EnermyTeamXY['Y']
df['NextX'] = df['S']*df['Dir_X']+df['X']
df['NextY'] = df['S']*df['Dir_Y']+df['Y']
rushers = df[df['IsRusher']][['PlayId', 'NextX', 'NextY']]
rushers.columns = ['PlayId', 'BallNextX', 'BallNextY']
df = df.join(rushers.set_index('PlayId'), on='PlayId')
 df['NextDistanceToBall'] =(( df['NextX'] - df['BallNextX'])**2 +(df['NextY'] - df['BallNextY'])**2)**0.5
dx = df['BallNextX']-df['NextX']
dy = df['BallNextY']-df['NextY']
angle = np.arctan(dy/(1e-8+dx)) *360/(2*np.pi)
 angle[dx<0] += 180
angle[(dx>0)&(dy<0)] += 360
angle = np.mod(angle,360)
 df['BallNextAngleX'] = np.sin(angle*2*np.pi/360)
 df['BallNextAngleY'] = np.cos(angle*2*np.pi/360)
 df['BallNextDistanceX'] = -dx
df['BallNextDistanceY'] = -dy
return df[num_cols]
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median<feature_engineering>
%%time
groups =(train_df['GameId'].astype(str ).str[:4] + train_df['Week'].astype(str))
groups = groups.groupby(train_df['PlayId'] ).apply(lambda x:x.iloc[0] ).values
train_y = train_df['Yards'].groupby(train_df['PlayId'] ).apply(lambda x:x.iloc[0] ).values
train_X_df = Create_group_features(train_df.copy())
 bool_cols = ['IsRusher','IsRusherTeam']
Scale_cols = [col for col in train_X_df.columns if col not in bool_cols]
X_mean = train_X_df[Scale_cols].mean()
X_std = train_X_df[Scale_cols].std()
train_X = train_X_df.copy()
train_X[bool_cols] = train_X[bool_cols].astype(np.float32)
 train_X[Scale_cols] =(train_X_df[Scale_cols] - X_mean[Scale_cols])/X_std[Scale_cols]
train_X.fillna(0,inplace=True)
 train_X = train_X.values.reshape(len(train_y),22,-1)
 train_X.shape
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11<categorify>
n = np.arange(-99, 100)
 n = np.row_stack([n] * train_y.shape[0])
 ym = train_y.reshape(train_y.shape[0], 1)
 train_y = np.heaviside(n - ym, 1 )
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )<concatenate>
Lower_yards = np.heaviside(99 - train_X_df['YardLine'].values[::22].reshape(train_y.shape[0], 1)- np.row_stack([np.arange(199)] * train_y.shape[0]),0)
 Upper_yards = np.heaviside(np.row_stack([np.arange(199)] * train_y.shape[0])- 199 + train_X_df['YardLine'].values[::22].reshape(train_y.shape[0], 1),0)
 Yard_mask = Lower_yards + Upper_yards
len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test )<data_type_conversions>
train_y_OH = np.zeros([train_y.shape[0],199])
 train_y_OH[np.arange(len(ym)) ,(ym+99 ).reshape(-1)] = 1
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts<feature_engineering>
class Model(nn.Module):
def __init__(self,params):
super(Model, self ).__init__()
Activation = params['Activation']
self.layers = params['layer']
self.emb = Seq(nn.Dropout(params['dropout_0']),Lin(params['input_size'],params['hidden_0']))
self.transformer_encoder = nn.ModuleList()
self.transformer_decoder = nn.ModuleList()
for l in range(self.layers):
self.transformer_encoder.append(TransformerEncoderLayer(params['hidden_0'],\
nhead=params['head'],\
dim_feedforward=params['hidden_1'],\
dropout=params['dropout_1'],\
activation = params['transformer_activation']))
self.transformer_decoder.append(TransformerDecoderLayer(params['hidden_0'],\
nhead=params['head'],\
dim_feedforward=params['hidden_1'],\
dropout=params['dropout_1'],\
activation = params['transformer_activation']))
self.mlp = Seq(Lin(params['hidden_0'], params['hidden_3']),Activation,nn.Dropout(params['dropout_3']),Lin(params['hidden_3'], 199))
def forward(self, X):
rusher_mask = X[:,:,0]>0
combine = self.emb(X)
 x = combine.permute(1,0,2)
 for l in range(self.layers):
x = self.transformer_encoder[l](x)
 rusher =(x.permute(1,0,2)[rusher_mask].unsqueeze(1)).permute(1,0,2)
 for l in range(self.layers):
rusher = self.transformer_decoder[l](rusher,x)
 x = rusher.permute(1,0,2)
 y = x.squeeze(1)
 y = self.mlp(y)
 y = F.softmax(y,dim=-1)
 return y,torch.cumsum(y,dim = -1 )
train['revenue'] = train['item_price'] * train['item_cnt_day']<merge>
def Schedule_scheme(x):
if x%3==0:
return 1
elif x%3==1:
return 0.5
else:
return 0.2
def Create_model(params,input_size = train_X.shape[-1]):
params['input_size'] = input_size
clf = Model(params ).to(device)
 vv = 0
for v in list(clf.parameters()):
vv += len(v.view(-1))
print("Num_Variables:",vv)
 base_opt = RAdam(clf.parameters() ,lr=params['lr'],\
weight_decay=params['weight_decay'],\
betas=(1-params['alpha'], 1-params['beta']), eps=params['eps'] 
)
 optimizer = Lookahead(base_opt, k=params['lookahead_k'], alpha=params['lookahead_alpha'])
 scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda= lambda x: 1 if x%2==0 else 0.5)
 return clf,optimizer,scheduler
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts<data_type_conversions>
config = {'Activation': 'lrelu1',
'alpha': 0.06177912913861566,
'batch_size': 82,
'beta': 0.025662159487370977,
'cosine_T': 67,
'dropout_0': 0.023898897998374913,
'dropout_1': 0.06080646510619758,
'dropout_2': 0.025011299009768506,
'dropout_3': 0.22165717850445166,
'eps': 5.884062604808768e-07,
'head': 6,
'hidden_0': 111,
'hidden_1': 2040,
'hidden_2': 1742,
'hidden_3': 933,
'layer': 4,
'lookahead_alpha': 0.5422623511158232,
'lookahead_k': 3,
'loss_1': 0.48305609964511376,
'loss_2': 0.2353079388007755,
'lr': 0.00014789689708999082,
'min_lr': 0.0038268377963604707,
'transformer_activation': 'gelu',
'weight_decay': 2.2445535063229495e-05}
act_map = {'lrelu1':LeakyReLU(0.1),
'lrelu2':LeakyReLU(0.2),
'relu':ReLU() ,
'selu':SELU() ,
'prelu':nn.PReLU() ,
'elu':nn.ELU() ,
'mish':Mish() }
params = {'Activation': Mish() ,
'dropout_0': config['dropout_0'],
'hidden_0': 128,
'head': 32,
'hidden_1': 512,
"dropout_1": 0.2,
'transformer_activation': config['transformer_activation'],
"hidden_2": config['hidden_2'],
"dropout_2": config['dropout_2'],
'hidden_3': 512,
'dropout_3': 0.3,
"layer": 2,
"lr": 5*config['lr'],
'weight_decay': 1e-1,
'alpha': config['alpha'],
'beta': config['beta'],
'eps': config['eps'],
'lookahead_k': 5,
'lookahead_alpha': 0.5,
'cosine_T': config['cosine_T'],
'min_lr': 0.01*config['lr'] * config['min_lr'],
'batch_size': 32,
'loss_1': 0.5,
'loss_2': 0.5
}
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )<concatenate>
model_list = []
oof = np.zeros([len(train_X),199])
 batch_size = params['batch_size']
for fold_ in range(999):
trn_idx = np.arange(len(train_X))
X_tr,y_tr,y_OH_tr = train_X[trn_idx], train_y[trn_idx],train_y_OH[trn_idx]
tmp = deepcopy(y_OH_tr)
 y_OH_tr[:,:-1] += 0.5 * tmp[:,1:]
y_OH_tr[:,1:] += 0.5 * tmp[:,:-1]
params['cosine_T'] = len(y_tr)//batch_size
clf,optimizer,scheduler = Create_model(params)
 best_loss = 9999
for epoch in tqdm_notebook(range(30)) :
clf.train()
batches = np.arange(len(trn_idx))
np.random.shuffle(batches)
 batches = batches[:batch_size *(len(batches)//batch_size)]
batches = batches.reshape(-1,batch_size)
 Train_pred = []
MA_loss = 0
for batch in batches:
optimizer.zero_grad()
X = torch.Tensor(X_tr[batch] ).to(device)
 y = torch.Tensor(y_tr[batch] ).to(device)
 y_OH = torch.Tensor(y_OH_tr[batch] ).to(device)
 y_pred_OH,y_pred = clf(X)
 loss = torch.mean(( y - y_pred)**2)
 loss2 = -torch.mean(y_OH * torch.log(1e-8 + y_pred_OH)+(1-y_OH)* torch.log(1-y_pred_OH+1e-8)) 
(params['loss_1']*loss+params['loss_2']*loss2 ).backward()
optimizer.step()
MA_loss += loss.item()
if epoch >= 10:
scheduler.step()
MA_loss /= len(batches)
 print("epoch",epoch,"Train:",MA_loss)
 if epoch >= 10 and epoch%2 == 1:
model_list.append(deepcopy(clf))
if time.time() -start_time > time_limit:
break
if time.time() -start_time > time_limit:
print("Overtime")
 break
<data_type_conversions><EOS> 
for(test_df, sample_prediction_df)in tqdm_notebook(env.iter_test()):
test_X = Create_group_features(test_df.copy() ,train=False)
 yardline = test_X['YardLine'].iloc[-1]
test_X[bool_cols] = test_X[bool_cols].astype(np.float32)
 test_X[Scale_cols] =(test_X[Scale_cols] - X_mean[Scale_cols])/X_std[Scale_cols]
test_X = torch.Tensor(test_X.fillna(0 ).values.reshape(-1,22,train_X.shape[-1]))
y_pred_p = np.zeros(199)
 for i,model in enumerate(model_list):
model.eval()
pred = model(test_X)[1][-1].detach().numpy()
y_pred_p += pred / len(model_list)
 y_pred_p = np.clip(y_pred_p,0,1)
 y_pred_p[:int(99-yardline)] = 0
y_pred_p[int(199-yardline):] = 1
env.predict(pd.DataFrame(data=[y_pred_p],columns=sample_prediction_df.columns))
env.write_submission_file()