Kaggle competition: NFL Big Data Bowl
<SOS> metric: CRPS Kaggle data source: nfl-big-data-bowl<load_pretrained>
tqdm.pandas()
matrix.to_pickle('data.pkl')
 del matrix
del cache
del group
del items
del shops
del cats
del train
gc.collect() ;<load_pretrained>
from kaggle.competitions import nflrush
data = pd.read_pickle('data.pkl' )<drop_column>
def init_seeds(seed):
np.random.seed(seed)
 rn.seed(seed)
 tf.set_random_seed(seed)
 sess = tf.Session(graph=tf.get_default_graph())
 K.set_session(sess)
 return sess
data = data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'item_cnt_month_lag_6',
'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3',
'date_item_avg_item_cnt_lag_6',
'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2',
'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6',
'date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month',
'days',
'item_shop_last_sale',
'item_last_sale',
'item_shop_first_sale',
'item_first_sale',
]]<prepare_x_and_y>
input_folder = '.. /input/nfl-big-data-bowl-2020'
train = pd.read_csv(os.path.join(input_folder, 'train.csv'), dtype={'WindSpeed': 'object'})
 train.shape
results = {}
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )<set_options>
player_cols = ['X', 'Y', 'X_dir', 'Y_dir', 'X_S', 'Y_S', 'S', 'A', 'IsRusher', 'IsOnOffense']
play_cols = ['X_rusher', 'Y_rusher', 'YardLine_std']
del data
gc.collect() ;<train_model>
def reorient(train, flip_left):
train['ToLeft'] = train.PlayDirection == "left"
train.loc[train.VisitorTeamAbbr == "ARI", 'VisitorTeamAbbr'] = "ARZ"
train.loc[train.HomeTeamAbbr == "ARI", 'HomeTeamAbbr'] = "ARZ"
train.loc[train.VisitorTeamAbbr == "BAL", 'VisitorTeamAbbr'] = "BLT"
train.loc[train.HomeTeamAbbr == "BAL", 'HomeTeamAbbr'] = "BLT"
train.loc[train.VisitorTeamAbbr == "CLE", 'VisitorTeamAbbr'] = "CLV"
train.loc[train.HomeTeamAbbr == "CLE", 'HomeTeamAbbr'] = "CLV"
train.loc[train.VisitorTeamAbbr == "HOU", 'VisitorTeamAbbr'] = "HST"
train.loc[train.HomeTeamAbbr == "HOU", 'HomeTeamAbbr'] = "HST"
train['TeamOnOffense'] = "home"
train.loc[train.PossessionTeam != train.HomeTeamAbbr, 'TeamOnOffense'] = "away"
train['IsOnOffense'] = train.Team == train.TeamOnOffense
train['YardLine_std'] = 100 - train.YardLine
train.loc[train.FieldPosition.fillna('')== train.PossessionTeam,
'YardLine_std'
] = train.loc[train.FieldPosition.fillna('')== train.PossessionTeam,
'YardLine']
train.loc[train.ToLeft, 'X'] = 120 - train.loc[train.ToLeft, 'X']
train.loc[train.ToLeft, 'Y'] = 160/3 - train.loc[train.ToLeft, 'Y']
train.loc[train.ToLeft, 'Orientation'] = np.mod(180 + train.loc[train.ToLeft, 'Orientation'], 360)
 train['Dir'] = 90 - train.Dir
train.loc[train.ToLeft, 'Dir'] = np.mod(180 + train.loc[train.ToLeft, 'Dir'], 360)
 train.loc[train.IsOnOffense, 'Dir'] = train.loc[train.IsOnOffense, 'Dir'].fillna(0 ).values
train.loc[~train.IsOnOffense, 'Dir'] = train.loc[~train.IsOnOffense, 'Dir'].fillna(180 ).values
train['S'] = 10 * train['Dis']
train.loc[train.Season == 2017, 'A'] = train.loc[train.Season == 2017, 'A'].mean()
train['IsRusher'] = train['NflId'] == train['NflIdRusher']
if flip_left:
df = train[train['IsRusher']].copy()
df['left'] = df.Dir < 0
train = train.merge(df[['PlayId', 'left']], how='left', on='PlayId')
 train['Y'] = train.Y
train.loc[train.left, 'Y'] = 160/3 - train.loc[train.left, 'Y']
train['Dir'] = train.Dir
train.loc[train.left, 'Dir'] = np.mod(- train.loc[train.left, 'Dir'], 360)
 train.drop('left', axis=1, inplace=True)
 train['X_dir'] = np.cos(( np.pi / 180)* train.Dir)
 train['Y_dir'] = np.sin(( np.pi / 180)* train.Dir)
 train['X_S'] = train.X_dir * train.S
train['Y_S'] = train.Y_dir * train.S
train['X_A'] = train.X_dir * train.A
train['Y_A'] = train.Y_dir * train.A
train['PlayerHeight'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
train['BMI'] =(train['PlayerWeight'] * 703)/(train['PlayerHeight'] ** 2)
 train['Energy'] = train['PlayerWeight'] *(train['S'] ** 2)
 train['time_step'] = 0.0
train = train.sort_values(by=['PlayId', 'IsOnOffense', 'IsRusher', 'Y'] ).reset_index(drop=True)
 return train
def add_features(train):
df = train[train.IsRusher][['PlayId', 'time_step', 'X', 'Y']].copy()
df.columns = ['PlayId', 'time_step', 'X_rusher', 'Y_rusher']
train = train.merge(df, how='left', on=['PlayId', 'time_step'])
 train.X -= train.X_rusher
train.Y -= train.Y_rusher
return train
def time_forward(train, time_step):
train = train.copy()
train['X'] = train['X'] + time_step * train['X_S'] + 0.5 * time_step**2 * train['X_A']
train['Y'] = train['Y'] + time_step * train['Y_S'] + 0.5 * time_step**2 * train['Y_A']
x_s = np.clip(train['X_S'] + time_step * train['X_A'], -14, 14)
 y_s = np.clip(train['Y_S'] + time_step * train['Y_A'], -14, 14)
 if 1:
train['X_A'] = np.clip(train['X_A'] * train['X_S'] /(x_s), -10, 10)
 train['Y_A'] = np.clip(train['Y_A'] * train['Y_S'] /(y_s), -10, 10)
 train['X_S'] = x_s
train['Y_S'] = y_s
train['time_step'] = time_step
return train
ts = time.time()
model = XGBRegressor(
 max_depth=16,
n_estimators=1200,
min_child_weight=400,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 10)
 time.time() - ts<save_to_csv>
train_reorient = reorient(train.copy() , flip_left=True)
 len_train_reorient = len(train_reorient)// 22
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('xgb_test.pickle', 'wb'))<load_from_csv>
def augment_data(train_reorient, time_steps):
train_reorient = pd.concat(time_forward(train_reorient, time_step)for time_step in time_steps)
 train_reorient = add_features(train_reorient)
 return train_reorient
items = pd.read_csv('.. /input/items.csv')
 shops = pd.read_csv('.. /input/shops.csv')
 cats = pd.read_csv('.. /input/item_categories.csv')
 train = pd.read_csv('.. /input/sales_train.csv')
 test = pd.read_csv('.. /input/test.csv' ).set_index('ID' )<filter>
time_steps = [0, 0.3, 0.6]
train_reorient = augment_data(train_reorient, time_steps )
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]<feature_engineering>
len_2017 = train[train.Season == 2017].shape[0] // 22
len_2017
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median<feature_engineering>
ss_players = StandardScaler()
X_players = df.values.astype('float32')
 X_players = ss_players.fit_transform(df)
 X_players = X_players.reshape(( -1, 22, len(player_cols)))
 X_players.shape
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11<categorify>
X_play = train_reorient[play_cols].values[::22]
ss_play = StandardScaler()
X_play = ss_play.fit_transform(X_play )
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )<concatenate>
@jit
def get_dmat(X, Y):
dmat = np.zeros(( 22, 22))
for i in range(22):
for j in range(i+1, 22):
d = np.sqrt(( X[i] - X[j])**2 +(Y[i] - Y[j])**2)
 dmat[i, j] = d
dmat[j, i] = d
dmat = dmat.reshape(( 1, 22, 22))
return dmat
len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test )<data_type_conversions>
X,Y = train_reorient.X.values, train_reorient.Y.values
dmats = [get_dmat(X[i:i+22], Y[i:i+22])for i in range(0, train_reorient.shape[0], 22)]
dmats = np.vstack(dmats ).reshape(( -1, 22, 22))
dmats.shape
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts<feature_engineering>
del train
train['revenue'] = train['item_price'] * train['item_cnt_day']<merge>
class LayerNormalization(keras.layers.Layer):
def __init__(self,
center=True,
scale=True,
epsilon=None,
gamma_initializer='ones',
beta_initializer='zeros',
gamma_regularizer=None,
beta_regularizer=None,
gamma_constraint=None,
beta_constraint=None,
**kwargs):

 super(LayerNormalization, self ).__init__(**kwargs)
 self.supports_masking = True
self.center = center
self.scale = scale
if epsilon is None:
epsilon = K.epsilon() * K.epsilon()
self.epsilon = epsilon
self.gamma_initializer = keras.initializers.get(gamma_initializer)
 self.beta_initializer = keras.initializers.get(beta_initializer)
 self.gamma_regularizer = keras.regularizers.get(gamma_regularizer)
 self.beta_regularizer = keras.regularizers.get(beta_regularizer)
 self.gamma_constraint = keras.constraints.get(gamma_constraint)
 self.beta_constraint = keras.constraints.get(beta_constraint)
 self.gamma, self.beta = None, None
def get_config(self):
config = {
'center': self.center,
'scale': self.scale,
'epsilon': self.epsilon,
'gamma_initializer': keras.initializers.serialize(self.gamma_initializer),
'beta_initializer': keras.initializers.serialize(self.beta_initializer),
'gamma_regularizer': keras.regularizers.serialize(self.gamma_regularizer),
'beta_regularizer': keras.regularizers.serialize(self.beta_regularizer),
'gamma_constraint': keras.constraints.serialize(self.gamma_constraint),
'beta_constraint': keras.constraints.serialize(self.beta_constraint),
}
base_config = super(LayerNormalization, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
def compute_mask(self, inputs, input_mask=None):
return input_mask
def build(self, input_shape):
shape = input_shape[-1:]
if self.scale:
self.gamma = self.add_weight(
 shape=shape,
initializer=self.gamma_initializer,
regularizer=self.gamma_regularizer,
constraint=self.gamma_constraint,
name='gamma', 
)
 if self.center:
self.beta = self.add_weight(
 shape=shape,
initializer=self.beta_initializer,
regularizer=self.beta_regularizer,
constraint=self.beta_constraint,
name='beta', 
)
 super(LayerNormalization, self ).build(input_shape)
 def call(self, inputs, training=None):
mean = K.mean(inputs, axis=-1, keepdims=True)
 variance = K.mean(K.square(inputs - mean), axis=-1, keepdims=True)
 std = K.sqrt(variance + self.epsilon)
 outputs =(inputs - mean)/ std
if self.scale:
outputs *= self.gamma
if self.center:
outputs += self.beta
return outputs
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts<data_type_conversions>
y_mae = train_reorient['Yards'].values[::22]
y_mean = np.median(y_mae)
 y_crps = np.zeros(shape=(y_mae.shape[0], 199))
for i,yard in enumerate(y_mae):
y_crps[i, yard+99:] = 1
yardline = train_reorient['YardLine_std'].values[::22]
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )<concatenate>
def crps(y_true, y_pred):
stops = np.arange(-99, 100)
 unit_steps = stops >= y_true.reshape(-1, 1)
 crps = np.mean(( y_pred - unit_steps)**2)
 return crps
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts<data_type_conversions>
def nondecreasing(x):
X_ir = np.arange(199 ).astype('float64')
 ir = IsotonicRegression(0, 1)
 x = ir.fit_transform(X_ir, x.astype('float64'))
return x
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts<merge>
from keras.models import Model, load_model
from keras.layers import Input, BatchNormalization, Activation, Add, Multiply, Dot
from keras.layers import Embedding, Permute, Reshape
from keras.layers.core import Dropout, Lambda, Dense, Flatten
from keras.layers.convolutional import Conv1D, Conv2D
from keras.layers.pooling import GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.layers.merge import Concatenate
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from keras.optimizers import Adam, SGD, Nadam
from keras import backend as K
from keras.engine.topology import Layer
import tensorflow as tf
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<merge>
class ScaleLayer(Layer):
def __init__(self, output_dim, **kwargs):
self.output_dim = output_dim
super(ScaleLayer, self ).__init__(**kwargs)
 def build(self, input_shape):
super(ScaleLayer, self ).build(input_shape)
 def call(self, x):
xx = K.arange(-99, 100, dtype=tf.float32)
 mu = y_mean + tf.reshape(x[:, 0],(-1, 1))
sigma_minus = tf.identity(K.exp(0.5 * tf.reshape(x[:, 1],(-1, 1))), name="sigma")
 sigma_plus = tf.identity(K.exp(0.5 * tf.reshape(x[:, 2],(-1, 1))), name="sigma")
 xx = tf.subtract(xx, mu)
 pcf = tf.where(xx >= 0, tf.divide(xx, sigma_plus), tf.divide(xx, sigma_minus))
return pcf
def compute_output_shape(self, input_shape):
return(input_shape[0], self.output_dim )
ts = time.time()
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
def dist_mult(dist, players):
res = Lambda(lambda c: K.batch_dot(c[0], c[1]))([dist, players])
 return res
def dist_attention(dist, players, dropout):
if 1:
dist1 = Reshape(( 22, 22, 1))(dist)
 dist1 = Conv2D(16, 1, activation='relu',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(dist1)
 dist1 = Conv2D(1, 1, activation='relu',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(dist1)
 dist1 = Reshape(( 22, 22))(dist1)
 dist = Add()([dist, dist1])
 dist = LayerNormalization()(dist)
 att = dist_mult(dist, players,)
 x_player = Add()([players, att])
 x_player = LayerNormalization()(x_player)
 if dropout > 0:
x_player = Dropout(dropout )(x_player)
 return x_player
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
def attention(x_inner, x_outer, n_factor, dropout):
x_Q = Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_inner)
 x_K = Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_V = Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_KT = Permute(( 2, 1))(x_K)
 res = Lambda(lambda c: K.batch_dot(c[0], c[1])/ np.sqrt(n_factor))([x_Q, x_KT])
 att = Lambda(lambda c: K.softmax(c, axis=-1))(res)
 att = Lambda(lambda c: K.batch_dot(c[0], c[1]))([att, x_V])
 return att
def multi_head_self_attention(x, n_factor, n_head, dropout):
if n_head == 1:
att = attention(x, x, n_factor, dropout)
 else:
n_factor_head = n_factor // n_head
heads = [attention(x, x, n_factor_head, dropout)for i in range(n_head)]
att = Concatenate()(heads)
 att = Dense(n_factor,
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(att)
 x = Add()([x, att])
 x = LayerNormalization()(x)
 if dropout > 0:
x = Dropout(dropout )(x)
 return x
def multi_head_outer_attention(x_inner, x_outer, n_factor, n_head, dropout):
if n_head == 1:
att = attention(x_inner, x_outer, n_factor, dropout)
 else:
n_factor_head = n_factor // n_head
heads = [attention(x_inner, x_outer, n_factor_head, dropout)for i in range(n_head)]
att = Concatenate()(heads)
 att = Dense(n_factor,
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(att)
 x_inner = Add()([x_inner, att])
 x_inner = LayerNormalization()(x_inner)
 if dropout > 0:
x = Dropout(dropout )(x_inner)
 return x
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
def se_bloc(in_bloc, ch, ratio):
x = GlobalAveragePooling1D()(in_bloc)
 x = Dense(ch//ratio, activation='relu' )(x)
 x = Dense(ch, activation='sigmoid' )(x)
 x = Multiply()([in_bloc, x])
 return Add()([x, in_bloc])
 def conv_bloc(players, n_factor, n_hidden, se_ratio, dropout):
players0 = players
players = Conv1D(n_hidden, 1, activation='relu',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(players)
 players = Conv1D(n_factor, 1, activation='relu',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(players)
 players = Add()([players0, players])
 players = se_bloc(players, n_factor, se_ratio)
 players = LayerNormalization()(players)
 if dropout > 0:
players = Dropout(dropout )(players)
 return players
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
def get_model(n_player, n_factor, n_loop, n_head, n_hidden, se_ratio, dropout, n_player_cols, n_play_cols):
input_players = Input(( n_player, n_player_cols), name="players")
 input_dmats = Input(( n_player, n_player), name="inv_dist")
 input_play = Input(( n_play_cols,), name="plays")
 x_player = input_players
x_player = Conv1D(n_factor, 1 )(x_player)
 x_player = LayerNormalization()(x_player)
 for l in range(n_loop):
x_player = dist_attention(input_dmats, x_player, dropout)
 x_player = conv_bloc(x_player, n_factor, n_hidden, se_ratio, dropout)
 x_player = multi_head_self_attention(x_player, n_factor, n_head, dropout)
 x_player = conv_bloc(x_player, n_factor, n_hidden, se_ratio, dropout)
 x_play = Dense(n_factor )(input_play)
 x_play = Reshape(( 1, -1))(x_play)
 readout = multi_head_outer_attention(x_play, x_player, n_factor, n_head, dropout)
 readout = Flatten()(readout)
 out1 = Dense(199, activation='sigmoid' )(readout)
 readout = Dense(4 )(readout)
 readout = ScaleLayer(output_dim=199 )(readout)
 out2 = keras.layers.Activation('sigmoid' )(readout)
 return Model(inputs=[input_players, input_dmats, input_play], outputs=[out1, out2])
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
simulate_test = False
if simulate_test:
len_data = X_players.shape[0]
len_train = int(0.9 * len_data)
 else:
len_data = X_players.shape[0]
len_train = len_data
X_players_train, X_players_test = X_players[:len_train], X_players[len_train:]
inv_dmats_train, inv_dmats_test = inv_dmats[:len_train], inv_dmats[len_train:]
X_play_train, X_play_test = X_play[:len_train], X_play[len_train:]
y_mae_train, y_mae_test = y_mae[:len_train], y_mae[len_train:]
y_crps_train, y_crps_test = y_crps[:len_train], y_crps[len_train:]
yardline_train, yardline_test = yardline[:len_train], yardline[len_train:]
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_type_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')
 matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')
 matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
kf = KFold(n_splits=12, shuffle=False)
 ind_2017 = np.arange(len_2017)
 def add_2017(x,y):
x = np.concatenate(( ind_2017, x + len_2017))
y = y + len_2017
return x,y
indices = [add_2017(x, y)for x, y in kf.split(X_players_train[len_2017:len_train_reorient],
y_mae_train[len_2017:len_train_reorient])]
n_steps = len(time_steps)
 def augment_indices(x, y, time_steps):
x_a = np.concatenate([x + i*len_train_reorient for i in range(n_steps)])
 y_a = np.concatenate([y + i*len_train_reorient for i in range(n_steps)])
 return x_a, y_a
indices = [augment_indices(x, y, time_steps)for x,y in indices ]
indices = [(i, ind)for i,ind in enumerate(indices)]
w = np.ones(len_train_reorient)
 w[:len_2017] = 0.5
w = np.concatenate([w for i in range(n_steps)] )
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
def post_process(y_pred, yardline_train, time_steps):
upper = 100 - yardline_train
lower = - yardline_train
for i in range(y_pred.shape[0]):
y_pred[i, 99 + upper[i]:] = 1
y_pred[i, :99 + lower[i]] = 0
y_pred[i] = nondecreasing(y_pred[i].ravel())
 if 1:
n_steps = len(time_steps)
 len_y_pred = len(y_pred)// n_steps
y_pred = np.mean([y_pred[len_y_pred * i: len_y_pred *(i+1)] for i in range(n_steps)], axis=0)
 return y_pred
ts = time.time()
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
name = 'nn'
results[name] = {'crps': [], 'y_test': [], 'imp': [], 'models': []}
try:
del sess
except:
pass
sess = init_seeds(0)
 pred_test = []
for index in indices:
if 1 and index[0] < 10:
continue
train_idx, val_idx = index[1]
train_players, train_inv_dmats, train_play, train_y, =(X_players_train[train_idx],
inv_dmats_train[train_idx],
X_play_train[train_idx],
y_crps_train[train_idx],)
 val_players, val_inv_dmats, val_play, val_y, =(X_players_train[val_idx],
inv_dmats_train[val_idx],
X_play_train[val_idx],
y_crps_train[val_idx],)
 w_train = w[train_idx]
n_player = 22
n_factor = 64
se_ratio = 2
n_loop = 1
n_head = 4
n_hidden = 2*n_factor
dropout = 0.25
n_player_cols = len(player_cols)
 n_play_cols = len(play_cols)
 model = get_model(n_player, n_factor, n_loop, n_head, n_hidden, se_ratio, dropout, n_player_cols, n_play_cols)
 if 1:
opm = keras.optimizers.Adam(lr=1e-3)
 model.compile(loss='mse', optimizer=opm, metrics=[])
 es = keras.callbacks.EarlyStopping(monitor='val_loss', mode='min',
restore_best_weights=True, verbose=0, patience=21)
 lr = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=10, verbose=0, mode='min', min_delta=0.00001)
 history = model.fit([train_players, train_inv_dmats, train_play], [train_y, train_y],
verbose=2, batch_size=256, epochs=500,
callbacks=[es,lr],
validation_data=[[val_players, val_inv_dmats, val_play], [val_y, val_y]],
sample_weight=[w_train, w_train])
 X_ir = np.arange(199 ).astype('float64')
 y_pred = model.predict([val_players, val_inv_dmats, val_play])
 y_pred =(y_pred[0]+y_pred[1])/2.
 score0 = crps(y_mae_train[val_idx], y_pred)
 y_pred = post_process(y_pred, yardline_train[val_idx], time_steps)
 score1 = crps(y_mae_train[val_idx][:len(y_pred)], y_pred)
 print(' val fold:', index[0], score0)
 print(' val fold:', index[0], score1)
 if score0 < score1:
break
results[name]['models'].append(model)
 results[name]['crps'].append(score1)
 if simulate_test:
y_pred = model.predict([X_players_test, inv_dmats_test, X_play_test])
 y_pred =(y_pred[0]+y_pred[1])/2.
 score0 = crps(y_mae_test, y_pred)
 y_pred = post_process(y_pred, yardline_test, time_steps)
 score1 = crps(y_mae_test[:len(y_pred)], y_pred)
 print('test fold:', index[0], score0)
 print('test fold:', index[0], score1)
 pred_test.append(y_pred)
 print('*' * 40)
 print('Folds:', results[name]['crps'])
 print('Average:', np.mean(results[name]['crps']))
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
del train_reorient
ts = time.time()
group = matrix.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')
 matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')
 matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<merge>
env = nflrush.make_env()
iter_test = env.iter_test()
ts = time.time()
group = matrix.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')
 matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')
 matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts<data_type_conversions>
for(test_df, sample_prediction_df)in tqdm(iter_test):
test_reorient = reorient(test_df.copy() , flip_left=True)
 test_reorient = augment_data(test_reorient, time_steps)
 df = test_reorient[player_cols]
X_players = df.values.astype('float32')
 X_players = ss_players.transform(df)
 X_players = X_players.reshape(( -1, 22, len(player_cols)))
 X,Y = test_reorient.X.values, test_reorient.Y.values
dmats = [get_dmat(X[i:i+22], Y[i:i+22])for i in range(0, test_reorient.shape[0], 22)]
dmats = np.vstack(dmats ).reshape(( -1, 22, 22))
inv_dmats = 1./(1e-2 + dmats)**2
inv_dmats /= inv_dmats.sum(axis=2, keepdims=True)
 X_play = test_reorient[play_cols].values[::22]
X_play = ss_play.transform(X_play)
 yardline = test_reorient['YardLine_std'].values[::22]
y_preds = [model.predict([X_players, inv_dmats, X_play] 
)for model in results[name]['models']]
y_preds = [(pred[0] + pred[1])/ 2.for pred in y_preds]
y_pred = np.mean(y_preds, axis=0)
 y_pred = post_process(y_pred, yardline, time_steps)
 sample_prediction_df.iloc[0, :] = y_pred.ravel()
env.predict(sample_prediction_df)
 env.write_submission_file()
<merge><EOS> 
X,Y = test_reorient.X.values, test_reorient.Y.values
dmats = [get_dmat(X[i:i+22], Y[i:i+22])for i in range(0, test_reorient.shape[0], 22)]
dmats = np.vstack(dmats ).reshape(( -1, 22, 22))
dmats.shape