Kaggle competition: NFL Big Data Bowl
best_params = {'learning_rate': 0.16, 'n_estimators': 500,
'max_depth': 6, 'min_child_weight': 7,
'subsample': 0.9, 'colsample_bytree': 0.7, 'nthread': -1,
'scale_pos_weight': 1, 'random_state': 42,
'tree_method': 'gpu_hist', 'predictor': 'gpu_predictor', 'gpu_id': 0}<train_model>
def error_correcting_codes(df):
df = df.replace('BLT', 'BAL')
 df = df.replace('HST', 'HOU')
 df = df.replace('ARZ', 'ARI')
 df = df.replace('CLV', 'CLE')
 return df
def organize_positions(df):
return(df.loc[(df['PossessionTeam']==df['HomeTeamAbbr'])&(df['Team']=='away')|(df['PossessionTeam']==df['VisitorTeamAbbr'])&(df['Team']=='home')].copy().reset_index() ,
df.loc[(( df['PossessionTeam']==df['HomeTeamAbbr'])&(df['Team']=='home')|(df['PossessionTeam']==df['VisitorTeamAbbr'])&(df['Team']=='away')) &(df['NflId']!=df['NflIdRusher'])].copy().reset_index() ,
df.loc[df['NflId']==df['NflIdRusher']].copy().reset_index())
 def doubledown(X, doublings=1):
np.random.seed(3)
 for w in range(doublings):
X_dupe2 = np.concatenate(( X.copy() , X.copy()), axis=0)
 for i in range(X.shape[0]):
X_dupe2[2*i, :] = X[i, :]
X_dupe2[2*i+1, :] = X[i, :]
X = X_dupe2
return X
def physics_init(df):
way = -2*(df['PlayDirection']=='left')+ 1
theta = way*df['Dir']*np.pi/180
df['X'] =(df['PlayDirection']=='right')*df['X'] +(df['PlayDirection']=='left')*(120 - df['X'])
 df['Sx'] = np.sin(theta)*df['S']
df['Sy'] = np.cos(theta)*df['S']
df['Ax'] = np.sin(theta)*df['A']
df['Ay'] = np.cos(theta)*df['A']
df['EquivYardLine'] =(df['PossessionTeam']==df['FieldPosition'])*(df['YardLine']+10)+(df['PossessionTeam']!=df['FieldPosition'])*(110-df['YardLine'])
 defn, off, RBs = organize_positions(df)
 defn['X'] -= RBs.loc[[i//11 for i in defn.index], 'X'].values
defn['Y'] -= RBs.loc[[i//11 for i in defn.index], 'Y'].values
defn = defn.loc[:,('PlayId', 'PlayerWeight', 'Sx', 'Sy', 'Ax', 'Ay', 'X', 'Y')]
defn.fillna(0, inplace=True)
 defn['Infl'] = defn['PlayerWeight']/(np.square(defn['X'])+ np.square(defn['Y'])) **0.5
defn['AngularMomentum'] = -defn['PlayerWeight']*(defn['X']*defn['Sx'] + defn['Y']*defn['Sy'])/(np.square(defn['X'])+ np.square(defn['Y']))
off['X'] -= RBs.loc[[i//10 for i in off.index], 'X'].values
off['Y'] -= RBs.loc[[i//10 for i in off.index], 'Y'].values
off = off.loc[:,('PlayId', 'PlayerWeight', 'Sx', 'Sy', 'Ax', 'Ay', 'X', 'Y')]
off.fillna(0, inplace=True)
 off['Infl'] = off['PlayerWeight']/(np.square(off['X'])+ np.square(off['Y'])) **0.5
off['AngularMomentum'] = -off['PlayerWeight']*(off['X']*off['Sx'] + off['Y']*off['Sy'])/(np.square(off['X'])+ np.square(off['Y']))
RBs['YardsBehindScrimmage'] = RBs['EquivYardLine'] - RBs['X']
RBs['X'] = 0
RBs['Y'] = 0
RBs = RBs.loc[:,('PlayId', 'YardsBehindScrimmage', 'PlayerWeight', 'Sx', 'Sy', 'Ax', 'Ay', 'X', 'Y')]
RBs.fillna(0, inplace=True)
 return defn, off, RBs
def action(defn, off, RBs, timestep=0.1):
t = 0.0
while t<timestep:
for X in(defn, off, RBs):
X['X'] += X['Sx']*0.01 +X['Ax']*0.01**2/2
X['Y'] += X['Sy']*0.01 +X['Ay']*0.01**2/2
X['Sx'] += X['Ax']*0.01
X['Sy'] += X['Ay']*0.01
X['Ax'] *= 0.99
X['Ay'] *= 0.99
t += 0.01
defn['X'] -= RBs.loc[[i//11 for i in defn.index], 'X'].values
defn['Y'] -= RBs.loc[[i//11 for i in defn.index], 'Y'].values
defn['Infl'] = defn['PlayerWeight']/(np.square(defn['X'])+ np.square(defn['Y'])) **0.5
defn['AngularMomentum'] = -defn['PlayerWeight']*(defn['X']*defn['Sx'] + defn['Y']*defn['Sy'])/(np.square(defn['X'])+ np.square(defn['Y']))
off['X'] -= RBs.loc[[i//10 for i in off.index], 'X'].values
off['Y'] -= RBs.loc[[i//10 for i in off.index], 'Y'].values
off['Infl'] = off['PlayerWeight']/(np.square(off['X'])+ np.square(off['Y'])) **0.5
off['AngularMomentum'] = -off['PlayerWeight']*(off['X']*off['Sx'] + off['Y']*off['Sy'])/(np.square(off['X'])+ np.square(off['Y']))
RBs['X'] = 0
RBs['Y'] = 0
return defn, off, RBs
def physics_doubledown(X, doublings, width):
np.random.seed(3)
 for w in range(doublings):
X_dupe = X.copy()
X_dupe2 = np.concatenate(( X.copy() , X.copy()), axis=0)
 numpy_sucks = np.arange(11)
 np.random.shuffle(numpy_sucks)
 for(i,j)in enumerate(numpy_sucks):
X_dupe[:, width*i:width*i+width] = X[:, width*j:width*j+width]
numpy_sucks = np.arange(10)
 np.random.shuffle(numpy_sucks)
 for(i,j)in enumerate(numpy_sucks):
X_dupe[:, width*(i+11):width*(i+12)] = X[:, width*(j+11):width*(j+12)]
for i in range(X.shape[0]):
X_dupe2[2*i, :] = X[i, :]
X_dupe2[2*i+1, :] = X_dupe[i, :]
X = X_dupe2
return X
def generate_physics(df, forward_action=0, timestep=0.1, doublings=0):
d, o, r = physics_init(df)
 df = None
defn = [d.copy() ]
off = [o.copy() ]
RBs = [r.copy() ]
for a in range(forward_action):
d, o, r = action(d, o, r, timestep)
 defn.append(d.copy())
 off.append(o.copy())
 RBs.append(r.copy())
 d, o, r = None, None, None
for X in(defn, off, RBs):
for i in range(len(defn)) :
X[i]['Px'] = X[i]['Sx']*X[i]['PlayerWeight']
X[i]['Py'] = X[i]['Sy']*X[i]['PlayerWeight']
X[i]['Fx'] = X[i]['Ax']*X[i]['PlayerWeight']
X[i]['Fy'] = X[i]['Ay']*X[i]['PlayerWeight']
for i in range(len(defn)) :
if i==0:
bigD = defn[i].loc[:,('Px', 'Py', 'X', 'Y', 'Infl', 'AngularMomentum')].astype(np.float32)
 bigO = off[i].loc[:,('Px', 'Py', 'X', 'Y', 'Infl', 'AngularMomentum')].astype(np.float32)
 backs = RBs[i].loc[:,('YardsBehindScrimmage', 'Px', 'Py', 'Fx', 'Fy')].astype(np.float32)
 if bigD.shape[0]==0 | bigO.shape[0]==0:
bigD = pd.DataFrame(data=np.random.randn(11,6), columns=('Px', 'Py', 'X', 'Y', 'Infl', 'AngularMomentum'))
bigO = pd.DataFrame(data=np.random.randn(10,6), columns=('Px', 'Py', 'X', 'Y', 'Infl', 'AngularMomentum'))
inst = np.concatenate(( np.reshape(bigD.copy().values,(bigD.shape[0]//11, 11*6)) ,
np.reshape(bigO.copy().values,(bigO.shape[0]//10, 10*6)) ,
backs.copy().values), axis=1)
 summary = physics_doubledown(inst, doublings, 6)
 else:
bigD = defn[i].loc[:,('Infl', 'AngularMomentum')].astype(np.float32)
 bigO = off[i].loc[:,('Infl', 'AngularMomentum')].astype(np.float32)
 if bigD.shape[0]==0 | bigO.shape[0]==0:
bigD = pd.DataFrame(data=np.random.randn(11,2), columns=('Infl', 'AngularMomentum'))
bigO = pd.DataFrame(data=np.random.randn(10,2), columns=('Infl', 'AngularMomentum'))
inst = np.concatenate(( np.reshape(bigD.copy().values,(bigD.shape[0]//11, 11*2)) ,
np.reshape(bigO.copy().values,(bigO.shape[0]//10, 10*2))), axis=1)
 summary = np.concatenate(( summary, physics_doubledown(inst, doublings, 2)) , axis=1)
 defn, off, RBs = None, None, None
return summary
def generate_physics_II(df):
defn, off, RBs = physics_init(df)
 bigD = defn[['PlayerWeight', 'X', 'Y', 'Sx', 'Sy', 'Ax', 'Ay']].astype(np.float32)
 bigO = off[['PlayerWeight', 'X', 'Y', 'Sx', 'Sy', 'Ax', 'Ay']].astype(np.float32)
 backs = RBs[['YardsBehindScrimmage', 'PlayerWeight', 'Sx', 'Sy', 'Ax', 'Ay']].astype(np.float32)
 if bigD.shape[0]==0 | bigO.shape[0]==0:
bigD = pd.DataFrame(data=np.random.randn(11,7), columns=('PlayerWeight', 'X', 'Y', 'Sx', 'Sy', 'Ax', 'Ay'))
bigO = pd.DataFrame(data=np.random.randn(10,7), columns=('PlayerWeight', 'X', 'Y', 'Sx', 'Sy', 'Ax', 'Ay'))
summary = np.concatenate(( backs.copy().values,
np.reshape(bigD.copy().values,(bigD.shape[0]//11, 11*7)) ,
np.reshape(bigO.copy().values,(bigO.shape[0]//10, 10*7))), axis=1)
 defn, off, RBs = None, None, None
bigD, bigO, backs = None, None, None
return summary
def down_situation(df):
X = df.loc[::22,('Down', 'Distance')].copy().astype(np.float32)
 X.fillna(-1, inplace=True)
 framer = pd.DataFrame(columns=(1.0, 2.0, 3.0, 4.0, 'Distance'), dtype=np.float32)
 concatenation = pd.concat(( pd.get_dummies(X['Down']), X['Distance']), axis=1, join='outer')
 concatenation = pd.concat(( framer, concatenation), axis=0, join='outer')
 concatenation.fillna(0, inplace=True)
 return concatenation.values
def RB_wins_it(df):
X = df.loc[::22,('NflIdRusher')]
framer = pd.DataFrame(columns=baxxx, dtype=np.float32)
 OHE = pd.get_dummies(X)
 concatenation = pd.concat(( framer, OHE.loc[:, :framer.shape[1]]), axis=0, join='outer')
 concatenation.fillna(0, inplace=True)
 return concatenation.values.astype(np.float32)
 def stats(array):
return array.mean(axis=0), array.std(axis=0)
 def stats_II(X):
means = X[:, :11].mean(axis=0)
 deviants = X[:, :11].std(axis=0)
 for start, step, amt in [(11,7,11),(88,7,10)]:
repmeans = []
repdevs = []
for i in range(step):
repmeans.append(X[:, start+i:start+step*amt:step].mean())
 repdevs.append(X[:, start+i:start+step*amt:step].std())
 for j in range(amt):
means = np.concatenate(( means, repmeans))
deviants = np.concatenate(( deviants, repdevs))
means = np.concatenate(( means, [0 for i in range(158, 529)]), axis=0)
 deviants = np.concatenate(( deviants, [1 for i in range(158, 529)]), axis=0)
 return means, deviants
def normalize(X, mn, stand):
return(X - mn)/ stand
def generate_yardudge(df):
Y = np.zeros(( df.shape[0]//22, 199))
for(i, yerds)in enumerate(df['Yards'][::22]):
Y[i, yerds+99] = 1
return Y.astype(np.float32 )
ts = time.time()
model = XGBRegressor(**best_params)
 model.fit(X_valid_train.values,
y_valid_train,
eval_metric="rmse",
eval_set=[(X_valid_test.values, y_valid_test)],
verbose=True,
early_stopping_rounds = 50)
 time.time() - ts<drop_column>
joblib.dump(model, 'xgboost_model.pkl')
 del model
del X_train
del X_test
del y_train
del y_test
del X_valid_train
del X_valid_test
del y_valid_train
del y_valid_test
gc.collect()<load_pretrained>
global meen_II, sigma,_II, baxxx
train_df = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', low_memory=False)
 baxxx = train_df['NflIdRusher'].unique()
Y = generate_yardudge(train_df)
 train_df = error_correcting_codes(train_df)
 X = np.concatenate(( down_situation(train_df),
generate_physics_II(train_df),
RB_wins_it(train_df)) , axis=1)
 train_df = None
meen_II, sigma_II = stats_II(X)
 X = normalize(X, meen_II, sigma_II)
 gc.collect()
all_data = pd.read_pickle('all_data.pkl' )<prepare_x_and_y>
stuff = [4.81725572e-02, 1.48089908e-05, 9.57455511e-01, 1.58078791e+00,
9.93707682e-02, 5.04421303e-01, 6.51849665e+01, 9.68255755e+02,
5.14218542e+02, 5.09462973e+02, 5.17812944e+02, 8.04146527e+02,
1.64480385e+02, 9.80945456e+02, 8.50852251e+01, 1.03668046e+03,
6.57080119e+01]
learning_rate = stuff[0]
lr_decay =stuff[1]
beta = stuff[2]
clipse = stuff[3]
drop_rate1 = stuff[4]
drop_rate2 = stuff[5]
epochs = int(round(stuff[6]))
batch_size = int(round(stuff[7]))
hidden_layer1_nodes = int(round(stuff[8]))
hidden_layer2_nodes = int(round(stuff[9]))
hidden_layer3_nodes = int(round(stuff[10]))
function_layer1_nodes = int(round(stuff[11]))
function_layer2_nodes = int(round(stuff[12]))
function_layer3_nodes = int(round(stuff[13]))
function_layer4_nodes = int(round(stuff[14]))
function_layer5_nodes = int(round(stuff[15]))
function_layer6_nodes = int(round(stuff[16]))
ipt = [Input(shape=(11,)) ] + [Input(shape=(7,)) for i in range(11)] + \
[Input(shape=(7,)) for i in range(10)] + [Input(shape=(baxxx.shape[0],)) ]
np.random.seed(314159)
 expanse = Dense(function_layer1_nodes, activation='relu')
 collapsor = Dense(function_layer2_nodes, activation='linear')
 expanse2 = Dense(function_layer3_nodes, activation='relu')
 collapsor2 = Dense(function_layer4_nodes, activation='linear')
 faux_embedding_expanse = Dense(function_layer5_nodes, activation='relu')
 faux_embedding_collapsor = Dense(function_layer6_nodes, activation='linear')
 D = Add()([collapsor(expanse(ipt[i])) for i in range(1,12)])
 O = Add()([collapsor(expanse(ipt[i])) for i in range(12,22)])
 R = faux_embedding_collapsor(faux_embedding_expanse(ipt[22]))
p = concatenate([ipt[0], D, O, R], axis=-1)
 x = Dropout(drop_rate1 )(p)
 x = Dense(hidden_layer1_nodes, activation='relu' )(x)
 x = Dropout(drop_rate2 )(x)
 x = Dense(hidden_layer2_nodes, activation='relu' )(x)
 x = Dropout(drop_rate2 )(x)
 x = Dense(hidden_layer3_nodes, activation='relu' )(x)
 x = Dropout(drop_rate2 )(x)
 output = Dense(199, activation='softmax' )(x)
 model_NN2 = Model(inputs=ipt, outputs=output)
 model_NN2.compile(optimizer=SGD(lr=learning_rate, decay=lr_decay, momentum=beta, nesterov=True, clipnorm=clipse), loss=categorical_crossentropy)
 model_NN2.fit([X[:,:11]]+[X[:,7*i+11:7*i+18] for i in range(0,11)]+
[X[:,7*i+11:7*i+18] for i in range(11,21)]+[X[:,158:]],
Y, epochs=epochs, batch_size=batch_size)
 gc.collect()
def compute_prediction_for_specifi_month(monthes_before, df, estimator, params, prefix):
predictions = pd.DataFrame(columns=['date_block_num', 'pred_'+prefix])
 for before in monthes_before:
last_valid_month = np.max(df['date_block_num'])
 print('train: 12 to {0}'.format(last_valid_month-before-1))
print('test: {0}'.format(last_valid_month-before))
cur_train = df[df['date_block_num'] < last_valid_month-before]
cur_test = df[df['date_block_num'] == last_valid_month-before]
cur_y_train = cur_train['target']
cur_train.drop('target', axis=1, inplace=True)
 cur_test.drop('target', axis=1, inplace=True)
 model = estimator(**params)
 model.fit(cur_train.values, cur_y_train.values)
 pred = model.predict(cur_test.values)
 cur_df = pd.DataFrame(columns=['date_block_num', 'pred_'+prefix])
 cur_df['pred_'+prefix] = pred
cur_df['date_block_num'] =(last_valid_month-before)
 predictions = pd.concat([predictions, cur_df])
 del model
del pred
del cur_test
del cur_df
del cur_y_train
gc.collect()
return predictions<find_best_params>
best_param = {
'learning_rate' :0.16,
'n_estimators':500,
'max_depth':6,
'min_child_weight':7,
'subsample':0.9,
'colsample_bytree':0.7,
'nthread':-1,
'scale_pos_weight':1,
'random_state':42,
'nthread': -1,
'tree_method':'gpu_hist',
'predictor':'gpu_predictor',
'gpu_id': 0,
}
ts= time.time()
monthes_before = [6, 5, 4, 3, 2, 1, 0]
stacks_xgb = compute_prediction_for_specifi_month(monthes_before, all_data, XGBRegressor, best_params,'xgb')
 time.time() -ts<feature_engineering>
Yxgb = np.argmax(Y, axis=1)- 99
model_xgb1 = XGBClassifier(objective='multi:softmax', seed=27182818, max_depth=1)
 model_xgb1.fit(X, Yxgb )
stacks_xgb['pred_xgb'] = stacks_xgb['pred_xgb'].clip(0, 20 )<find_best_params>
ts = time.time()
monthes_before = [6, 5, 4, 3, 2, 1, 0]
best_param ={'alpha': 2000}
stacks_lin_reg = compute_prediction_for_specifi_month(monthes_before, all_data, Ridge, best_param,'lin')
 time.time() - ts<feature_engineering>
stacks_lin_reg['pred_lin'] = stacks_lin_reg['pred_lin'].clip(0, 20 )<prepare_x_and_y>
env = nflrush.make_env()
X_valid_train = pd.concat([stacks_xgb[stacks_xgb['date_block_num'] < 33],
stacks_lin_reg[stacks_lin_reg['date_block_num'] < 33]], axis=1)
 X_valid_test = pd.concat([stacks_xgb[stacks_xgb['date_block_num'] == 33],
stacks_lin_reg[stacks_lin_reg['date_block_num'] == 33]], axis=1)
 y_valid_train = all_data.loc[(all_data['date_block_num']<33)&(all_data['date_block_num']>27), 'target']
y_valid_test = all_data.loc[all_data['date_block_num']==33, 'target']<prepare_x_and_y>
def make_my_predictions(model1, model2, model3, test_df, sample_predictions):
test_df = error_correcting_codes(test_df)
 X = np.concatenate(( down_situation(test_df.copy()),
generate_physics_II(test_df.copy()),
RB_wins_it(test_df.copy())) , axis=1)
 X = normalize(X, meen_II, sigma_II)
 my_predictions_II = model2.predict([X[:,:11]]+[X[:,7*i+11:7*i+18] for i in range(0,11)]+
[X[:,7*i+11:7*i+18] for i in range(11,21)]+[X[:,158:]])
 offset = int(( model3.predict(X)- np.argmax(model3.predict_proba(X), axis=1)) [0])
 my_predictions_III = model3.predict_proba(X)
 my_predictions_III = np.concatenate(( np.zeros(( 1, 99+offset)) , my_predictions_III, np.ones(( 1, 100-my_predictions_III.shape[1]-offset))), axis=1)
 my_predictions = np.reshape(0.62*my_predictions_II + 0.38*my_predictions_III,(199,0))
cucumber = 0
for i in range(1, my_predictions.shape[0]-1):
cucumber += np.maximum(my_predictions[i], 0)
 sample_predictions.iloc[0, i] = np.minimum(cucumber, 1)
 return sample_predictions
X_train = pd.concat([stacks_xgb[stacks_xgb['date_block_num'] < 34],
stacks_lin_reg[stacks_lin_reg['date_block_num'] < 34]], axis=1)
 X_test = pd.concat([stacks_xgb[stacks_xgb['date_block_num'] == 34],
stacks_lin_reg[stacks_lin_reg['date_block_num'] == 34]], axis=1)
 y_train = all_data.loc[(all_data['date_block_num']<34)&(all_data['date_block_num']>27), 'target']<drop_column>
for(test_df, sample_prediction_df)in env.iter_test() :
predictions_df = make_my_predictions(model_xgb1, model_NN2, model_xgb1, test_df, sample_prediction_df)
 env.predict(predictions_df)
 env.write_submission_file()