Kaggle competition: NFL Big Data Bowl
<SOS> metric: CRPS Kaggle data source: nfl-big-data-bowl<choose_model_class>
fname = "/kaggle/input/nfl-big-data-bowl-2020/train.csv"
class NeuralNet(nn.Module):
def __init__(self, num_aux_targets, num_sentence_features):
super(NeuralNet, self ).__init__()
self.bert = BertModel.from_pretrained(WORK_DIR)
 self.dropout = nn.Dropout(OUT_DROPOUT)
 self.linear_sentence1 = nn.Linear(num_sentence_features, num_sentence_features)
 n_hidden = BERT_HIDDEN_SIZE + num_sentence_features
self.linear1 = nn.Linear(n_hidden, n_hidden)
 self.linear_out = nn.Linear(n_hidden, 1)
 self.linear_aux_out = nn.Linear(n_hidden, num_aux_targets)
 def forward(self, x_features, sentence_features):
_, bert_output = self.bert(*x_features, output_all_encoded_layers=False)
 bert_output = self.dropout(bert_output)
 h_sentence = self.linear_sentence1(sentence_features)
 h_cat = torch.cat(( bert_output, h_sentence), 1)
 h_conc_linear1 = F.relu(self.linear1(h_cat))
hidden = h_cat + h_conc_linear1
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out<init_hyperparams>
def lookup(s):

 dates = {date:pd.to_datetime(date)for date in s.unique() }
return s.map(dates)
 def ageNow(d):
now = datetime.datetime.now()
difference = relativedelta(now, d)
 return(difference.years)
loss_fn=nn.BCEWithLogitsLoss(reduction='sum')
 aux_col_list = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
test_dict = {}
fold_list = [0]
i_epoch = 2
start_time = time.time()
for i_fold in fold_list:
fold_start_time = time.time()
MODEL_PATH = '.. /input/bert-base-uncased-ftpe18-e2-5e6'
print(f'start fold {i_fold}')
 scaler = joblib.load(os.path.join(MODEL_PATH, f'scaler-seed{0}-fold{i_fold}.joblib'))
test_sentence_feature_mat = scaler.transform(sentence_feature_mat)
 test_loader = DynamicBucketIterator(x_test,
torch.tensor(np.concatenate([np.zeros(( len(x_test), 1+len(aux_col_list))), test_sentence_feature_mat], axis=1),dtype=torch.float32 ).cuda() ,
capacity=MAX_LEN*batch_size, pad_token=0, shuffle=False, length_quantile=1, max_batch_size=2048, for_bert=True)
 print(torch.cuda.memory_allocated())
 model = NeuralNet(len(aux_col_list), sentence_feature_mat.shape[-1])
 load_path = os.path.join(MODEL_PATH, f'seed{0}-fold{i_fold}-epoch{i_epoch}.torchModelState')
 model.load_state_dict(torch.load(load_path)['model_state_dict'])
 model.cuda()
model.eval()
epoch_test_pred = np.zeros(len(x_test))
eval_start_time = time.time()
for batch in test_loader:
x_batch = batch[0]
segment_id_batch = batch[1]
input_mask_batch = batch[2]
y_batch = batch[3]
index_batch = batch[4]
y_true_batch = y_batch[:, :1+len(aux_col_list)]
sample_weight_batch = y_batch[:, 1+len(aux_col_list)]
sentence_feature_batch = y_batch[:, -sentence_feature_mat.shape[-1]:]
x_features = [torch.tensor(feature, dtype=torch.long ).cuda() for feature in [x_batch, segment_id_batch, input_mask_batch]]
y_pred = model(x_features, sentence_feature_batch)
 epoch_test_pred[index_batch] = sigmoid(y_pred[:, 0].detach().cpu().numpy())
 del x_features, y_pred
torch.cuda.empty_cache()
print(f'uncased {i_fold} finishd in {time.time() - fold_start_time}', file=open('time.txt', 'a'))
test_dict[i_fold] = epoch_test_pred
del model, epoch_test_pred
gc.collect()
torch.cuda.empty_cache()
print(f'uncased finishd in {time.time() - start_time}', file=open('time.txt', 'a'))<choose_model_class>
clf = []
def randomForest(dfIn,cols):
global clf
np.random.seed(0)
 print("Starting 1")
 clf = RandomForestClassifier(n_jobs=1, random_state=0,n_estimators=10)
 training_df = dfIn[cols][::22]
y=training_df['Yards']
X=training_df[[i for i in training_df.columns if "Yards" not in i]]
clf.fit(X, y)
 print("Trained")
 def modifyXY(dfx):
dfIn = dfx
print('Doing ModifyXY')
 dfIn['XM'] = dfIn['X'] - 10
dfIn['XS'] = dfIn['AbsoluteYardLine']
T = 0.5
print('Doing ModifyXY ',T)
 dfIn['S05'] = dfIn['S'] + dfIn['A'] * T
dfIn['X05'] = dfIn[['X','S','A','Orientation']].apply(lambda x: x['X'] + np.sin(x['Orientation'] *(np.pi/180)) +(x['S'] * T + x['A'] * x['A'] * T / 2.0),axis=1)
 dfIn['X05'] = dfIn[['Y','S','A','Orientation']].apply(lambda x: x['Y'] + np.cos(x['Orientation'] *(np.pi/180)) +(x['S'] * T + x['A'] * x['A'] * T / 2.0),axis=1)
 T = 1
print('Doing ModifyXY',T)
 dfIn['S10'] = dfIn['S'] + dfIn['A'] * T
dfIn['Y10'] = dfIn[['X','S','A','Orientation']].apply(lambda x: x['X'] + np.sin(x['Orientation'] *(np.pi/180)) +(x['S'] * T + x['A'] * x['A'] * T / 2.0),axis=1)
 dfIn['Y10'] = dfIn[['Y','S','A','Orientation']].apply(lambda x: x['Y'] + np.cos(x['Orientation'] *(np.pi/180)) +(x['S'] * T + x['A'] * x['A'] * T / 2.0),axis=1)
 print('Done ModifyXY')
 print(dfIn.columns)
 dfOut = dfIn
return dfOut
def isRequired(strIn):
global flat_list,isTraining
str2 = ".*"+strIn+".*"
if(isTraining):
print("--> isRequired? ",str2,strIn,flat_list)
 r = re.compile(str2)
 l = list(filter(r.match, flat_list))
rc = len(l)
 if rc > 0:
if(isTraining):
print("Field Required = ",strIn)
 return rc
def addFeatures(dfy):
global flatlist,isTraining
dfIn = dfy
map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}
for abb in dfIn['PossessionTeam'].unique() :
map_abbr[abb] = abb
dfIn['PossessionTeam'] = dfIn['PossessionTeam'].map(map_abbr)
 dfIn['HomeTeamAbbr'] = dfIn['HomeTeamAbbr'].map(map_abbr)
 dfIn['VisitorTeamAbbr'] = dfIn['VisitorTeamAbbr'].map(map_abbr)
 dfIn['AbsoluteYardLine'] = dfIn[['YardLine','PlayDirection','HomeTeamAbbr','PossessionTeam','FieldPosition']].apply(lambda x: int(( 100-x['YardLine'])) if x['PossessionTeam'] == x['FieldPosition'] else int(int(x['YardLine'])) ,axis=1)
 dfIn['PlayerIsRusher'] = dfIn['NflIdRusher'] == dfIn['NflId']
if isRequired("XS"):
dfIn = modifyXY(dfIn)
 if isRequired("PlayerWeight"):
dfIn['PlayerWeightSumDefense'] = dfIn['PlayerWeight']
dfIn['PlayerWeightSumOffense'] = dfIn['PlayerWeight']
dfIn.loc[(dfIn['PossessionTeam'] == dfIn['VisitorTeamAbbr'])&(dfIn['Team'] == 'away'), 'PlayerWeightSumDefense'] = 0
dfIn.loc[(dfIn['PossessionTeam'] == dfIn['HomeTeamAbbr'])&(dfIn['Team'] == 'home'), 'PlayerWeightSumDefense'] = 0
dfIn.loc[(dfIn['PossessionTeam'] != dfIn['VisitorTeamAbbr'])&(dfIn['Team'] == 'away'), 'PlayerWeightSumOffense'] = 0
dfIn.loc[(dfIn['PossessionTeam'] != dfIn['HomeTeamAbbr'])&(dfIn['Team'] == 'home'), 'PlayerWeightSumOffense'] = 0
w = dfIn.groupby('PlayId')['PlayerWeightSumDefense','PlayerWeightSumOffense'].sum()
w.rename(columns={'PlayerWeightSumDefense' : 'PlayerWeightSumDefenseSum', 'PlayerWeightSumOffense' : 'PlayerWeightSumOffenseSum'},inplace=True)
 w = w.reset_index()
dfIn.drop(['PlayerWeightSumDefense','PlayerWeightSumOffense'],axis=1,inplace=True)
 dfIn = pd.merge(dfIn,w,how='left',left_on='PlayId',right_on='PlayId')
 dfIn['PlayerWeightSumDiff'] = dfIn['PlayerWeightSumDefenseSum'] - dfIn['PlayerWeightSumOffenseSum']
dfIn2 = dfIn
if isRequired("PlayerBirthDate"): dfIn2['PlayerBirthDate'] = lookup(dfIn2['PlayerBirthDate'])
 if isRequired("PlayerAge"): dfIn2['PlayerAge'] = dfIn2['PlayerBirthDate'].apply(lambda x: ageNow(x))
if isRequired("PlayerYearsInNFL"): dfIn2['PlayerYearsInNFL'] = dfIn2['PlayerAge'] - 22
dfIn2['GameClock2'] = dfIn2['GameClock'].apply(lambda x: int(x[0:2])+int(x[3:5])/60)
 dfIn2['AbsoluteGameClock'] = dfIn2[['GameClock','Quarter']].apply(lambda x: int(x['GameClock'][0:2])+float(x['GameClock'][3:5])/60+(( 4-x['Quarter'])*15),axis=1)
 dfIn2['AbsoluteGameClockRemaining'] = 60 - dfIn2['AbsoluteGameClock']
dfOut = dfIn2
return dfOut
class NeuralNet(nn.Module):
def __init__(self, num_aux_targets, num_sentence_features):
super(NeuralNet, self ).__init__()
self.bert_model = BertModel.from_pretrained(WORK_DIR)
 self.dropout = nn.Dropout(OUT_DROPOUT)
 self.linear_sentence1 = nn.Linear(num_sentence_features, num_sentence_features)
 n_hidden = BERT_HIDDEN_SIZE + num_sentence_features
self.linear1 = nn.Linear(n_hidden, n_hidden)
 self.linear_out = nn.Linear(n_hidden, 1)
 self.linear_aux_out = nn.Linear(n_hidden, num_aux_targets)
 def forward(self, x_features, sentence_features):
_, bert_output = self.bert_model(*x_features, output_all_encoded_layers=False)
 bert_output = self.dropout(bert_output)
 h_sentence = self.linear_sentence1(sentence_features)
 h_cat = torch.cat(( bert_output, h_sentence), 1)
 h_conc_linear1 = F.relu(self.linear1(h_cat))
hidden = h_cat + h_conc_linear1
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out<init_hyperparams>
def predict_using_my_model_RandomForest(dfIn,sample_prediction_df):
global clfs
global dfME
global colsSet
test_dfIn = dfIn[dfIn['NflIdRusher'] == dfIn['NflId']].copy()
test_dfIn = addFeatures(test_dfIn)
 df3 = pd.DataFrame()
c = 0
AbsoluteYardLine = test_dfIn['AbsoluteYardLine'].values[0]
minYards = - AbsoluteYardLine
maxYards = 100 - AbsoluteYardLine
for cols in colsSet:
clf = clfs[c]
c = c + 1
y_df = pd.DataFrame(list(zip(clf.classes_,[x for x in clf.predict_proba(test_dfIn[cols])[0]])))
 i = range(-99,100)
 missingyards = set(i ).difference(sorted(y))
y_df = y_df.append(pd.DataFrame(list(zip(missingyards,[0]*len(missingyards)))))
 y_df.columns = ['Yards','probablity']
y_df = y_df.sort_values('Yards')
 df1 = pd.DataFrame(y_df.set_index('Yards' ).reset_index() ['probablity'] ).T
df1.columns = ["Yards"+str(x)for x in range(-99,100)]
df2 = df1.cumsum(axis=1)
 df3 = pd.concat([df3,df2])
 df4 = pd.DataFrame(df3.sum() ).T
sample_prediction_df = df4.div(df4.max(axis=1), axis=0)
 assert sample_prediction_df['Yards99'].values[0] == 1.0, "Values is not 1.0"
return sample_prediction_df
loss_fn=nn.BCEWithLogitsLoss(reduction='sum')
 aux_col_list = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
test_dict = {}
fold_list = [1, 2]
i_epoch = 1
start_time = time.time()
for i_fold in fold_list:
fold_start_time = time.time()
MODEL_PATH = get_model_path(i_seed, i_fold, i_epoch, '.. /input/base-uncased')
 print(f'start fold {i_fold}')
 scaler = joblib.load(os.path.join(MODEL_PATH, f'scaler-seed{0}-fold{i_fold}.joblib'))
test_sentence_feature_mat = scaler.transform(sentence_feature_mat)
 test_loader = DynamicBucketIterator(x_test,
torch.tensor(np.concatenate([np.zeros(( len(x_test), 1+len(aux_col_list))), test_sentence_feature_mat], axis=1),dtype=torch.float32 ).cuda() ,
capacity=MAX_LEN*batch_size, pad_token=0, shuffle=False, length_quantile=1, max_batch_size=2048, for_bert=True)
 print(torch.cuda.memory_allocated())
 model = NeuralNet(len(aux_col_list), sentence_feature_mat.shape[-1])
 load_path = os.path.join(MODEL_PATH, f'seed{0}-fold{i_fold}-epoch{i_epoch}.torchModelState')
 model.load_state_dict(torch.load(load_path)['model_state_dict'])
 model.cuda()
model.eval()
epoch_test_pred = np.zeros(len(x_test))
eval_start_time = time.time()
for batch in test_loader:
x_batch = batch[0]
segment_id_batch = batch[1]
input_mask_batch = batch[2]
y_batch = batch[3]
index_batch = batch[4]
y_true_batch = y_batch[:, :1+len(aux_col_list)]
sample_weight_batch = y_batch[:, 1+len(aux_col_list)]
sentence_feature_batch = y_batch[:, -sentence_feature_mat.shape[-1]:]
x_features = [torch.tensor(feature, dtype=torch.long ).cuda() for feature in [x_batch, segment_id_batch, input_mask_batch]]
y_pred = model(x_features, sentence_feature_batch)
 epoch_test_pred[index_batch] = sigmoid(y_pred[:, 0].detach().cpu().numpy())
 del x_features, y_pred
torch.cuda.empty_cache()
print(f'uncased {i_fold} finishd in {time.time() - fold_start_time}', file=open('time.txt', 'a'))
test_dict[i_fold] = epoch_test_pred
del model, epoch_test_pred
gc.collect()
torch.cuda.empty_cache()
print(f'uncased finishd in {time.time() - start_time}', file=open('time.txt', 'a')) 
<define_variables>
isTraining = True
os.system(f'echo Training')
 clf = 0
colsSet = [
['Down','JerseyNumber']
]
i = 0
flat_list = list(set([item for sublist in colsSet for item in sublist]))
np.random.seed(0)
 print("Starting 1")
 train = True
if train == True:
filename = fname
dfIn = pd.read_csv(filename, low_memory=False)
 train_df = dfIn
test_df = dfIn
map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}
for abb in train_df['PossessionTeam'].unique() :
map_abbr[abb] = abb
train_df['PossessionTeam'] = train_df['PossessionTeam'].map(map_abbr)
 train_df['HomeTeamAbbr'] = train_df['HomeTeamAbbr'].map(map_abbr)
 train_df['VisitorTeamAbbr'] = train_df['VisitorTeamAbbr'].map(map_abbr)
 train_df = addFeatures(train_df)
 isTraining = False
print('Done adding to train_df')
fold_list = [4, 5]
i_seed = 0
SENTENCE_FEAURE_USED = ['word_count', 'n_unique', 'n_unique_ratio']
batch_size = 32
OUT_DROPOUT = 0.3
BERT_HIDDEN_SIZE = 768
BERT_MODEL_PATH = '.. /input/bert-pretrained-models/cased_l-12_h-768_a-12/cased_L-12_H-768_A-12/'
BERT_DO_LOWER = 'uncased' in BERT_MODEL_PATH
MAX_LEN = 220<load_from_csv>
i = 0
clfs = []
for cols in colsSet:
clf = RandomForestClassifier(n_jobs=1, random_state=0,n_estimators=10)
 Y = ['Yards']
training_df = train_df[train_df['PlayerIsRusher'] == True][cols+Y].copy()
y=training_df['Yards']
X=training_df[[i for i in training_df.columns if "Yards" not in i]]
clf = RandomForestClassifier(n_jobs=2, random_state=0,n_estimators=100)
 print("Training")
 os.system(f'echo training')
 clf.fit(X, y)
 print("Trained")
 os.system(f'echo trained')
 clfs.append(clf)
if DEBUG:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', nrows=DEBUG_DATA_SIZE)
 else:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_test = test['comment_text']
del test
gc.collect()<load_pretrained>
env = nflrush.make_env()
i = 0
for(test_df, sample_prediction_df)in env.iter_test() :
dfPredict = predict_using_my_model_RandomForest(test_df,sample_prediction_df)
 os.system(f'echo {i} ')
 i = i + 1
env.predict(dfPredict)
 env.write_submission_file()
<string_transform><EOS> 
env.write_submission_file()