Kaggle competition: NFL Big Data Bowl
<SOS> metric: CRPS Kaggle data source: nfl-big-data-bowl<create_dataframe>
sns.set_style('darkgrid')
 mpl.rcParams['figure.figsize'] = [15,10]
batch_size = 1024
train_dataset = data.TensorDataset(x_train_torch, y_train_torch)
 valid_dataset = data.TensorDataset(x_train_torch[:batch_size], y_train_torch[:batch_size])
 test_dataset = data.TensorDataset(x_test_torch)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)
 databunch = DataBunch(train_dl=train_loader,valid_dl=valid_loader )<compute_test_metric>
env = nflrush.make_env()
def custom_loss(data, targets):

 bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2] )(data[:,:1],targets[:,:1])
 bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])
 return(bce_loss_1 * loss_weight)+ math.sqrt(bce_loss_2 )<choose_model_class>
train = pd.read_csv('.. /input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'} )
all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1234 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 learn = Learner(databunch,model,loss_func=custom_loss)
 test_preds = train_model(learn,test_dataset,output_dim=7)
 all_test_preds.append(test_preds )<save_to_csv>
train['DefendersInTheBox_vs_Distance'] = train['DefendersInTheBox'] / train['Distance']
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
})
 submission.to_csv('submission.csv', index=False )<define_variables>
train['StadiumType'].value_counts()
use_own_text = True
use_train_data = False<define_variables>
def clean_StadiumType(txt):
if pd.isna(txt):
return np.nan
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = re.sub(' +', ' ', txt)
 txt = txt.strip()
txt = txt.replace('outside', 'outdoor')
 txt = txt.replace('outdor', 'outdoor')
 txt = txt.replace('outddors', 'outdoor')
 txt = txt.replace('outdoors', 'outdoor')
 txt = txt.replace('oudoor', 'outdoor')
 txt = txt.replace('indoors', 'indoor')
 txt = txt.replace('ourdoor', 'outdoor')
 txt = txt.replace('retractable', 'rtr.')
 return txt
BERT_MODEL_PATHS = [
'bert_bert_sb_0.93405_out_1_cpa.pt',
'bert_wss_boosted2_cpa_nlabels31_0.93774.pt',
'bert_wss_aux_ot_rt_0.93803.pt',
'bert_sb1t9_no_identity_ot_rt_cont_rand_sb1t9_0.93574_out_13_cpa.pt',
'bert_base_cont_boosted3_sge6_0.93672_out_12_cpa.pt',
'bert_base_last_year_competition_cont_pp_no_identity_cont__0.94643_out_1_cpa.pt',
'bert_sb_mini_epoch_0.93451_out_8_cpa.pt',
'bert_wss_aux_ot_0.93722.pt',
'bert_public_0.93373.pt',
'bert_subgroup_balanced_20p_target_ist_bert_ga2_0.93370.pt',
'bert_yuval_baseline_bert_model_cont_boost_0.93352.pt',
'bert_base_0.93327_out_1_cpa.pt',
]
BERT_MODEL_PATHS = [DATASET_PATH+bmp for bmp in BERT_MODEL_PATHS]<define_variables>
train['StadiumType'] = train['StadiumType'].apply(clean_StadiumType )
BERT_MODEL_PATHS.append(DATASET_PATH2+'bert_beLSTM_sb2t8_0.91725.pt')
 BERT_MODEL_PATHS.append(DATASET_PATH2+'bert_beLSTM_aux_wss_bad_minor_0.91675.pt')
 BERT_MODEL_PATHS.append(DATASET_PATH2+'bert_beLSTM_aux_wss_0.92162.pt' )<define_variables>
def transform_StadiumType(txt):
if pd.isna(txt):
return np.nan
if 'outdoor' in txt or 'open' in txt:
return 1
if 'indoor' in txt or 'closed' in txt:
return 0
return np.nan
package_dir = '.. /input/pytorch-pretrained-bert-master/pytorch-pretrained-bert-master/pytorch-pretrained-BERT-master/'
sys.path.append(package_dir )<init_hyperparams>
train['StadiumType'] = train['StadiumType'].apply(transform_StadiumType )
args = {
'device' : 'cuda',
'learning_rate' : 25e-6,
'warmup_proportion': 0.2,
'batch_size' : 32,
'ebbs_epoch_samples' : 25e3,
'num_eebs_epochs' : 15e6//100e3,
'ebbs_test_multipl' : 1,
'eval_sample_size' : 10e3,
'num_labels' : 1,
'max_seq_length' : 250,
'gradient_accumulation_steps' : 2,
'fp16' : False,
'seed' : 70135
}<define_variables>
Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural', 'UBU Sports Speed S5-M':'Artificial',
'Artificial':'Artificial', 'DD GrassMaster':'Artificial', 'Natural Grass':'Natural',
'UBU Speed Series-S5-M':'Artificial', 'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural',
'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural', 'Field turf':'Artificial',
'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial', 'natural grass':'Natural'}
train['Turf'] = train['Turf'].map(Turf)
 train['Turf'] = train['Turf'] == 'Natural'
label_clms = ['target','severe_toxicity', 'obscene','identity_attack',
'insult', 'threat','funny', 'wow','sad','sexual_explicit','rating']
identity_columns = ['male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
target_clms = [*label_clms,*identity_columns]<set_options>
train[(train['PossessionTeam']!=train['HomeTeamAbbr'])&(train['PossessionTeam']!=train['VisitorTeamAbbr'])][['PossessionTeam', 'HomeTeamAbbr', 'VisitorTeamAbbr']]
warnings.filterwarnings(action='once')
 device = torch.device('cuda' )<string_transform>
map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}
for abb in train['PossessionTeam'].unique() :
map_abbr[abb] = abb
def replace_underscore_with_backspace(txt):

 return re.sub('_+',' ',txt)
 def replace_newline_with_dot(txt):

 return re.sub('
+','.',txt)
 def replace_newline_with_space(txt):

 return re.sub('
+',' ',txt)
 def remove_urls(txt):

 return re.sub(r"(http|ftp|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+
def remove_multi_symbols(txt,symbols=string.punctuation):

 for symbol in symbols:
txt = re.sub('\'+symbol+'+',symbol,txt)
 return txt<load_from_csv>
train['PossessionTeam'] = train['PossessionTeam'].map(map_abbr)
 train['HomeTeamAbbr'] = train['HomeTeamAbbr'].map(map_abbr)
 train['VisitorTeamAbbr'] = train['VisitorTeamAbbr'].map(map_abbr )
test_df = pd.read_csv(".. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv")
 if use_train_data:
test_df = pd.read_csv(".. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv" ).sample(10000 )<define_variables>
train['HomePossesion'] = train['PossessionTeam'] == train['HomeTeamAbbr']
if use_own_text:
my_text = [
'asdf',
'feck-you',
'trolling the nation',
'I am proud to be gay',
'Nothing to do here',
'Hey, keep your Islamophobia out of public domains.',
'Well, the money the education system spent on you was wasted',
'This clown confuses the word "substance" with "abuse".',
'"Two things are infinite: the universe and human stupidity; and I´m not sure about the universe." - Albert Einstein',
'I have epilepsy',
'I am a woman with epilepsy',
'I am a person who uses a wheelchair',
'I use a wheelchair',
'I am gay and have to use a wheelchair',
'I am blind',
'I am a person who is deaf',
'I am a woman who is deaf',
'I am a non-binary-gender person who is deaf',
'I am a black person who is deaf',
'I am pround to be gay',
'I am a proud homo',
'I am a proud lesbian',
'I am a proud to be black',
'KKK',
'a55hle',
'a$$hole',
'I am a very proud lesbian',
'I am a blind homosexual.',
"I don't feel ashamed for beeing lesbian",
"Proud to be a fu**ing gay person",
"Totally wrong to think that this wouldn't be gay as shit",
' ',
'!',
'',
'???',
'I hate trump',
'I hate obama',
'Trump',
'I like that you are gay',
]
test_df = pd.DataFrame({
'id': range(len(my_text)) ,
'comment_text': my_text
} )<feature_engineering>
train['Field_eq_Possession'] = train['FieldPosition'] == train['PossessionTeam']
train['HomeField'] = train['FieldPosition'] == train['HomeTeamAbbr']
test_df['comment_text_orig'] = test_df['comment_text']<feature_engineering>
off_form = train['OffenseFormation'].unique()
train['OffenseFormation'].value_counts()
functions = [
remove_urls,
replace_underscore_with_backspace,
replace_newline_with_dot,
lambda k: remove_multi_symbols(k,symbols=['.','-','='])
 ]
for func in functions:
print(func.__name__)
 test_df['comment_text'] = test_df['comment_text'].apply(func )<categorify>
train = pd.concat([train.drop(['OffenseFormation'], axis=1), pd.get_dummies(train['OffenseFormation'], prefix='Formation')], axis=1)
 dummy_col = train.columns
def convert_lines(example, max_seq_length,tokenizer):
max_seq_length -=2
all_tokens = np.zeros(( len(example),max_seq_length+2))
token_len = np.zeros(len(example))
for i,text in enumerate(example):
tokens_a = tokenizer.tokenize(text)
 if len(tokens_a)>max_seq_length:
tokens_a = tokens_a[:max_seq_length]
tokens = tokenizer.convert_tokens_to_ids(tokens_a)
 tokens = [101, *tokens, 102]
token_len[i] = len(tokens)
 all_tokens[i,:len(tokens)] = np.asarray(tokens)
 return all_tokens, token_len<load_pretrained>
train['GameClock'].value_counts()
SEED = 1234
TOKENIZER_PATH = '.. /input/berttokenizer/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
tokenizer = BertTokenizer.from_pretrained(TOKENIZER_PATH, cache_dir=None,do_lower_case=True )<data_type_conversions>
train['GameClock'] = train['GameClock'].apply(strtoseconds )
test_df['comment_text'] = test_df['comment_text'].astype(str)
 X_test, token_len = convert_lines(test_df["comment_text"], args['max_seq_length'], tokenizer )<feature_engineering>
train['PlayerHeight'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
test_df['token_len'] = token_len<import_modules>
train['PlayerBMI'] = 703*(train['PlayerWeight']/(train['PlayerHeight'])**2 )
device = torch.device('cuda')
 class BertForMultiLabelSequenceClassification(BertForSequenceClassification):

 def __init__(self, config, num_labels=2):
super(BertForMultiLabelSequenceClassification,
self ).__init__(config, num_labels)
 self.num_labels = num_labels
self.bert = BertModel(config)
 self.dropout = torch.nn.Dropout(config.hidden_dropout_prob)
 self.classifier = torch.nn.Linear(config.hidden_size, num_labels)
 self.apply(self.init_bert_weights)
 def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None, return_pooled_output=False, return_last_unpooled_layer=False):
full_output, pooled_output = self.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False)
 logits = self.classifier(pooled_output)
 if labels is not None:
loss_fct = BCEWithLogitsLoss()
mask =(labels != -1 ).to(dtype=torch.float)
 logits = logits * mask
label_tensor = labels * mask
loss = loss_fct(logits, labels)
 return loss, logits
elif return_pooled_output: return logits, pooled_output
elif return_last_unpooled_layer: return logits, full_output
return logits
def forward_old(self, input_ids, token_type_ids=None, attention_mask=None, labels=None, loss_weight=None):
_, pooled_output = self.bert(
 input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False)
 pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 if labels is not None:
loss_fct = BCEWithLogitsLoss()
mask =(labels != -1 ).to(dtype=torch.float)
 logits = logits * mask
labels = labels * mask
loss = loss_fct(logits.view(-1, self.num_labels),
labels.view(-1, self.num_labels))
return loss
else:
return logits
def freeze_bert_encoder(self):
for param in self.bert.parameters() :
param.requires_grad = False
def unfreeze_bert_encoder(self):
for param in self.bert.parameters() :
param.requires_grad = True
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class BertLSTM1(nn.Module):

 def __init__(self, embedding_matrix, num_labels, n_aux_units = 7, max_features=args['max_seq_length'],LSTM_UNITS=128,DENSE_HIDDEN_UNITS=4*128):
super(BertLSTM2, self ).__init__()
self.num_labels = num_labels
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, num_labels)
 def forward(self, input_ids, attention_mask=None, labels=None, loss_weight=None, return_pooled_output=False):
h_embedding = self.embedding(input_ids)
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 return result<load_pretrained>
train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
def load_bert_model(path, num_labels=args['num_labels']):
state_dict = torch.load(path)
 if path.find('.. /input/bertlstm-weights/')==0:
print('loading bertLSTM model')
 if num_labels is None:
num_labels = int(state_dict[list(state_dict.keys())[-1]].shape[0])
 print('Number of Labels set to',num_labels)
 try:
model = BertLSTM1(np.zeros(( 30522, 768)) ,num_labels)
 print(model.load_state_dict(state_dict))
except:
print('trying BertLSTM2')
 model = BertLSTM2(np.zeros(( 30522, 768)) ,num_labels)
 print(model.load_state_dict(state_dict))
model = model.to('cuda')
 del state_dict
return model
config = BertConfig(30522)
 if num_labels is None:
if 'classifier.bias' in state_dict.keys() :
num_labels = int(state_dict['classifier.bias'].shape[0])
 else:
print('No classifier.bias in state_dict --> assuming num_labels = 1')
 num_labels = 1
model = BertForMultiLabelSequenceClassification(config,num_labels=num_labels)
 model.load_state_dict(state_dict)
 model.to(device)
 del state_dict
return model<load_pretrained>
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1 )
out_clms = []
for ii, BERT_MODEL_PATH in enumerate(BERT_MODEL_PATHS):
print('
using model',BERT_MODEL_PATH)
 model = load_bert_model(BERT_MODEL_PATH,num_labels=None)
 for param in model.parameters() :
param.requires_grad = False
model.eval()
buckets = 30
old_thld = -1
for b in range(buckets):
thld = int(np.percentile(test_df.token_len,(b+1)*100//buckets))
mask =(test_df.token_len>old_thld)&(test_df.token_len<=thld)
 if mask.sum() >0:
batch_size = args['batch_size'] if thld<=220 else 16
old_thld = thld
print('\rBucketNr',b,'thld:',thld,'samples:',mask.sum() ,end='')
 preds = None
test = torch.utils.data.TensorDataset(torch.tensor(X_test[mask,:thld], dtype=torch.long))
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 for i,(x_batch,)in enumerate(test_loader):
logits = model(x_batch.to(device), attention_mask=(x_batch > 0 ).to(device), labels=None)
 y_pred = logits.sigmoid().detach().cpu().numpy()
if preds is None: preds = y_pred
else: preds = np.concatenate(( preds,y_pred),axis=0)
 test_df.loc[mask,'bo'+str(ii)] = preds[:,0]
out_clms.append('bo'+str(ii))<feature_engineering>
train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
weights = np.ones(len(out_clms))
print([(BERT_MODEL_PATHS[i].split('/')[-1],weights[i])for i in range(len(weights)) ])
 test_df.fillna(0,inplace=True)
 test_df['prediction'] = np.average(test_df[out_clms].values, weights=weights, axis=1 )<feature_engineering>
seconds_in_year = 60*60*24*365.25
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1 )
use_ranked_sum = False
if use_ranked_sum:
for key in out_clms:
test_df[key + '_rank'] = test_df[key].rank()
test_df['rank_sum'] = np.sum(
 test_df[col] for col in test_df.columns if '_rank' in col)
 test_df['rank_sum_prediction'] = test_df['rank_sum']/(len(out_clms)*
test_df.shape[0] )<define_variables>
train = train.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate'], axis=1 )
if use_train_data:
identity_columns = ['male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
target_columns = ['severe_toxicity', 'obscene','identity_attack', 'insult', 'threat']
offtarget_columns = ['funny', 'wow','sad', 'likes', 'disagree', 'sexual_explicit']
TOXICITY_COLUMN = 'target'
SUBGROUP_AUC = 'subgroup_auc'
BPSN_AUC = 'bpsn_auc'
BNSP_AUC = 'bnsp_auc'
def convert_to_bool(df, col_name):
df[col_name] = np.where(df[col_name] >= 0.5, True, False)
 def convert_dataframe_to_bool(df):
bool_df = df.copy()
for col in ['target'] + identity_columns:
convert_to_bool(bool_df, col)
 return bool_df
def compute_auc(y_true, y_pred):
try:
return metrics.roc_auc_score(y_true, y_pred)
 except ValueError:
return np.nan
def compute_subgroup_auc(df, subgroup, label, model_name):
subgroup_examples = df[df[subgroup]]
return compute_auc(subgroup_examples[label], subgroup_examples[model_name])
 def compute_bpsn_auc(df, subgroup, label, model_name):

 subgroup_negative_examples = df[df[subgroup] & ~df[label]]
non_subgroup_positive_examples = df[~df[subgroup] & df[label]]
examples = subgroup_negative_examples.append(non_subgroup_positive_examples)
 return compute_auc(examples[label], examples[model_name])
 def compute_bnsp_auc(df, subgroup, label, model_name):

 subgroup_positive_examples = df[df[subgroup] & df[label]]
non_subgroup_negative_examples = df[~df[subgroup] & ~df[label]]
examples = subgroup_positive_examples.append(non_subgroup_negative_examples)
 return compute_auc(examples[label], examples[model_name])
 def compute_bias_metrics_for_model(dataset,subgroups,model,label_col,include_asegs=False):

 dataset = convert_dataframe_to_bool(dataset)
 records = []
for subgroup in subgroups:
record = {
'subgroup': subgroup,
'subgroup_size': len(dataset[dataset[subgroup]])
 }
record[SUBGROUP_AUC] = compute_subgroup_auc(dataset, subgroup, label_col, model)
 record[BPSN_AUC] = compute_bpsn_auc(dataset, subgroup, label_col, model)
 record[BNSP_AUC] = compute_bnsp_auc(dataset, subgroup, label_col, model)
 records.append(record)
 return pd.DataFrame(records ).sort_values('subgroup_auc', ascending=True)
 def calculate_overall_auc(df, model_name, TOXICITY_COLUMN):
true_labels = df[TOXICITY_COLUMN]>=0.5
predicted_labels = df[model_name]
return metrics.roc_auc_score(true_labels, predicted_labels)
 def power_mean(series, p):
total = sum(np.power(series, p))
return np.power(total / len(series), 1 / p)
 def get_final_metric(bias_df, overall_auc, POWER=-5, OVERALL_MODEL_WEIGHT=0.25):
bias_score = np.average([
power_mean(bias_df[SUBGROUP_AUC], POWER),
power_mean(bias_df[BPSN_AUC], POWER),
power_mean(bias_df[BNSP_AUC], POWER)
 ])
 return(OVERALL_MODEL_WEIGHT * overall_auc)+(( 1 - OVERALL_MODEL_WEIGHT)* bias_score), bias_score
def get_scores(in_df, model_name,TOXICITY_COLUMN='target',fillna=True,consolidation_method='mean'):
df = in_df[[model_name,'id',*identity_columns, TOXICITY_COLUMN]].groupby('id')
 if consolidation_method=='mean': df = df.mean()
if consolidation_method=='max': df = df.max()
if consolidation_method=='min': df = df.min()
bias_metrics_df = compute_bias_metrics_for_model(df, identity_columns, model_name, TOXICITY_COLUMN)
 if fillna: bias_metrics_df.fillna(0.5,inplace=True)
 try:
overall_auc = calculate_overall_auc(df, model_name, TOXICITY_COLUMN)
 except ValueError:
print('Error - only one class present in df')
 overall_auc = np.nan
final_metric, bias_score = get_final_metric(bias_metrics_df, overall_auc)
 bias_metrics_df['overall_auc']=overall_auc
bias_metrics_df['final_bias_score'] = bias_score
bias_metrics_df['final_metric'] = final_metric
bias_metrics_df.index = bias_metrics_df.subgroup
return bias_metrics_df<compute_test_metric>
train['WindSpeed'].value_counts()
if use_train_data:
score = get_scores(test_df,'prediction' )<compute_train_metric>
train['WindSpeed'] = train['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x )
if use_train_data:
for i,clm in enumerate(out_clms):
print(i, str(get_scores(test_df,clm ).final_metric.mean())[:5],weights[i],BERT_MODEL_PATHS[i].split('/')[-1],sep='\t' )<save_to_csv>
train['WindSpeed'].value_counts()
for out_clm in ['prediction',*out_clms]:
submission = pd.DataFrame.from_dict({
'id': test_df['id'],
'prediction': test_df[out_clm]
})
 if out_clm=='prediction': out_clm = 'blend'
submission.to_csv('submission_'+out_clm+'.csv', index=False )<save_to_csv>
train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 train['WindSpeed'] = train['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x )
test_df.to_csv('test_df.csv', index=False )<concatenate>
def str_to_float(txt):
try:
return float(txt)
 except:
return -1
pd.concat(( test_df['comment_text'],(test_df[[*out_clms,'prediction']]*100 ).astype(np.int)) ,axis=1 )<define_variables>
train['WindSpeed'] = train['WindSpeed'].apply(str_to_float )
DEBUG = False
batch_size = 256
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 2 * LSTM_UNITS
n_seeds = 1
n_splits = 10
n_epochs = 15
EMBED_SIZE = 300
SUBGROUP_NEGATIVE_WEIGHT_COEF = 1
BACKGROUND_POSITIVE_WEIGHT_COEF = 0
ENSEMBLE_START_EPOCH = 3
MAX_LEN = 300
EMB_DROPOUT = 0.3
MIDDLE_DROPOUT = 0.3
if DEBUG:
DEBUG_DATA_SIZE = 1000<set_options>
train['WindDirection'].value_counts()
%matplotlib inline<set_options>
def clean_WindDirection(txt):
if pd.isna(txt):
return np.nan
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = txt.replace('from', '')
 txt = txt.replace(' ', '')
 txt = txt.replace('north', 'n')
 txt = txt.replace('south', 's')
 txt = txt.replace('west', 'w')
 txt = txt.replace('east', 'e')
 return txt
@contextmanager
def suppress_stdout() :
with open(os.devnull, "w")as devnull:
old_stdout = sys.stdout
sys.stdout = devnull
try:
yield
finally:
sys.stdout = old_stdout<define_variables>
train['WindDirection'] = train['WindDirection'].apply(clean_WindDirection )
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', '
', '\r']
def clean_text(x: str)-> str:
for punct in puncts:
if punct in x:
x = x.replace(punct, ' {} '.format(punct))
return x
def clean_numbers(x):
return re.sub('\d+', ' ', x)
 if DEBUG:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', nrows=DEBUG_DATA_SIZE)
 else:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_test = test['comment_text'].apply(lambda x: clean_text(x))
gc.collect()<define_variables>
train['WindDirection'].value_counts()
aux_col_list = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']<load_pretrained>
def transform_WindDirection(txt):
if pd.isna(txt):
return np.nan
if txt=='n':
return 0
if txt=='nne' or txt=='nen':
return 1/8
if txt=='ne':
return 2/8
if txt=='ene' or txt=='nee':
return 3/8
if txt=='e':
return 4/8
if txt=='ese' or txt=='see':
return 5/8
if txt=='se':
return 6/8
if txt=='ses' or txt=='sse':
return 7/8
if txt=='s':
return 8/8
if txt=='ssw' or txt=='sws':
return 9/8
if txt=='sw':
return 10/8
if txt=='sww' or txt=='wsw':
return 11/8
if txt=='w':
return 12/8
if txt=='wnw' or txt=='nww':
return 13/8
if txt=='nw':
return 14/8
if txt=='nwn' or txt=='nnw':
return 15/8
return np.nan
%%time
with open('.. /input/reducing-oov-of-crawl300d2m-no-appos-result/jigsaw-crawl-300d-2M.joblib', 'rb')as f:
crawl_emb_dict = joblib.load(f )<string_transform>
train['WindDirection'] = train['WindDirection'].apply(transform_WindDirection )
def build_vocab(texts: pd.DataFrame)-> Dict[str, int]:

 sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in tqdm(sentences):
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def check_coverage(vocab: Dict[str, int], embeddings_index: Dict)-> List[str]:

 known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in tqdm(vocab.keys()):
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(known_words)/ float(len(vocab))))
print('Found embeddings for {:.2%} of all text'.format(float(nb_known_words)/(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words
p_stemmer = PorterStemmer()
l_stemmer = LancasterStemmer()
s_stemmer = SnowballStemmer("english")
 def edits1(word):

 letters = 'abcdefghijklmnopqrstuvwxyz'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts)
 def known(words, embed):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in embed)
 def spellcheck(word, word_rank_dict):
return min(known(edits1(word), word_rank_dict), key=lambda w: word_rank_dict[w])
 punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "",
"€": "e", "™": "tm", "√": " sqrt ", "×": "x",
"²": "2", "—": "-", "–": "-", "’": "'",
"_": "-", "`": "'", '“': '"', '”': '"',
'“': '"', "£": "e", '∞': 'infinity',
'θ': 'theta', '÷': '/', 'α': 'alpha',
'•': '.', 'à': 'a', '−': '-', 'β': 'beta',
'∅': '', '³': '3', 'π': 'pi', }
def process_stemmer(vocab, embed):
oov_word_set = set()
for word in tqdm(vocab.keys()):
vector = embed.get(word, None)
 if vector is not None:
continue
vector = embed.get(word.lower() , None)
 if vector is not None:
embed[word] = vector
continue
vector = embed.get(word.upper() , None)
 if vector is not None:
embed[word] = vector
continue
vector = embed.get(word.capitalize() , None)
 if vector is not None:
embed[word] = vector
continue
corr_word = punct_mapping.get(word, None)
 if corr_word is not None:
vector = embed.get(corr_word, None)
 if vector is not None:
embed[word] = vector
continue
try:
vector = embed.get(p_stemmer.stem(word), None)
 except:
vector = embed.get(p_stemmer.stem(word.decode('utf-8')) , None)
 if vector is not None:
embed[word] = vector
continue
try:
vector = embed.get(l_stemmer.stem(word), None)
 except:
vector = embed.get(l_stemmer.stem(word.decode('utf-8')) , None)
 if vector is not None:
embed[word] = vector
continue
try:
vector = embed.get(s_stemmer.stem(word), None)
 except:
vector = embed.get(s_stemmer.stem(word.decode('utf-8')) , None)
 if vector is not None:
embed[word] = vector
continue
oov_word_set.add(word)
 return embed, oov_word_set
def process_small_capital(vocab, embed, oov_set):
oov_word_set = set()
for word in tqdm(vocab.keys()):
if word not in oov_set:
continue
char_list = []
any_small_capitial = False
for char in word:
try:
uni_name = unicodedata.name(char)
 except ValueError:
continue
if 'LATIN LETTER SMALL CAPITAL' in uni_name:
char = uni_name[-1]
any_small_capitial = True
if 'CYRILLIC SMALL LETTER GHE WITH STROKE' in uni_name:
char = 'F'
any_small_capitial = True
char_list.append(char)
 if not any_small_capitial:
oov_word_set.add(word)
 continue
legit_word = ''.join(char_list)
 vector = embed.get(legit_word, None)
 if vector is not None:
embed[word] = vector
continue
vector = embed.get(legit_word.lower() , None)
 if vector is not None:
embed[word] = vector
continue
vector = embed.get(legit_word.upper() , None)
 if vector is not None:
embed[word] = vector
continue
vector = embed.get(legit_word.capitalize() , None)
 if vector is not None:
embed[word] = vector
continue
corr_word = punct_mapping.get(legit_word, None)
 if corr_word is not None:
vector = embed.get(corr_word, None)
 if vector is not None:
embed[word] = vector
continue
try:
vector = embed.get(p_stemmer.stem(legit_word), None)
 except:
vector = embed.get(p_stemmer.stem(legit_word.decode('utf-8')) , None)
 if vector is not None:
embed[word] = vector
continue
try:
vector = embed.get(l_stemmer.stem(legit_word), None)
 except:
vector = embed.get(l_stemmer.stem(legit_word.decode('utf-8')) , None)
 if vector is not None:
embed[word] = vector
continue
try:
vector = embed.get(s_stemmer.stem(legit_word), None)
 except:
vector = embed.get(s_stemmer.stem(legit_word.decode('utf-8')) , None)
 if vector is not None:
embed[word] = vector
continue
oov_word_set.add(word)
 return embed, oov_word_set
def process_spellcheck(vocab, embed, word_rank_dict, oov_set):
oov_word_set = set()
for word in tqdm(vocab.keys()):
if word not in oov_set:
continue
try:
vector = embed.get(spellcheck(word, word_rank_dict), None)
 except:
continue
if vector is not None:
embed[word] = vector
continue
oov_word_set.add(word)
 return embed, oov_word_set
def make_word_rank(embed):
word_rank = {}
for i, word in enumerate(embed):
word_rank[word] = i
return word_rank
def head(enumerable, n=10):
for i, item in enumerate(enumerable):
print(str(i)+ '
', item)
 if i > n:
return 
<compute_test_metric>
train['PlayDirection'].value_counts()
%%time
vocab = build_vocab(x_test)
 oov = check_coverage(vocab, crawl_emb_dict)
 head(oov )<feature_engineering>
train['PlayDirection'] = train['PlayDirection'].apply(lambda x: x.strip() == 'right' )
%%time
emb, oov_stemmer = process_stemmer(vocab, crawl_emb_dict)
 oov = check_coverage(vocab, emb)
 head(oov )<statistical_test>
train['Team'] = train['Team'].apply(lambda x: x.strip() =='home' )
%%time
emb, oov_small_capital = process_small_capital(vocab, emb, oov_stemmer)
 oov = check_coverage(vocab, emb)
 head(oov )<compute_test_metric>
train['GameWeather'] = train['GameWeather'].str.lower()
indoor = "indoor"
train['GameWeather'] = train['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('clear and sunny', 'sunny and clear')if not pd.isna(x)else x)
 train['GameWeather'] = train['GameWeather'].apply(lambda x: x.replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x )
%%time
word_rank = make_word_rank(emb)
 emb, oov_spell = process_spellcheck(vocab, emb, word_rank, oov_small_capital)
 oov = check_coverage(vocab, emb)
 head(oov )<string_transform>
weather_count = Counter()
for weather in train['GameWeather']:
if pd.isna(weather):
continue
for word in weather.split() :
weather_count[word]+=1
weather_count.most_common() [:15]
with open('.. /input/googleprofanitywords/list.txt', 'r')as f:
p_words = f.readlines()
set_puncts = set(puncts)
 p_word_set = set([t.replace('
', '')for t in p_words])
 def sentence_fetures(text):
word_list = text.split()
word_count = len(word_list)
 n_upper = len([word for word in word_list if any([c.isupper() for c in word])])
 n_unique = len(set(word_list))
n_ex = word_list.count('!')
 n_que = word_list.count('?')
 n_puncts = len([word for word in word_list if word in set_puncts])
 n_prof = len([word for word in word_list if word in p_word_set])
 n_oov = len([word for word in word_list if word not in crawl_emb_dict])
 return word_count, n_upper, n_unique, n_ex, n_que, n_puncts, n_prof, n_oov
sentence_feature_cols = ['word_count', 'n_upper', 'n_unique', 'n_ex', 'n_que', 'n_puncts', 'n_prof', 'n_oov']
feature_dict = defaultdict(list)
 for text in progress_bar(x_test):
feature_list = sentence_fetures(text)
 for i_feature, feature_name in enumerate(sentence_feature_cols):
feature_dict[sentence_feature_cols[i_feature]].append(feature_list[i_feature])
 sentence_df = pd.DataFrame.from_dict(feature_dict)
 for col in ['n_upper', 'n_unique', 'n_ex', 'n_que', 'n_puncts', 'n_prof', 'n_oov']:
sentence_df[col + '_ratio'] = sentence_df[col] / sentence_df['word_count']
sentence_df.head() 
<drop_column>
def map_weather(txt):
ans = 1
if pd.isna(txt):
return 0
if 'partly' in txt:
ans*=0.5
if 'climate controlled' in txt or 'indoor' in txt:
return ans*3
if 'sunny' in txt or 'sun' in txt:
return ans*2
if 'clear' in txt:
return ans
if 'cloudy' in txt:
return -ans
if 'rain' in txt or 'rainy' in txt:
return -2*ans
if 'snow' in txt:
return -3*ans
return 0
sentence_feature_mat = sentence_df.drop(columns=['n_oov', 'n_oov_ratio'] ).values
gc.collect()<feature_engineering>
train['GameWeather'] = train['GameWeather'].apply(map_weather )
%%time
x_test = x_test.apply(lambda x: clean_numbers(x))
tokenizer = preprocessing.text.Tokenizer(filters="", lower=False)
 tokenizer.fit_on_texts(list(x_test))
x_test = tokenizer.texts_to_sequences(x_test)
 max_features = len(tokenizer.word_index)+ 1
max_features
def build_matrix(word_index, embedding_dict):
embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_dict[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words
emb_mat_crawl, oov = build_matrix(tokenizer.word_index, emb)
 del crawl_emb_dict, oov, emb
gc.collect()
emb_mat_torch = torch.tensor(emb_mat_crawl, dtype=torch.float32 ).cuda()
del emb_mat_crawl
gc.collect() 
<define_variables>
train['IsRusher'] = train['NflId'] == train['NflIdRusher']
class DynamicBucketIterator(object):
def __init__(self, data, label, capacity, pad_token, shuffle, length_quantile, max_batch_size, for_bert):
self.data = data
self.label = label
self.pad_token = pad_token
self.capacity = capacity
self.shuffle = shuffle
self.length_quantile = length_quantile
self.for_bert = for_bert
self.index_sorted = sorted(range(len(self.data)) , key=lambda i: len(self.data[i]))
old_separator_index = 0
self.separator_index_list = [0]
for i_sample in range(len(self.data)) :
sample_index = self.index_sorted[i_sample]
sample = self.data[sample_index]
current_batch_size = i_sample - old_separator_index + 1
if min(len(sample), MAX_LEN)* current_batch_size <= self.capacity and current_batch_size <= max_batch_size:
pass
else:
old_separator_index = i_sample
self.separator_index_list.append(i_sample)
 self.separator_index_list.append(len(self.data))
if not self.shuffle:
self.bucket_index = range(self.__len__())
 self.reset_index()
def reset_index(self):
self.i_batch = 0
if self.shuffle:
self.index_sorted = sorted(np.random.permutation(len(self.data)) , key=lambda i: len(self.data[i]))
self.bucket_index = np.random.permutation(self.__len__())
 def __len__(self):
return len(self.separator_index_list)- 1
def __iter__(self):
return self
def __next__(self):
try:
i_bucket = self.bucket_index[self.i_batch]
except IndexError as e:
self.reset_index()
raise StopIteration
start_index, end_index = self.separator_index_list[i_bucket : i_bucket + 2]
index_batch = self.index_sorted[start_index : end_index]
raw_batch_data = [self.data[i] for i in index_batch]
batch_label = self.label[index_batch]
math.ceil(1)
 max_len = int(math.ceil(np.quantile([len(x)for x in raw_batch_data], self.length_quantile)))
 max_len = min([max_len, MAX_LEN])
 if max_len == 0:
max_len = 1
if self.for_bert:
segment_id_batch = np.zeros(( len(raw_batch_data), max_len))
padded_batch = []
input_mask_batch = []
for sample in raw_batch_data:
input_mask = [1] * len(sample)+ [0] *(max_len - len(sample))
input_mask_batch.append(input_mask[:max_len])
 sample = sample + [self.pad_token for _ in range(max_len - len(sample)) ]
padded_batch.append(sample[:max_len])
 self.i_batch += 1
return padded_batch, segment_id_batch, input_mask_batch, batch_label, index_batch
else:
padded_batch = []
for sample in raw_batch_data:
sample = sample + [self.pad_token for _ in range(max_len - len(sample)) ]
padded_batch.append(sample[:max_len])
 self.i_batch += 1
return padded_batch, batch_label, index_batch<normalization>
train.drop(['NflId', 'NflIdRusher'], axis=1, inplace=True )
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class GRULayer(nn.Module):
def __init__(self, input_size, hidden_size):
super(GRULayer, self ).__init__()
self.gru = nn.GRU(input_size=input_size,
hidden_size=hidden_size,
bias=True,
bidirectional=True,
batch_first=True)
 self.init_weights()
def init_weights(self):
ih =(param.data for name, param in self.named_parameters() if 'weight_ih' in name)
 hh =(param.data for name, param in self.named_parameters() if 'weight_hh' in name)
 b =(param.data for name, param in self.named_parameters() if 'bias' in name)
 for k in ih:
nn.init.xavier_uniform_(k)
 for k in hh:
nn.init.orthogonal_(k)
 for k in b:
nn.init.constant_(k, 0)
 def forward(self, x):
gru_outputs, gru_state = self.gru(x)
 return gru_outputs, gru_state
class NeuralNet(nn.Module):
def __init__(self, embed_size, num_aux_targets, num_sentence_features):
super(NeuralNet, self ).__init__()
self.embedding_dropout = SpatialDropout(EMB_DROPOUT)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.middle_dropout = SpatialDropout(MIDDLE_DROPOUT)
 self.lstm2 = GRULayer(LSTM_UNITS * 2, LSTM_UNITS)
 self.linear_sentence1 = nn.Linear(num_sentence_features, num_sentence_features)
 n_hidden = LSTM_UNITS * 2 + num_sentence_features
self.linear1 = nn.Linear(n_hidden, n_hidden)
 self.linear_out = nn.Linear(n_hidden, 1)
 self.linear_aux_out = nn.Linear(n_hidden, num_aux_targets)
 def forward(self, x, sentence_features):
h_embedding = self.embedding_dropout(x)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm1 = self.middle_dropout(h_lstm1)
 h_lstm2, _ = self.lstm2(h_lstm1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_sentence = self.linear_sentence1(sentence_features)
 h_cat = torch.cat(( max_pool, h_sentence), 1)
 h_conc_linear1 = F.relu(self.linear1(h_cat))
hidden = h_cat + h_conc_linear1
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out<compute_test_metric>
train['X'] = train.apply(lambda row: row['X'] if row['PlayDirection'] else 120-row['X'], axis=1 )
def sigmoid(x):
return 1 /(1 + np.exp(-x))
OOF_TRAIN_COL = 'oof_train'
SUBGROUP_AUC_COL = 'subgroup_auc'
BPSN_AUC_COL = 'bpsn_auc'
BNSP_AUC_COL = 'bnsp_auc'
def compute_auc(y_true, y_pred):
try:
return metrics.roc_auc_score(y_true, y_pred)
 except ValueError:
return np.nan
'model'
def compute_subgroup_auc(df, subgroup_col, label_col, oof_col):
subgroup_examples = df[df[subgroup_col]]
return compute_auc(subgroup_examples[label_col], subgroup_examples[oof_col])
 def compute_bpsn_auc(df, subgroup_col, label_col, oof_col):

 subgroup_negative_examples = df[df[subgroup_col] & ~df[label_col]]
non_subgroup_positive_examples = df[~df[subgroup_col] & df[label_col]]
examples = subgroup_negative_examples.append(non_subgroup_positive_examples)
 return compute_auc(examples[label_col], examples[oof_col])
 def compute_bnsp_auc(df, subgroup_col, label_col, oof_col):

 subgroup_positive_examples = df[df[subgroup_col] & df[label_col]]
non_subgroup_negative_examples = df[~df[subgroup_col] & ~df[label_col]]
examples = subgroup_positive_examples.append(non_subgroup_negative_examples)
 return compute_auc(examples[label_col], examples[oof_col])
 def compute_bias_metrics_for_model(df,
subgroup_list,
oof_col,
label_col,
include_asegs=False):

 record_list = []
for subgroup in subgroup_list:
record = {
'subgroup': subgroup,
'subgroup_size': len(df[df[subgroup]])
 }
record[SUBGROUP_AUC_COL] = compute_subgroup_auc(df, subgroup, label_col, oof_col)
 record[BPSN_AUC_COL] = compute_bpsn_auc(df, subgroup, label_col, oof_col)
 record[BNSP_AUC_COL] = compute_bnsp_auc(df, subgroup, label_col, oof_col)
 record_list.append(record)
 return pd.DataFrame(record_list ).sort_values('subgroup_auc', ascending=True)
 TOXICITY_COLUMN = 'target'
identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
def convert_to_bool(df, col_name):
df[col_name] = np.where(df[col_name] >= 0.5, True, False)
 def convert_dataframe_to_bool(df):
bool_df = df.copy()
for col in ['target'] + identity_columns:
convert_to_bool(bool_df, col)
 return bool_df
def calculate_overall_auc(df, model_name):
true_labels = df[TOXICITY_COLUMN]
predicted_labels = df[model_name]
return metrics.roc_auc_score(true_labels, predicted_labels)
 def power_mean(series, p):
total = sum(np.power(series, p))
return np.power(total / len(series), 1 / p)
 def get_final_metric(bias_df, overall_auc, POWER=-5, OVERALL_MODEL_WEIGHT=0.25):
bias_score = np.average([
power_mean(bias_df[SUBGROUP_AUC_COL], POWER),
power_mean(bias_df[BPSN_AUC_COL], POWER),
power_mean(bias_df[BNSP_AUC_COL], POWER)
 ])
 return(OVERALL_MODEL_WEIGHT * overall_auc)+(( 1 - OVERALL_MODEL_WEIGHT)* bias_score)
 def get_various_auc(valid_df, y_pred):
valid_df = convert_dataframe_to_bool(valid_df.fillna(0))
valid_df.loc[:, OOF_TRAIN_COL] = y_pred
valid_df = convert_dataframe_to_bool(valid_df.fillna(0))
bias_metrics_df = compute_bias_metrics_for_model(valid_df, identity_columns, OOF_TRAIN_COL, TOXICITY_COLUMN)
 overall_auc = calculate_overall_auc(valid_df, OOF_TRAIN_COL)
 return get_final_metric(bias_metrics_df, overall_auc), overall_auc, bias_metrics_df<set_options>
def new_orientation(angle, play_direction):
if play_direction == 0:
new_angle = 360.0 - angle
if new_angle == 360.0:
new_angle = 0.0
return new_angle
else:
return angle
train['Orientation'] = train.apply(lambda row: new_orientation(row['Orientation'], row['PlayDirection']), axis=1)
 train['Dir'] = train.apply(lambda row: new_orientation(row['Dir'], row['PlayDirection']), axis=1 )
gc.collect()
torch.cuda.empty_cache()
torch.cuda.memory_allocated()<normalization>
train['YardsLeft'] = train.apply(lambda row: 100-row['YardLine'] if row['HomeField'] else row['YardLine'], axis=1)
 train['YardsLeft'] = train.apply(lambda row: row['YardsLeft'] if row['PlayDirection'] else 100-row['YardsLeft'], axis=1 )
loss_fn=nn.BCEWithLogitsLoss(reduction='sum')
 auc_array = np.zeros(( n_seeds, n_splits, n_epochs))
test_array = np.zeros(( n_seeds, n_splits, n_epochs, len(x_test)))
 kf = KFold(n_splits=n_splits, shuffle=True)
 for i_seed in range(n_seeds):
print(f'start seed {i_seed}')
 for i_fold,(dev_index, val_index)in enumerate(kf.split(x_test)) :
print(f'start fold {i_fold}')
 scaler = joblib.load(os.path.join(MODEL_PATH, f'scaler-seed{i_seed}-fold{i_fold}.joblib'))
test_sentence_feature_mat = scaler.transform(sentence_feature_mat)
 test_loader = DynamicBucketIterator(x_test,
torch.tensor(np.concatenate([np.zeros(( len(x_test), 1+len(aux_col_list))),
test_sentence_feature_mat], axis=1),dtype=torch.float32 ).cuda() ,
capacity=MAX_LEN*batch_size, pad_token=0, shuffle=False,
length_quantile=1, max_batch_size=10000000, for_bert=False)
 print(torch.cuda.memory_allocated())
 for i_epoch in range(ENSEMBLE_START_EPOCH, n_epochs):
start_time = time.time()
model = NeuralNet(EMBED_SIZE, len(aux_col_list), sentence_feature_mat.shape[-1])
 load_path = os.path.join(MODEL_PATH, f'seed{i_seed}-fold{i_fold}-epoch{i_epoch}.torchModelState')
 model.load_state_dict(torch.load(load_path))
model.cuda()
model.eval()
epoch_test_pred = np.zeros(len(x_test))
mem_list = []
for batch in test_loader:
x_batch = batch[0]
y_batch = batch[1]
index_batch = batch[2]
y_true_batch = y_batch[:, :1+len(aux_col_list)]
sample_weight_batch = y_batch[:, 1+len(aux_col_list)]
sentence_feature_batch = y_batch[:, -sentence_feature_mat.shape[-1]:]
if len(x_batch)>= 32:
y_pred = model(emb_mat_torch[x_batch], sentence_feature_batch)
 else:
len_last = len(x_batch)
 x_batch.extend([[0 for _ in range(len(x_batch[0])) ] for _ in range(32)])
 y_pred = model(emb_mat_torch[x_batch[:32]][:len_last], sentence_feature_batch)
 mem_list.append(torch.cuda.memory_allocated())
 epoch_test_pred[index_batch] = sigmoid(y_pred[:, 0].detach().cpu().numpy())
 del y_pred
torch.cuda.empty_cache()
test_array[i_seed, i_fold, i_epoch] = epoch_test_pred
elapsed_time = time.time() - start_time
del model
gc.collect()
torch.cuda.empty_cache()
print(f'epoch {i_epoch}: {elapsed_time: 0.3f}, {max(mem_list)/10**9}')
 del epoch_test_pred
gc.collect()
torch.cuda.empty_cache()<set_options>
((train['YardsLeft']<train['Yards'])|(train['YardsLeft']-100>train['Yards'])).mean()
del test_loader, emb_mat_torch, test_array
gc.collect()
torch.cuda.empty_cache()
torch.cuda.memory_allocated()<define_variables>
train.drop(train.index[(train['YardsLeft']<train['Yards'])|(train['YardsLeft']-100>train['Yards'])], inplace=True )
i_seed = 0
SENTENCE_FEAURE_USED = ['word_count', 'n_unique', 'n_unique_ratio']
batch_size = 32
OUT_DROPOUT = 0.3
BERT_HIDDEN_SIZE = 768
MAX_LEN = 220<set_options>
train = train.sort_values(by=['PlayId', 'Team', 'IsRusher', 'JerseyNumber'] ).reset_index()
tqdm.pandas()
%matplotlib inline<split>
train.drop(['GameId', 'PlayId', 'index', 'IsRusher', 'Team'], axis=1, inplace=True )
sentence_feature_mat = sentence_df[SENTENCE_FEAURE_USED].values
gc.collect()<load_from_csv>
cat_features = []
for col in train.columns:
if train[col].dtype =='object':
cat_features.append(col)
 train = train.drop(cat_features, axis=1 )
%%time
if DEBUG:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', nrows=DEBUG_DATA_SIZE)
 else:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_test = test['comment_text']
del test
gc.collect()<set_options>
train.fillna(-999, inplace=True )
logging.basicConfig(level=logging.INFO )<choose_model_class>
X_train = np.array(train[players_col] ).reshape(-1, len(players_col)*22 )
tokenizer = GPT2Tokenizer(vocab_file='.. /input/gpt2-models/vocab.json', merges_file='.. /input/gpt2-models/merges.txt' )<sort_values>
play_col = train.drop(players_col+['Yards'], axis=1 ).columns
X_play_col = np.zeros(shape=(X_train.shape[0], len(play_col)))
 for i, col in enumerate(play_col):
X_play_col[:, i] = train[col][::22]
%%time
x_test = x_test.progress_apply(lambda x: tokenizer.encode(x[:MAX_LEN]))<choose_model_class>
X_train = np.concatenate([X_train, X_play_col], axis=1)
 y_train = np.zeros(shape=(X_train.shape[0], 199))
for i,yard in enumerate(train['Yards'][::22]):
y_train[i, yard+99:] = np.ones(shape=(1, 100-yard))
class NeuralNet(nn.Module):
def __init__(self, num_aux_targets, num_sentence_features):
super(NeuralNet, self ).__init__()
self.gpt2 = GPT2Model.from_pretrained('.. /input/gpt2-models/')
 self.dropout = nn.Dropout(OUT_DROPOUT)
 self.linear_sentence1 = nn.Linear(num_sentence_features, num_sentence_features)
 n_hidden = BERT_HIDDEN_SIZE + num_sentence_features
self.linear1 = nn.Linear(n_hidden, n_hidden)
 self.linear_out = nn.Linear(n_hidden, 1)
 self.linear_aux_out = nn.Linear(n_hidden, num_aux_targets)
 def forward(self, x_features, sentence_features):
h_gpt2, _ = self.gpt2(x_features)
 h_gpt2, _ = torch.max(h_gpt2, 1)
 h_sentence = self.linear_sentence1(sentence_features)
 h_cat = torch.cat(( h_gpt2, h_sentence), 1)
 h_conc_linear1 = F.relu(self.linear1(h_cat))
hidden = h_cat + h_conc_linear1
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out<normalization>
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train )
loss_fn=nn.BCEWithLogitsLoss(reduction='sum')
 aux_col_list = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']
test_dict = {}
fold_list = [1]
i_epoch = 1
start_time = time.time()
for i_fold in fold_list:
fold_start_time = time.time()
print(f'start fold {i_fold}')
 scaler = joblib.load(os.path.join(MODEL_PATH, f'scaler-seed{0}-fold{i_fold}.joblib'))
test_sentence_feature_mat = scaler.transform(sentence_feature_mat)
 test_loader = DynamicBucketIterator(x_test,
torch.tensor(np.concatenate([np.zeros(( len(x_test), 1+len(aux_col_list))), test_sentence_feature_mat], axis=1),dtype=torch.float32 ).cuda() ,
capacity=MAX_LEN*batch_size, pad_token=0, shuffle=False, length_quantile=1, max_batch_size=2048, for_bert=False)
 print(torch.cuda.memory_allocated())
 model = NeuralNet(len(aux_col_list), sentence_feature_mat.shape[-1])
 load_path = os.path.join(MODEL_PATH, f'seed{0}-fold{i_fold}-epoch{i_epoch}.torchModelState')
 model.load_state_dict(torch.load(load_path)['model_state_dict'])
 model.cuda()
model.eval()
epoch_test_pred = np.zeros(len(x_test))
eval_start_time = time.time()
for batch in progress_bar(test_loader):
x_batch = torch.tensor(batch[0], dtype=torch.long ).cuda()
y_batch = batch[1]
index_batch = batch[2]
y_true_batch = y_batch[:, :1+len(aux_col_list)]
sample_weight_batch = y_batch[:, 1+len(aux_col_list)]
sentence_feature_batch = y_batch[:, -sentence_feature_mat.shape[-1]:]
y_pred = model(x_batch, sentence_feature_batch)
 epoch_test_pred[index_batch] = sigmoid(y_pred[:, 0].detach().cpu().numpy())
 del x_batch, y_pred
torch.cuda.empty_cache()
print(f'uncased {i_fold} finishd in {time.time() - fold_start_time}', file=open('time.txt', 'a'))
test_dict[i_fold] = epoch_test_pred
del model, epoch_test_pred
gc.collect()
torch.cuda.empty_cache()
print(f'uncased finishd in {time.time() - start_time}', file=open('time.txt', 'a')) 
<set_options>
batch_size=64
del test_loader, test_dict
gc.collect()
torch.cuda.empty_cache()
torch.cuda.memory_allocated()<define_variables>
class RAdam(keras.optimizers.Optimizer):

 def __init__(self, learning_rate=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., weight_decay=0., amsgrad=False,
total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):
learning_rate = kwargs.pop('lr', learning_rate)
 super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.learning_rate = K.variable(learning_rate, name='learning_rate')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.weight_decay = K.variable(weight_decay, name='weight_decay')
 self.total_steps = K.variable(total_steps, name='total_steps')
 self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')
 self.min_lr = K.variable(min_lr, name='min_lr')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.initial_weight_decay = weight_decay
self.initial_total_steps = total_steps
self.amsgrad = amsgrad
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = K.maximum(self.total_steps - warmup_steps, 1)
 decay_rate =(self.min_lr - lr)/ decay_steps
lr = K.switch(
 t <= warmup_steps,
lr *(t / warmup_steps),
lr + decay_rate * K.minimum(t - warmup_steps, decay_steps), 
)
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for(i, p)in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for(i, p)in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for(i, p)in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params)) ]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
 beta_2_t = K.pow(self.beta_2, t)
 sma_inf = 2.0 /(1.0 - self.beta_2)- 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t /(1.0 - beta_2_t)
 for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 m_corr_t = m_t /(1.0 - beta_1_t)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 v_corr_t = K.sqrt(vhat_t /(1.0 - beta_2_t))
self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t /(1.0 - beta_2_t))
r_t = K.sqrt(( sma_t - 4.0)/(sma_inf - 4.0)* 
(sma_t - 2.0)/(sma_inf - 2.0)*
sma_inf / sma_t)
 p_t = K.switch(sma_t >= 5, r_t * m_corr_t /(v_corr_t + self.epsilon), m_corr_t)
 if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
@property
def lr(self):
return self.learning_rate
@lr.setter
def lr(self, learning_rate):
self.learning_rate = learning_rate
def get_config(self):
config = {
'learning_rate': float(K.get_value(self.learning_rate)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.weight_decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad,
'total_steps': float(K.get_value(self.total_steps)) ,
'warmup_proportion': float(K.get_value(self.warmup_proportion)) ,
'min_lr': float(K.get_value(self.min_lr)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
i_seed = 0
SENTENCE_FEAURE_USED = ['word_count', 'n_unique', 'n_unique_ratio']
batch_size = 32
OUT_DROPOUT = 0.3
BERT_HIDDEN_SIZE = 768
BERT_MODEL_PATH = '.. /input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'
BERT_DO_LOWER = 'uncased' in BERT_MODEL_PATH
MAX_LEN = 220 
<set_options>
def crps(y_true, y_pred):
return K.mean(K.square(y_true - K.cumsum(y_pred, axis=1)) , axis=1 )
tqdm.pandas()
%matplotlib inline<load_from_csv>
def get_model() :
x = keras.layers.Input(shape=[X_train.shape[1]])
 fc1 = keras.layers.Dense(units=450, input_shape=[X_train.shape[1]] )(x)
 act1 = keras.layers.PReLU()(fc1)
 bn1 = keras.layers.BatchNormalization()(act1)
 dp1 = keras.layers.Dropout(0.55 )(bn1)
 gn1 = keras.layers.GaussianNoise(0.15 )(dp1)
 concat1 = keras.layers.Concatenate()([x, gn1])
 fc2 = keras.layers.Dense(units=600 )(concat1)
 act2 = keras.layers.PReLU()(fc2)
 bn2 = keras.layers.BatchNormalization()(act2)
 dp2 = keras.layers.Dropout(0.55 )(bn2)
 gn2 = keras.layers.GaussianNoise(0.15 )(dp2)
 concat2 = keras.layers.Concatenate()([concat1, gn2])
 fc3 = keras.layers.Dense(units=400 )(concat2)
 act3 = keras.layers.PReLU()(fc3)
 bn3 = keras.layers.BatchNormalization()(act3)
 dp3 = keras.layers.Dropout(0.55 )(bn3)
 gn3 = keras.layers.GaussianNoise(0.15 )(dp3)
 concat3 = keras.layers.Concatenate([concat2, gn3])
 output = keras.layers.Dense(units=199, activation='softmax' )(concat2)
 model = keras.models.Model(inputs=[x], outputs=[output])
 return model
def train_model(X_train, y_train, X_val, y_val):
model = get_model()
model.compile(optimizer=RAdam(warmup_proportion=0.1, min_lr=1e-7), loss=crps)
 er = EarlyStopping(patience=20, min_delta=1e-4, restore_best_weights=True, monitor='val_loss')
 model.fit(X_train, y_train, epochs=200, callbacks=[er], validation_data=[X_val, y_val], batch_size=batch_size)
 return model
%%time
if DEBUG:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', nrows=DEBUG_DATA_SIZE)
 else:
test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_test = test['comment_text']
del test
gc.collect()<split>
rkf = RepeatedKFold(n_splits=5, n_repeats=5)
 models = []
for tr_idx, vl_idx in rkf.split(X_train, y_train):
x_tr, y_tr = X_train[tr_idx], y_train[tr_idx]
x_vl, y_vl = X_train[vl_idx], y_train[vl_idx]
model = train_model(x_tr, y_tr, x_vl, y_vl)
 models.append(model )
sentence_feature_mat = sentence_df[SENTENCE_FEAURE_USED].values
gc.collect()<set_options>
def make_pred(df, sample, env, models):
df['StadiumType'] = df['StadiumType'].apply(clean_StadiumType)
 df['StadiumType'] = df['StadiumType'].apply(transform_StadiumType)
 df['DefendersInTheBox_vs_Distance'] = df['DefendersInTheBox'] / df['Distance']
df['OffenseFormation'] = df['OffenseFormation'].apply(lambda x: x if x in off_form else np.nan)
 df = pd.concat([df.drop(['OffenseFormation'], axis=1), pd.get_dummies(df['OffenseFormation'], prefix='Formation')], axis=1)
 missing_cols = set(dummy_col)- set(df.columns)-set('Yards')
 for c in missing_cols:
df[c] = 0
df = df[dummy_col]
df.drop(['Yards'], axis=1, inplace=True)
 df['Turf'] = df['Turf'].map(Turf)
 df['Turf'] = df['Turf'] == 'Natural'
df['PossessionTeam'] = df['PossessionTeam'].map(map_abbr)
 df['HomeTeamAbbr'] = df['HomeTeamAbbr'].map(map_abbr)
 df['VisitorTeamAbbr'] = df['VisitorTeamAbbr'].map(map_abbr)
 df['HomePossesion'] = df['PossessionTeam'] == df['HomeTeamAbbr']
df['Field_eq_Possession'] = df['FieldPosition'] == df['PossessionTeam']
df['HomeField'] = df['FieldPosition'] == df['HomeTeamAbbr']
df['GameClock'] = df['GameClock'].apply(strtoseconds)
 df['PlayerHeight'] = df['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))
df['PlayerBMI'] = 703*(df['PlayerWeight']/(df['PlayerHeight'])**2)
 df['TimeHandoff'] = df['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeSnap'] = df['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
df['TimeDelta'] = df.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1)
 df['PlayerBirthDate'] = df['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))
seconds_in_year = 60*60*24*365.25
df['PlayerAge'] = df.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x: x.lower().replace('mph', '' ).strip() if not pd.isna(x)else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split('-')[0])+int(x.split('-')[1])) /2 if not pd.isna(x)and '-' in x else x)
 df['WindSpeed'] = df['WindSpeed'].apply(lambda x:(int(x.split() [0])+int(x.split() [-1])) /2 if not pd.isna(x)and type(x)!=float and 'gusts up to' in x else x)
 df['WindSpeed'] = df['WindSpeed'].apply(str_to_float)
 df['WindDirection'] = df['WindDirection'].apply(clean_WindDirection)
 df['WindDirection'] = df['WindDirection'].apply(transform_WindDirection)
 df['PlayDirection'] = df['PlayDirection'].apply(lambda x: x.strip() == 'right')
 df['Team'] = df['Team'].apply(lambda x: x.strip() =='home')
 indoor = "indoor"
df['GameWeather'] = df['GameWeather'].apply(lambda x: indoor if not pd.isna(x)and indoor in x else x)
 df['GameWeather'] = df['GameWeather'].apply(lambda x: x.lower().replace('coudy', 'cloudy' ).replace('clouidy', 'cloudy' ).replace('party', 'partly' ).replace('clear and sunny', 'sunny and clear' ).replace('skies', '' ).replace("mostly", "" ).strip() if not pd.isna(x)else x)
 df['GameWeather'] = df['GameWeather'].apply(map_weather)
 df['IsRusher'] = df['NflId'] == df['NflIdRusher']
df['X'] = df.apply(lambda row: row['X'] if row['PlayDirection'] else 120-row['X'], axis=1)
 df['Orientation'] = df.apply(lambda row: new_orientation(row['Orientation'], row['PlayDirection']), axis=1)
 df['Dir'] = df.apply(lambda row: new_orientation(row['Dir'], row['PlayDirection']), axis=1)
 df['YardsLeft'] = df.apply(lambda row: 100-row['YardLine'] if row['HomeField'] else row['YardLine'], axis=1)
 df['YardsLeft'] = df.apply(lambda row: row['YardsLeft'] if row['PlayDirection'] else 100-row['YardsLeft'], axis=1)
 df = df.sort_values(by=['PlayId', 'Team', 'IsRusher', 'JerseyNumber'] ).reset_index()
df = df.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate', 'NflId', 'NflIdRusher', 'GameId', 'PlayId', 'index', 'IsRusher', 'Team'], axis=1)
 cat_features = []
for col in df.columns:
if df[col].dtype =='object':
cat_features.append(col)
 df = df.drop(cat_features, axis=1)
 df.fillna(-999, inplace=True)
 X = np.array(df[players_col] ).reshape(-1, len(players_col)*22)
 play_col = df.drop(players_col, axis=1 ).columns
X_play_col = np.zeros(shape=(X.shape[0], len(play_col)))
 for i, col in enumerate(play_col):
X_play_col[:, i] = df[col][::22]
X = np.concatenate([X, X_play_col], axis=1)
 X = scaler.transform(X)
 y_pred = np.mean([np.cumsum(model.predict(X), axis=1)for model in models], axis=0)
 yardsleft = np.array(df['YardsLeft'][::22])
 for i in range(len(yardsleft)) :
y_pred[i, :yardsleft[i]-1] = 0
y_pred[i, yardsleft[i]+100:] = 1
env.predict(pd.DataFrame(data=y_pred.clip(0,1),columns=sample.columns))
return y_pred
logging.basicConfig(level=logging.INFO)
<load_pretrained>
for test, sample in tqdm.tqdm(env.iter_test()):
make_pred(test, sample, env, models )
<string_transform><EOS> 
env.write_submission_file()