Kaggle competition: Google Cloud & NCAA® ML Competition 2019-Men's
<SOS> metric: LogLoss Kaggle data source: mens-machine-learning-competition-2019<feature_engineering>
%matplotlib inline
plt.rcParams['figure.figsize'] =(15, 9)
 season_df = pd.read_csv('.. /input/datafiles/RegularSeasonCompactResults.csv')
 tourney_df = pd.read_csv('.. /input/datafiles/NCAATourneyCompactResults.csv')
 ordinals_df = pd.read_csv('.. /input/masseyordinals/MasseyOrdinals.csv')\ 
.rename(columns={'RankingDayNum':'DayNum'})
 ordinals_df = ordinals_df.groupby(['SystemName','Season','TeamID'] ).last().reset_index()
ordinals_df.head()
class TweetDataset(torch.utils.data.Dataset):
def __init__(self, df, max_len=96):
self.df = df
self.max_len = max_len
self.labeled = 'selected_text' in df
self.tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file='.. /input/roberta-base/vocab.json',
merges_file='.. /input/roberta-base/merges.txt',
lowercase=True,
add_prefix_space=True)
 def __getitem__(self, index):
data = {}
row = self.df.iloc[index]
ids, masks, tweet, offsets = self.get_input_data(row)
 data['ids'] = ids
data['masks'] = masks
data['tweet'] = tweet
data['offsets'] = offsets
if self.labeled:
start_idx, end_idx = self.get_target_idx(row, tweet, offsets)
 data['start_idx'] = start_idx
data['end_idx'] = end_idx
return data
def __len__(self):
return len(self.df)
 def get_input_data(self, row):
tweet = " " + " ".join(row.text.lower().split())
 encoding = self.tokenizer.encode(tweet)
 sentiment_id = self.tokenizer.encode(row.sentiment ).ids
ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]
offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = self.max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 return ids, masks, tweet, offsets
def get_target_idx(self, row, tweet, offsets):
selected_text = " " + " ".join(row.selected_text.lower().split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
target_idx = []
for j,(offset1, offset2)in enumerate(offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
return start_idx, end_idx
def get_train_val_loaders(df, train_idx, val_idx, batch_size=8):
train_df = df.iloc[train_idx]
val_df = df.iloc[val_idx]
train_loader = torch.utils.data.DataLoader(
 TweetDataset(train_df),
batch_size=batch_size,
shuffle=True,
num_workers=2,
drop_last=True)
 val_loader = torch.utils.data.DataLoader(
 TweetDataset(val_df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 dataloaders_dict = {"train": train_loader, "val": val_loader}
return dataloaders_dict
def get_test_loader(df, batch_size=32):
loader = torch.utils.data.DataLoader(
 TweetDataset(df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 return loader<load_pretrained>
games_df = tourney_df.merge(ordinals_df,left_on=['Season','WTeamID'],
right_on=['Season','TeamID'])
 games_df.head()
games_df = games_df.merge(ordinals_df,left_on=['Season','LTeamID','SystemName'],
right_on=['Season','TeamID','SystemName'],
suffixes = ['W','L'])
 games_df.head()
class TweetModel(nn.Module):
def __init__(self):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 '.. /input/roberta-base/config.json', output_hidden_states=True)
 self.roberta = RobertaModel.from_pretrained(
 '.. /input/roberta-base/pytorch_model.bin', config=config)
 self.dropout = nn.Dropout(0.5)
 self.fc = nn.Linear(config.hidden_size, 2)
 nn.init.normal_(self.fc.weight, std=0.02)
 nn.init.normal_(self.fc.bias, 0)
 def forward(self, input_ids, attention_mask):
_, _, hs = self.roberta(input_ids, attention_mask)
 x = torch.stack([hs[-1], hs[-2], hs[-3], hs[-4]])
 x = torch.mean(x, 0)
 x = self.dropout(x)
 x = self.fc(x)
 start_logits, end_logits = x.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<compute_test_metric>
results_by_system['prediction'].sort_values('mean',ascending=False)[:50]
def loss_fn(start_logits, end_logits, start_positions, end_positions):
ce_loss = nn.CrossEntropyLoss()
start_loss = ce_loss(start_logits, start_positions)
 end_loss = ce_loss(end_logits, end_positions)
 total_loss = start_loss + end_loss
return total_loss<statistical_test>
games_df['Wrating'] = 100-4*np.log(games_df['OrdinalRankW']+1)-games_df['OrdinalRankW']/22
games_df['Lrating'] = 100-4*np.log(games_df['OrdinalRankL']+1)-games_df['OrdinalRankL']/22
games_df['prob'] = 1/(1+10**(( games_df['Lrating']-games_df['Wrating'])/15))
loss_results = games_df[games_df.Season>=2015].groupby('SystemName')['prob'].agg([('loss',lambda p: -np.mean(np.log(p))),('count','count')])
 loss_results['loss'].sort_values() [:50].plot.bar(ylim=[.4,.6] )
<train_model><EOS> 
ref_system = 'POM'
ordinals_df['Rating']= 100-4*np.log(ordinals_df['OrdinalRank']+1)-ordinals_df['OrdinalRank']/22
ordinals_df = ordinals_df[ordinals_df.SystemName==ref_system]
submission_df = pd.read_csv('.. /input/SampleSubmissionStage1.csv')
 submission_df['Season'] = submission_df['ID'].map(lambda x: int(x.split('_')[0]))
submission_df['Team1'] = submission_df['ID'].map(lambda x: int(x.split('_')[1]))
submission_df['Team2'] = submission_df['ID'].map(lambda x: int(x.split('_')[2]))
submission_df = submission_df.merge(ordinals_df[['Season','TeamID','Rating']],how='left',
left_on = ['Season','Team1'], right_on = ['Season','TeamID'])
 submission_df = submission_df.merge(ordinals_df[['Season','TeamID','Rating']],how='left',
left_on = ['Season','Team2'], right_on = ['Season','TeamID'],
suffixes=['W','L'])
 submission_df['Pred'] = 1/(1+10**(( submission_df['RatingL']-submission_df['RatingW'])/15))
submission_df[['ID', 'Pred']].to_csv('submission.csv', index=False)
 submission_df[['ID', 'Pred']].head()