Kaggle competition: University of Liverpool - Ion Switching
class TweetModel(transformers.RobertaModel):
def __init__(self, conf):
super(TweetModel, self ).__init__(conf)
 self.roberta = transformers.RobertaModel.from_pretrained(config.ROBERTA_PATH + "pytorch_model.bin", config=conf)
 self.drop_out = nn.Dropout(0.1)
 self.conv1 = torch.nn.Conv1d(768, 128, 3, stride=1, padding=1, dilation=1, groups=1, bias=True, padding_mode='zeros')
 self.conv2 = torch.nn.Conv1d(128, 2, 3, stride=1, padding=1, dilation=1, groups=1, bias=True, padding_mode='zeros')
 self.l0 = nn.Linear(768, 2)
 torch.nn.init.normal_(self.l0.weight, std=0.02)
 def forward(self, ids, mask, token_type_ids):
_, _, out = self.roberta(
 ids,
attention_mask=mask,
token_type_ids=token_type_ids 
)
 out = torch.cat([out[-1][:,:,:, None], out[-2][:,:,:, None]], axis = -1)
 out, _ = torch.max(( out), dim=-1)
 out = self.drop_out(out)
 conv1_logits = self.conv1(out.transpose(1, 2))
conv2_logits = self.conv2(conv1_logits)
 logits = conv2_logits.transpose(1, 2)
 start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<compute_train_metric>
sns.set_context("talk")
 style.use('fivethirtyeight')
def loss_fn(start_logits, end_logits, start_positions, end_positions):
loss_fct = nn.CrossEntropyLoss()
start_loss = loss_fct(start_logits, start_positions)
 end_loss = loss_fct(end_logits, end_positions)
 total_loss = start_loss + end_loss
return total_loss<normalization>
train['batches'] =(train.index // 500_000)+ 1
def train_fn(data_loader, model, optimizer, device, scheduler=None):
model.train()
losses = utils.AverageMeter()
jaccards = utils.AverageMeter()
tk0 = tqdm(data_loader, total=len(data_loader))
start_preds = []
end_preds = []
start_labels = []
end_labels = []
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
offsets = d["offsets"]
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 model.zero_grad()
outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids, 
)
 loss = loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 if apex_on:
with amp.scale_loss(loss, optimizer)as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
optimizer.step()
scheduler.step()
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
start_preds.append(outputs_start)
 end_preds.append(outputs_end)
 start_labels.append(targets_start)
 end_labels.append(targets_end)
 jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
jaccard_score, _ = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.where(outputs_start[px, :].cumsum(axis = 0)> cdf_threshold)[0].min() ,
idx_end=np.where(outputs_end[px, :][::-1].cumsum(axis = 0)[::-1] > cdf_threshold)[0].max() ,
offsets=offsets[px] 
)
 jaccard_scores.append(jaccard_score)
 jaccards.update(np.mean(jaccard_scores), ids.size(0))
losses.update(loss.item() , ids.size(0))
tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg)
 return start_preds, end_preds, start_labels, end_labels<define_variables>
KNN = 100
batch = 1000
test_pred = np.zeros(( test.shape[0]),dtype=np.int8)
 for g in [0,1,2,3,4]:
print('Infering group %i'%g)
 data = train.loc[train.group==g]
X_train = np.zeros(( len(data)-6,7))
X_train[:,0] = 0.25*data.signal[:-6]
X_train[:,1] = 0.5*data.signal[1:-5]
X_train[:,2] = 1.0*data.signal[2:-4]
X_train[:,3] = 4.0*data.signal[3:-3]
X_train[:,4] = 1.0*data.signal[4:-2]
X_train[:,5] = 0.5*data.signal[5:-1]
X_train[:,6] = 0.25*data.signal[6:]
y_train = data.open_channels[3:].values
data = test.loc[test.group==g]
X_test = np.zeros(( len(data)-6,7))
X_test[:,0] = 0.25*data.signal[:-6]
X_test[:,1] = 0.5*data.signal[1:-5]
X_test[:,2] = 1.0*data.signal[2:-4]
X_test[:,3] = 4.0*data.signal[3:-3]
X_test[:,4] = 1.0*data.signal[4:-2]
X_test[:,5] = 0.5*data.signal[5:-1]
X_test[:,6] = 0.25*data.signal[6:]
model = NearestNeighbors(n_neighbors=KNN)
 model.fit(X_train)
 distances, indices = model.kneighbors(X_test)
 ct = indices.shape[0]
pred = np.zeros(( ct+6),dtype=np.int8)
 it = ct//batch + int(ct%batch!=0)
 print('Processing %i batches:'%(it))
for k in range(it):
a = batch*k; b = batch*(k+1); b = min(ct,b)
 pred[a+3:b+3] = np.median(y_train[ indices[a:b].astype(int)], axis=1)
 test_pred[test.group==g] = pred
def calculate_jaccard_score(
 original_tweet,
target_string,
sentiment_val,
idx_start,
idx_end,
offsets,
verbose=False):
if idx_end < idx_start:
idx_end = idx_start
if idx_start < 0:
idx_start = 0
filtered_output = ""
for ix in range(idx_start, idx_end + 1):
filtered_output += original_tweet[offsets[ix][0]: offsets[ix][1]]
if(ix+1)< len(offsets)and offsets[ix][1] < offsets[ix+1][0]:
filtered_output += " "
if sentiment_val == "neutral" or len(original_tweet.split())< 2:
filtered_output = original_tweet
jac = utils.jaccard(target_string.strip() , filtered_output.strip())
 return jac, filtered_output
def eval_fn(data_loader, model, device):
model.eval()
losses = utils.AverageMeter()
jaccards = utils.AverageMeter()
start_preds = []
end_preds = []
start_labels = []
end_labels = []
with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start, outputs_end = model(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 loss = loss_fn(outputs_start, outputs_end, targets_start, targets_end)
 outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
start_preds.append(outputs_start)
 end_preds.append(outputs_end)
 start_labels.append(targets_start)
 end_labels.append(targets_end)
 best_score = 0
jaccard_scores = []
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
jaccard_score, _ = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.where(outputs_start[px, :].cumsum(axis = 0)> cdf_threshold)[0].min() ,
idx_end=np.where(outputs_end[px, :][::-1].cumsum(axis = 0)[::-1] > cdf_threshold)[0].max() ,
offsets=offsets[px] 
)
 jaccard_scores.append(jaccard_score)
 jaccards.update(np.mean(jaccard_scores), ids.size(0))
losses.update(loss.item() , ids.size(0))
tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg)
 print(f"Jaccard = {jaccards.avg}")
 return jaccards.avg, start_preds, end_preds, start_labels, end_labels<categorify>
def collate_fn(data):
data = {k: [dic[k] for dic in data] for k in data[0]}
max_len = 0
for i in data["ids"]:
str_len = len(i)
 if str_len > max_len:
max_len = str_len
for i in range(len(data["ids"])) :
padding_length = max_len - len(data["ids"][i])
 if padding_length > 0:
data["ids"][i] = data["ids"][i] +([1] * padding_length)
 data["mask"][i] = data["mask"][i] +([0] * padding_length)
 data["token_type_ids"][i] = data["token_type_ids"][i] +([0] * padding_length)
 data["offsets"][i] = data["offsets"][i] +([(0, 0)] * padding_length)
 return {
'ids': torch.tensor(data["ids"], dtype=torch.long),
'mask': torch.tensor(data["mask"], dtype=torch.long),
'token_type_ids': torch.tensor(data["token_type_ids"], dtype=torch.long),
'targets_start': torch.tensor(data["targets_start"], dtype=torch.long),
'targets_end': torch.tensor(data["targets_end"], dtype=torch.long),
'orig_tweet': data["orig_tweet"],
'orig_selected': data["orig_selected"],
'sentiment': data["sentiment"],
'offsets': torch.tensor(data["offsets"], dtype=torch.long)
 }<feature_engineering>
sub = pd.read_csv('.. /input/liverpool-ion-switching/sample_submission.csv')
 sub.open_channels = test_pred
sub.to_csv('submission.csv',index=False,float_format='%.4f')
 res=200
plt.figure(figsize=(20,5))
plt.plot(sub.time[::res],sub.open_channels[::res])
 plt.show()
<split><EOS> 
')