Kaggle competition: Gendered Pronoun Resolution
with open(os.path.join(SPLIT,'tr_names.txt'), 'r')as text_file:
tr_n = text_file.read().split(',')
 with open(os.path.join(SPLIT,'val_names.txt'), 'r')as text_file:
val_n = text_file.read().split(',')
 test_names = sorted({f[:36] for f in os.listdir(TEST)})
 print(len(tr_n),len(val_n))<load_from_csv>
!conda remove -y greenlet
!pip install pytorch-pretrained-bert
!pip install allennlp
class Oversampling:
def __init__(self,path):
self.train_labels = pd.read_csv(path ).set_index('Id')
 self.train_labels['Target'] = [[int(i)for i in s.split() ]
for s in self.train_labels['Target']]
self.multi = [1,1,1,1,1,1,1,1,
4,4,4,1,1,1,1,4,
1,1,1,1,2,1,1,1,
1,1,1,4]
def get(self,image_id):
labels = self.train_labels.loc[image_id,'Target'] if image_id \
in self.train_labels.index else []
m = 1
for l in labels:
if m < self.multi[l]: m = self.multi[l]
return m
s = Oversampling(os.path.join(PATH,LABELS))
tr_n = [idx for idx in tr_n for _ in range(s.get(idx)) ]
print(len(tr_n),flush=True )<data_type_conversions>
nlp = English()
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
 sentencizer = nlp.create_pipe('sentencizer')
 nlp.add_pipe(sentencizer )
def open_rgby(path,id):
colors = ['red','green','blue','yellow']
flags = cv2.IMREAD_GRAYSCALE
img = [cv2.imread(os.path.join(path, id+'_'+color+'.png'), flags ).astype(np.float32)/255
for color in colors]
return np.stack(img, axis=-1 )<prepare_x_and_y>
def candidate_length(candidate):
count = 0
for i in range(len(candidate)) :
if candidate[i] != " ": count += 1
return count
def count_char(text, offset):
count = 0
for pos in range(offset):
if text[pos] != " ": count +=1
return count
def count_token_length_special(token):
count = 0
special_token = ["
for i in range(len(token)) :
if token[i] not in special_token:
count+=1
return count
def find_word_index(tokenized_text, char_start, target):
tar_len = candidate_length(target)
 char_count = 0
word_index = []
special_token = ["[CLS]", "[SEP]"]
for i in range(len(tokenized_text)) :
token = tokenized_text[i]
if char_count in range(char_start, char_start+tar_len):
if token in special_token:
continue
word_index.append(i)
 if token not in special_token:
token_length = count_token_length_special(token)
 char_count += token_length
if len(word_index)== 1:
return [word_index[0], word_index[0]]
else:
return [word_index[0], word_index[-1]]
def create_tokenizer_input(sents):
tokenizer_input = str()
for i, sent in enumerate(sents):
if i == 0:
tokenizer_input += "[CLS] "+sent.text+" [SEP] "
elif i == len(sents)- 1:
tokenizer_input += sent.text+" [SEP]"
else:
tokenizer_input += sent.text+" [SEP] "
return tokenizer_input
def create_inputs(dataframe):
idxs = dataframe.index
columns = ['indexed_token', 'offset']
features_df = pd.DataFrame(index=idxs, columns=columns)
 max_len = 0
for i in tqdm(range(len(dataframe))):
text = dataframe.loc[i, 'Text']
Pronoun_offset = dataframe.loc[i, 'Pronoun-offset']
A_offset = dataframe.loc[i, "A-offset"]
B_offset = dataframe.loc[i, "B-offset"]
Pronoun = dataframe.loc[i, "Pronoun"]
A = dataframe.loc[i, "A"]
B = dataframe.loc[i, "B"]
doc = nlp(text)
 sents = []
for sent in doc.sents: sents.append(sent)
 token_input = create_tokenizer_input(sents)
 token_input = token_input.replace("
tokenized_text = tokenizer.tokenize(token_input)
 if len(tokenized_text)> max_len:
max_len = len(tokenized_text)
 indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
 A_char_start, B_char_start = count_char(text, A_offset), count_char(text, B_offset)
 Pronoun_char_start = count_char(text, Pronoun_offset)
 word_indexes = []
for char_start, target in zip([A_char_start, B_char_start, Pronoun_char_start], [A, B, Pronoun]):
word_indexes.append(find_word_index(tokenized_text, char_start, target))
features_df.iloc[i] = [indexed_tokens, word_indexes]
print('max length of sentence:', max_len)
 return features_df
class pdFilesDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.labels = pd.read_csv(LABELS ).set_index('Id')
 self.labels['Target'] = [[int(i)for i in s.split() ] for s in self.labels['Target']]
super().__init__(fnames, transform, path)
 def get_x(self, i):
return open_rgby(self.path,self.fnames[i])
 def get_y(self, i):
if(self.path == TEST): return np.zeros(len(name_label_dict),dtype=np.int)
 else:
labels = self.labels.loc[self.fnames[i]]['Target']
return np.eye(len(name_label_dict),dtype=np.float)[labels].sum(axis=0)
 @property
def is_multi(self): return True
@property
def is_reg(self):return True
def get_c(self): return len(name_label_dict )<train_model>
train_df = pd.read_table('.. /input/gap-coreference/gap-test.tsv')
 test_df = pd.read_table('.. /input/gap-coreference/gap-development.tsv')
 val_df = pd.read_table('.. /input/gap-coreference/gap-validation.tsv')
 new_train_df = create_inputs(train_df)
 new_test_df = create_inputs(test_df)
 new_val_df = create_inputs(val_df )
bs = 32
sz = 256
md = get_data(sz,bs,is_test=True)
 x,y = next(iter(md.aug_dl))
x.shape, y.shape<categorify>
def get_label(dataframe):
labels = []
for i in range(len(dataframe)) :
if dataframe.loc[i, 'A-coref']:
labels.append(0)
 elif dataframe.loc[i, 'B-coref']:
labels.append(1)
 else:
labels.append(2)
 return labels
new_train_df['label'] = get_label(train_df)
 new_val_df['label'] = get_label(val_df)
 new_df = pd.concat([new_train_df, new_val_df])
 new_df = new_df.reset_index(drop=True)
 new_df.to_csv('train.csv', index=False)
 new_test_df['label'] = get_label(test_df)
 new_test_df.to_csv('test.csv', index=False )
class Resnet34_4(nn.Module):
def __init__(self, pre=True):
super().__init__()
encoder = resnet34(pretrained=pre)
 self.conv1 = nn.Conv2d(4, 64, kernel_size=7, stride=2, padding=3, bias=False)
 if(pre):
w = encoder.conv1.weight
self.conv1.weight = nn.Parameter(torch.cat(( w,
0.5*(w[:,:1,:,:]+w[:,2:,:,:])) ,dim=1))
self.bn1 = encoder.bn1
self.relu = nn.ReLU(inplace=True)
 self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
 self.layer0 = nn.Sequential(self.conv1,self.relu,self.bn1,self.maxpool)
 self.layer1 = encoder.layer1
self.layer2 = encoder.layer2
self.layer3 = encoder.layer3
self.layer4 = encoder.layer4
def forward(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x<train_model>
del new_df
del new_val_df
del new_test_df
del new_train_df
class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, input, target):
if not(target.size() == input.size()):
raise ValueError("Target size({})must be the same as input size({})" 
.format(target.size() , input.size()))
max_val =(-input ).clamp(min=0)
 loss = input - input * target + max_val + \ 
(( -max_val ).exp() +(-input - max_val ).exp() ).log()
invprobs = F.logsigmoid(-input *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
return loss.sum(dim=1 ).mean()<compute_test_metric>
gc.collect()
def acc(preds,targs,th=0.0):
preds =(preds > th ).int()
targs = targs.int()
return(preds==targs ).float().mean()<feature_engineering>
class MyDataset(Dataset):
def __init__(self, dataframe, transform=None):
self.df = dataframe
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
index_token = self.df.loc[idx, 'indexed_token']
index_token = literal_eval(index_token)
 index_token = pad_sequences([index_token], maxlen=360, padding='post')[0]
offset = self.df.loc[idx, 'offset']
offset = literal_eval(offset)
 offset = np.asarray(offset, dtype='int32')
 label = int(self.df.loc[idx, 'label'])
 distP_A = self.df.loc[idx, 'D_PA']
distP_B = self.df.loc[idx, 'D_PB']
if self.transform:
index_token = self.transform(index_token)
 offset = self.transform(offset)
 label = self.transform(label)
 return(index_token, offset, distP_A, distP_B), label
class F1:
__name__ = 'F1 macro'
def __init__(self,n=28):
self.n = n
self.TP = np.zeros(self.n)
 self.FP = np.zeros(self.n)
 self.FN = np.zeros(self.n)
 def __call__(self,preds,targs,th=0.0):
preds =(preds > th ).int()
targs = targs.int()
self.TP +=(preds*targs ).float().sum(dim=0)
 self.FP +=(preds > targs ).float().sum(dim=0)
 self.FN +=(preds < targs ).float().sum(dim=0)
 score =(2.0*self.TP/(2.0*self.TP + self.FP + self.FN + 1e-6)).mean()
return score
def reset(self):
score =(2.0*self.TP/(2.0*self.TP + self.FP + self.FN + 1e-6))
print('F1 macro:',score.mean() ,flush=True)
 self.TP = np.zeros(self.n)
 self.FP = np.zeros(self.n)
 self.FN = np.zeros(self.n)
 class F1_callback(Callback):
def __init__(self, n=28):
self.f1 = F1(n)
 def on_epoch_end(self, metrics):
self.f1.reset()<choose_model_class>
class score(torch.nn.Module):
def __init__(self, embed_dim, hidden_dim):
super(score, self ).__init__()
self.score = torch.nn.Sequential(
 torch.nn.Linear(embed_dim, hidden_dim),
torch.nn.ReLU(inplace=True),
torch.nn.Dropout(0.6),
torch.nn.Linear(hidden_dim, 1))
def forward(self, x):
return self.score(x)
 class mentionpair_score(torch.nn.Module):
def __init__(self, input_dim, hidden_dim):
super(mentionpair_score, self ).__init__()
self.score = score(input_dim, hidden_dim)
 def forward(self, g1, g2, dist_embed):
element_wise = g1 * g2
pair_score = self.score(torch.cat(( g1, g2, element_wise, dist_embed), dim=-1))
return pair_score
class score_model(torch.nn.Module):
def __init__(self):
super(score_model, self ).__init__()
self.buckets = [1, 2, 3, 4, 5, 8, 16, 32, 64]
self.bert = BertModel.from_pretrained('bert-base-uncased')
 self.embedding = torch.nn.Embedding(len(self.buckets)+1, 20)
 self.span_extractor = EndpointSpanExtractor(768, "x,y,x*y")
 self.pair_score = mentionpair_score(2304*3+20, 150)
 def forward(self, sent, offsets, distP_A, distP_B):
bert_output, _ = self.bert(sent, output_all_encoded_layers=False)
 distPA_embed = self.embedding(distP_A)
 distPB_embed = self.embedding(distP_B)
 span_repres = self.span_extractor(bert_output, offsets)
 span_repres = torch.unbind(span_repres, dim=1)
 span_norm = []
for i in range(len(span_repres)) :
span_norm.append(F.normalize(span_repres[i], p=2, dim=1))
ap_score = self.pair_score(span_norm[2], span_norm[0], distPA_embed)
 bp_score = self.pair_score(span_norm[2], span_norm[1], distPB_embed)
 nan_score = torch.zeros_like(ap_score)
 output = torch.cat(( ap_score, bp_score, nan_score), dim=1)
 return output
sz = 256
bs = 100
md = get_data(sz,bs)
 learner = ConvLearner.pretrained(Resnet34_4, md, ps=0.7)
 learner.opt_fn = optim.Adam
learner.clip = 1.0
learner.crit = FocalLoss()
f1_callback = F1_callback()
learner.metrics = [acc,f1_callback.f1]
learner.summary<train_model>
def children(m):
return m if isinstance(m,(list, tuple)) else list(m.children())
 def set_trainable_attr(m, b):
m.trainable = b
for p in m.parameters() :
p.requires_grad = b
def apply_leaf(m, f):
c = children(m)
 if isinstance(m, torch.nn.Module):
f(m)
 if len(c)> 0:
for l in c:
apply_leaf(l, f)
 def set_trainable(l, b):
apply_leaf(l, lambda m: set_trainable_attr(m, b))
lr = 0.5e-3
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 learner.fit(lr,1,callbacks=[f1_callback] )<define_search_space>
train_dist = pd.read_csv('.. /input/bert-score-layer-lb-0-475/train_dist_df.csv')
 val_dist = pd.read_csv('.. /input/bert-score-layer-lb-0-475/val_dist_df.csv')
 test_dist = pd.read_csv('.. /input/bert-score-layer-lb-0-475/test_dist_df.csv')
 train_dist = pd.concat([train_dist, val_dist])
 train_dist = train_dist.reset_index(drop=True)
 train_dist.head()
learner.unfreeze()
lrs=np.array([lr/10,lr/3,lr] )<train_model>
n_split = 5
train = pd.read_csv('.. /working/train.csv')
 test = pd.read_csv('.. /working/test.csv')
 train = pd.concat([train, train_dist], axis=1)
 test = pd.concat([test, test_dist], axis=1)
 train.head()
Kfold = StratifiedKFold(n_splits=n_split, random_state=2019 ).split(train, train['label'] )
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 learner.fit(lrs/4,4,cycle_len=2,use_clr=(10,20),callbacks=[f1_callback] )<train_model>
def softmax(x):
exp_x = np.exp(x)
 y = exp_x / np.sum(exp_x, axis=1, keepdims=True)
 return y
output = np.zeros(( len(test_df), 3))
testset = MyDataset(test)
 test_loader = torch.utils.data.DataLoader(testset, batch_size=20)
 n_epochs = 30
for n_fold,(train_index, val_index)in enumerate(Kfold):
min_val_loss = 100.0
PATH = "./best_model_{}.hdf5".format(n_fold+1)
 train_df = train.loc[train_index]
train_df = train_df.reset_index(drop=True)
 val_df = train.loc[val_index]
val_df = val_df.reset_index(drop=True)
 trainset = MyDataset(train_df)
 train_loader = torch.utils.data.DataLoader(trainset, batch_size=20, shuffle=True)
 valset = MyDataset(val_df)
 val_loader = torch.utils.data.DataLoader(valset, batch_size=20, shuffle=True)
 model = score_model()
set_trainable(model.bert, False)
 set_trainable(model.embedding, True)
 set_trainable(model.pair_score, True)
 model.cuda()
optimizer = torch.optim.Adam(model.parameters() , lr=0.001, weight_decay=0.00001)
 criterion = torch.nn.CrossEntropyLoss().cuda()
print('fold:', n_fold+1)
 for i in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for idx,(inputs, label)in enumerate(train_loader):
index_token, offset, distP_A, distP_B = inputs
index_token = index_token.type(torch.LongTensor ).cuda()
offset = offset.type(torch.LongTensor ).cuda()
label = label.type(torch.LongTensor ).cuda()
distP_A = distP_A.type(torch.LongTensor ).cuda()
distP_B = distP_B.type(torch.LongTensor ).cuda()
optimizer.zero_grad()
output_train = model(index_token, offset, distP_A, distP_B)
 loss = criterion(output_train, label)
 loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 avg_val_loss = 0.
 model.eval()
with torch.no_grad() :
for idx,(inputs, label)in enumerate(val_loader):
index_token, offset, distP_A, distP_B = inputs
index_token = index_token.type(torch.LongTensor ).cuda()
offset = offset.type(torch.LongTensor ).cuda()
label = label.type(torch.LongTensor ).cuda()
distP_A = distP_A.type(torch.LongTensor ).cuda()
distP_B = distP_B.type(torch.LongTensor ).cuda()
output_test = model(index_token, offset, distP_A, distP_B)
 avg_val_loss += criterion(output_test, label ).item() / len(val_loader)
 elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 i + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
if min_val_loss > avg_val_loss:
min_val_loss = avg_val_loss
torch.save(model.state_dict() , PATH)
 del model
model = score_model()
model.load_state_dict(torch.load(PATH))
model.cuda()
model.eval()
with torch.no_grad() :
for idx,(inputs, label)in enumerate(test_loader):
index_token, offset, distP_A, distP_B = inputs
index_token = index_token.type(torch.LongTensor ).cuda()
offset = offset.type(torch.LongTensor ).cuda()
label = label.type(torch.LongTensor ).cuda()
distP_A = distP_A.type(torch.LongTensor ).cuda()
distP_B = distP_B.type(torch.LongTensor ).cuda()
y_pred = model(index_token, offset, distP_A, distP_B)
 y_pred = softmax(y_pred.cpu().numpy())
 start = idx * 20
end = start + 20
output[start:end, :] += y_pred
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 learner.fit(lrs/16,2,cycle_len=4,use_clr=(10,20),callbacks=[f1_callback] )<train_model>
output /= 5
sub_df_path = os.path.join('.. /input/gendered-pronoun-resolution/', 'sample_submission_stage_1.csv')
 sub_df = pd.read_csv(sub_df_path)
 sub_df.loc[:, 'A'] = pd.Series(output[:, 0])
 sub_df.loc[:, 'B'] = pd.Series(output[:, 1])
 sub_df.loc[:, 'NEITHER'] = pd.Series(output[:, 2])
 sub_df.head(20 )
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 learner.fit(lrs/32,1,cycle_len=8,use_clr=(10,20),callbacks=[f1_callback] )<save_model>
sub_df.to_csv("submission.csv", index=False )
learner.save('ResNet34_256_1' )<feature_engineering>
y_test = pd.read_csv('.. /working/test.csv')['label']
y_one_hot = np.zeros(( 2000, 3))
for i in range(len(y_test)) :
y_one_hot[i, y_test[i]] = 1
log_loss(y_one_hot, output)
md = get_data(sz,bs,is_test=True)
 learner.set_data(md)
 preds,y = learner.TTA(n_aug=8)
 preds = np.stack(preds, axis=-1)
 pred = preds.mean(axis=-1 )<train_model>
_output = np.argmax(output, axis=1)
 print('acc:', np.asarray(np.where(_output == y_test)).shape[1]/ 2000 )