Kaggle competition: Gendered Pronoun Resolution
<SOS> metric: MulticlassLoss Kaggle data source: gendered-pronoun-resolution<train_model>
!wget https://github.com/google-research-datasets/gap-coreference/raw/master/gap-development.tsv -q
!wget https://github.com/google-research-datasets/gap-coreference/raw/master/gap-test.tsv -q
!wget https://github.com/google-research-datasets/gap-coreference/raw/master/gap-validation.tsv -q
train_datagen = DataGenerator()
generator = train_datagen.create_train(
 train_dataset_info, 5,(299,299,3))<split>
!pip install pytorch-pretrained-bert
data['target_list'] = data['Target'].map(lambda x: [int(a)for a in x.split(' ')])
 all_labels = list(chain.from_iterable(data['target_list'].values))
c_val = Counter(all_labels)
 n_keys = c_val.keys()
max_idx = max(n_keys)
 data['target_vec'] = data['target_list'].map(lambda ck: [i in ck for i in range(max_idx+1)])
 train_df, valid_df = train_test_split(data,
test_size = 0.2,
stratify = data['Target'].map(lambda x: x[:3] if '27' not in x else '0'), random_state=42)
 print(train_df.shape[0], 'training masks')
 print(valid_df.shape[0], 'validation masks')
 train_df.to_csv('train_part.csv')
 valid_df.to_csv('valid_part.csv')
<string_transform>
import time
import os
import random
import numpy as np
import pandas as pd
import torch
from torch.optim import Adam
from torch.utils.data import Dataset
from torch.nn import Module, Linear, Dropout
import torch.nn.functional as F
from pytorch_pretrained_bert.modeling import BertModel, BertLayer
from pytorch_pretrained_bert import BertTokenizer
from pytorch_pretrained_bert.optimization import BertAdam
from pytorch_pretrained_bert.optimization import warmup_linear
from torch.utils.data import DataLoader
from torch.utils.data import RandomSampler
from sklearn.metrics import log_loss
import matplotlib.pyplot as plt
train_dataset_info = []
for name, labels in zip(train_df['Id'], train_df['Target'].str.split(' ')) :
train_dataset_info.append({
'path':os.path.join(path_to_train, name),
'labels':np.array([int(label)for label in labels])})
 train_dataset_info = np.array(train_dataset_info)
 valid_dataset_info = []
for name, labels in zip(valid_df['Id'], valid_df['Target'].str.split(' ')) :
valid_dataset_info.append({
'path':os.path.join(path_to_train, name),
'labels':np.array([int(label)for label in labels])})
 valid_dataset_info = np.array(valid_dataset_info)
 print(train_dataset_info.shape, valid_dataset_info.shape )<choose_model_class>
seed = 9876
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
def create_model(input_shape, n_out):
inp_image = Input(shape=input_shape)
 inp_mask = Input(shape=input_shape)
 pretrain_model_image = Xception(
 include_top=False,
weights='imagenet',
pooling='max')
 pretrain_model_image.name='xception_image'
pretrain_model_mask = Xception(
 include_top=False,
weights='imagenet',
pooling='max')
 pretrain_model_mask.name='xception_mask'
x = Multiply()([pretrain_model_image(inp_image), pretrain_model_mask(inp_mask)])
 out = Dense(n_out, activation='sigmoid' )(x)
 model = Model(inputs=[inp_image, inp_mask], outputs=[out])
 return model<compute_test_metric>
bert_model = "bert-large-cased"
n_bertlayers = 22
dropout = 0.1
do_lower_case = False
train_batch_size = 4
gradient_accumulation_steps = 5
lr = 1e-5
num_train_epochs = 2
warmup_proportion = 0.1
optim = "bertadam"
weight_decay = False
n_models = 10
eval_batch_size = 32
device = torch.device("cuda")
 data_dir = ""
def f1(y_true, y_pred):
y_pred = K.round(y_pred)
 tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)
 tn = K.sum(K.cast(( 1-y_true)*(1-y_pred), 'float'), axis=0)
 fp = K.sum(K.cast(( 1-y_true)*y_pred, 'float'), axis=0)
 fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)
 p = tp /(tp + fp + K.epsilon())
 r = tp /(tp + fn + K.epsilon())
 f1 = 2*p*r /(p+r+K.epsilon())
 f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)
 return K.mean(f1 )<choose_model_class>
def insert_tag(row):

 to_be_inserted = sorted([ 
(row["A-offset"], " [A] "), 
(row["B-offset"], " [B] "), 
(row["Pronoun-offset"], " [P] ")
 ], key=lambda x: x[0], reverse=True)
 text = row["Text"]
for offset, tag in to_be_inserted:
text = text[:offset] + tag + text[offset:]
return text
def tokenize(text, tokenizer):

 entries = {}
final_tokens = []
for token in tokenizer.tokenize(text):
if token in("[A]", "[B]", "[P]"):
entries[token] = len(final_tokens)
 continue
final_tokens.append(token)
 return final_tokens,(entries["[A]"], entries["[B]"], entries["[P]"])
 class GAPDataset(Dataset):

 def __init__(self, df, tokenizer, labeled=True):
self.labeled = labeled
if labeled:
tmp = df[["A-coref", "B-coref"]].copy()
tmp["Neither"] = ~(df["A-coref"] | df["B-coref"])
 self.y = tmp.values.astype("bool")
 self.offsets, self.tokens = [], []
self.seq_len = []
for _, row in df.iterrows() :
text = insert_tag(row)
 tokens, offsets = tokenize(text, tokenizer)
 self.offsets.append(offsets)
 self.tokens.append(tokenizer.convert_tokens_to_ids(
 ["[CLS]"] + tokens + ["[SEP]"]))
self.seq_len.append(len(self.tokens[-1]))
def __len__(self):
return len(self.tokens)
 def __getitem__(self, idx):
if self.labeled:
return self.tokens[idx], self.offsets[idx], self.y[idx]
return self.tokens[idx], self.offsets[idx]
def get_seq_len(self):
return self.seq_len
def collate_examples(batch, truncate_len=500):

 transposed = list(zip(*batch))
max_len = min(
 max(( len(x)for x in transposed[0])) ,
truncate_len 
)
 tokens = np.zeros(( len(batch), max_len), dtype=np.int64)
 for i, row in enumerate(transposed[0]):
row = np.array(row[:truncate_len])
 tokens[i, :len(row)] = row
token_tensor = torch.from_numpy(tokens)
 offsets = torch.stack([
torch.LongTensor(x)for x in transposed[1]
], dim=0)+ 1
if len(transposed)== 2:
return token_tensor, offsets, None
one_hot_labels = torch.stack([
torch.from_numpy(x.astype("uint8")) for x in transposed[2]
], dim=0)
 _, labels = one_hot_labels.max(dim=1)
 return token_tensor, offsets, labels
keras.backend.clear_session()
model = create_model(
 input_shape=(299,299,3),
n_out=28)
 model.compile(
 loss='binary_crossentropy',
optimizer='adam',
metrics=['acc', f1])
 model.summary()<train_model>
def get_pretrained_bert(modelname, num_hidden_layers=None):
bert = BertModel.from_pretrained(modelname)
 if num_hidden_layers is None:
return bert
old_num_hidden_layers = bert.config.num_hidden_layers
if num_hidden_layers < old_num_hidden_layers:
del bert.encoder.layer[num_hidden_layers:]
elif num_hidden_layers > old_num_hidden_layers:
for i in range(old_num_hidden_layers, num_hidden_layers):
bert.encoder.layer.add_module(str(i), BertLayer(bert.config))
if num_hidden_layers != old_num_hidden_layers:
bert.config.num_hidden_layers = num_hidden_layers
bert.init_bert_weights(bert.pooler.dense)
 return bert
class BertCl_GAP(Module):

 def __init__(self, bert, dropout, n_offsets=3):
super().__init__()
self.bert = bert
self.bert_hidden_size = self.bert.config.hidden_size
self.dropout = Dropout(dropout)
 self.classifier = Linear(self.bert.config.hidden_size * n_offsets, n_offsets)
 def forward(self, token_tensor, offsets, label_id=None):
bert_outputs, _ = self.bert(
 token_tensor, attention_mask=(token_tensor > 0 ).long() ,
token_type_ids=None, output_all_encoded_layers=False)
 extracted_outputs = bert_outputs.gather(
 1, offsets.unsqueeze(2 ).expand(-1, -1, bert_outputs.size(2))
).view(bert_outputs.size(0), -1)
 outputs = self.classifier(self.dropout(extracted_outputs))
return outputs
def get_param_size(model):
trainable_psize = np.sum([np.prod(p.size())for p in model.parameters() if p.requires_grad])
 total_psize = np.sum([np.prod(p.size())for p in model.parameters() ])
 return total_psize, trainable_psize
def run_epoch(model, dataloader, optimizer, criterion, device,
verbose_step=10000):
model.train()
t1 = time.time()
tr_loss = 0
for step, batch in enumerate(dataloader):
batch = tuple(t.to(device)for t in batch)
 label_ids = batch[-1]
outputs = model(*batch[:-1])
 if criterion._get_name() == "BCEWithLogitsLoss":
outputs = outputs[:, 0]
label_ids = label_ids.float()
loss = criterion(outputs, label_ids)
 if gradient_accumulation_steps > 1:
loss = loss / gradient_accumulation_steps
loss.backward()
tr_loss += loss.item()
if(step + 1)% verbose_step == 0:
loss_now = gradient_accumulation_steps * tr_loss /(step + 1)
 print(f'step:{step+1} loss:{loss_now:.7f} time:{time.time() - t1:.1f}s')
 if(step + 1)% gradient_accumulation_steps == 0:
optimizer.step()
model.zero_grad()
return gradient_accumulation_steps * tr_loss /(step + 1)
 def predict(model, data_loader, device, proba=True, to_numpy=True):
model.eval()
preds = []
for step, batch in enumerate(data_loader):
batch = batch[:2]
batch = tuple(t.to(device)for t in batch)
 with torch.no_grad() :
logits = model(*batch)
 preds.append(logits.detach().cpu())
 preds = torch.cat(preds)if len(preds)> 1 else preds[0]
if proba:
if preds.size(-1)> 1:
preds = F.softmax(preds, dim=1)
 else:
preds = torch.sigmoid(preds)
 if to_numpy:
preds = preds.numpy()
return preds
epochs = 4; batch_size = 12
checkpointer = ModelCheckpoint(
 '.. /working/Xception.model',
verbose=2,
save_best_only=False)
 train_generator = train_datagen.create_train(
 train_dataset_info, batch_size,(299,299,3))
validation_generator = train_datagen.create_train(
 valid_dataset_info, batch_size,(299,299,3))
K.set_value(model.optimizer.lr, 0.0002)
 history = model.fit_generator(
 train_generator,
steps_per_epoch=len(train_df)//batch_size,
validation_data=validation_generator,
validation_steps=len(valid_df)//batch_size//10,
epochs=epochs,
verbose=1,
callbacks=[checkpointer] )<load_from_csv>
def get_gap_model(bert_model, n_bertlayers, dropout,
steps_per_epoch, device):
bert = get_pretrained_bert(bert_model, n_bertlayers)
 model = BertCl_GAP(bert, dropout)
 model.to(device)
 param_optimizer = list(model.named_parameters())
 if weight_decay:
no_decay = ["bias", "gamma", "beta", "head"]
optimizer_grouped_parameters = [
{"params": [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
"weight_decay": 0.01},
{"params": [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
"weight_decay": 0.0}
]
else:
optimizer_grouped_parameters = [
{"params": [p for n, p in param_optimizer], "weight_decay": 0.0}
]
t_total = int(
 steps_per_epoch / gradient_accumulation_steps * num_train_epochs)
 if optim == 'bertadam':
optimizer = BertAdam(optimizer_grouped_parameters,
lr=lr,
warmup=warmup_proportion,
t_total=t_total)
 elif optim == 'adam':
optimizer = Adam(optimizer_grouped_parameters, lr=lr)
 return model, optimizer
def get_loader(train_df, val_df, test_df):
tokenizer = BertTokenizer.from_pretrained(
 bert_model,
do_lower_case=do_lower_case,
never_split=("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]", "[A]", "[B]", "[P]")
)
 tokenizer.vocab['[A]'] = -1
tokenizer.vocab['[B]'] = -1
tokenizer.vocab['[P]'] = -1
train_ds = GAPDataset(train_df, tokenizer)
 val_ds = GAPDataset(val_df, tokenizer)
 test_ds = GAPDataset(test_df, tokenizer, labeled=False)
 train_loader = DataLoader(
 train_ds,
collate_fn=collate_examples,
batch_size=train_batch_size,
shuffle=True,
drop_last=True 
)
 val_loader = DataLoader(
 val_ds,
collate_fn=collate_examples,
batch_size=eval_batch_size,
shuffle=False 
)
 test_loader = DataLoader(
 test_ds,
collate_fn=collate_examples,
batch_size=eval_batch_size,
shuffle=False 
)
 return train_loader, val_loader, test_loader
submit = pd.read_csv('.. /input/sample_submission.csv' )<predict_on_test>
train_df = pd.concat([pd.read_csv(data_dir + "gap-test.tsv", delimiter="\t"),
pd.read_csv(data_dir + "gap-development.tsv", delimiter="\t")])
 val_df = pd.read_csv(data_dir + "gap-validation.tsv", delimiter="\t")
 val_y = val_df[['A-coref', 'B-coref']].astype(int)
 val_y['None'] = 1 - val_y.sum(1)
 test_df = pd.read_csv(".. /input/test_stage_2.tsv", delimiter="\t")
 print(f"Train:{train_df.shape[0]}, Valid:{val_df.shape[0]}, Test:{test_df.shape[0]}" )
%%time
predicted = []
for name in tqdm(submit['Id']):
path = os.path.join('.. /input/test/', name)
 image1, image2 = train_datagen.load_image(path,(299,299,3))
score_predict = model.predict([image1[np.newaxis], image2[np.newaxis]])[0]
label_predict = np.arange(28)[score_predict>=0.5]
str_predict_label = ' '.join(str(l)for l in label_predict)
 predicted.append(str_predict_label )<save_to_csv>
train_loader, val_loader, test_loader = get_loader(train_df, val_df, test_df )
submit['Predicted'] = predicted
submit.to_csv('submission.csv', index=False )<define_variables>
steps_per_epoch = len(train_loader)
 steps_per_epoch
name_label_dict = {
0: "Nucleoplasm",
1: "Nuclear membrane",
2: "Nucleoli",
3: "Nucleoli fibrillar center",
4: "Nuclear speckles",
5: "Nuclear bodies",
6: "Endoplasmic reticulum",
7: "Golgi apparatus",
8: "Peroxisomes",
9: "Endosomes",
10: "Lysosomes",
11: "Intermediate filaments",
12: "Actin filaments",
13: "Focal adhesion sites",
14: "Microtubules",
15: "Microtubule ends",
16: "Cytokinetic bridge",
17: "Mitotic spindle",
18: "Microtubule organizing center",
19: "Centrosome",
20: "Lipid droplets",
21: "Plasma membrane",
22: "Cell junctions",
23: "Mitochondria",
24: "Aggresome",
25: "Cytosol",
26: "Cytoplasmic bodies",
27: "Rods & rings"
}<feature_engineering>
scores = []
criterion = torch.nn.CrossEntropyLoss()
val_pr_avg = [np.zeros(val_y.shape)for _ in range(num_train_epochs)]
test_pr_avg = np.zeros(( test_df.shape[0], 3))
submit['target_list'] = submit['Predicted'].map(lambda x: [int(a)for a in str(x ).split(' ')])
 submit['target_vec'] = submit['target_list'].map(lambda ck: [i in ck for i in range(max_idx+1)])
 all_labels = list(chain.from_iterable(submit['target_list'].values))
c_val = Counter(all_labels)
 n_keys = c_val.keys()
max_idx = max(n_keys)
 for k,v in name_label_dict.items() :
print(v, 'count:', c_val[k] if k in c_val else 0 )<install_modules>
for model_id in range(n_models):
model, optimizer = get_gap_model(bert_model, n_bertlayers, dropout,
steps_per_epoch, device)
 total_psize, trainalbe_psize = get_param_size(model)
 print(f"Total params: {total_psize}
Trainable params: {trainalbe_psize}")
 for e in range(num_train_epochs):
t1 = time.time()
tr_loss = run_epoch(model, train_loader, optimizer, criterion, device)
 val_pr = predict(model, val_loader, device)
 val_pr_avg[e] += val_pr
val_loss = log_loss(val_y, val_pr)
 val_avg_loss = log_loss(val_y, val_pr_avg[e] /(model_id + 1))
elapsed = time.time() - t1
print(f"Epoch:{e + 1} tr_loss:{tr_loss:.4f} val_loss:{val_loss:.4f}"
f" val_avg_loss: {val_avg_loss:.4f} time:{elapsed:.1f}s")
 scores.append({"model_id": model_id, "epoch": e + 1, "time": elapsed,
"tr_loss": tr_loss, "val_loss": val_loss, "val_avg_loss": val_avg_loss})
 test_pr = predict(model, test_loader, device)
 test_pr_avg += test_pr
del model, optimizer
torch.cuda.empty_cache()
!pip install fastai==0.7.0 --no-deps
!pip install torch==0.4.1 torchvision==0.2.1<import_modules>
df = pd.DataFrame(scores)
 pd.set_option("precision", 5)
 print("
Single model")
 print(df.groupby("epoch")[['tr_loss', 'val_loss']].mean())
 if n_models > 1:
print(f"
Avg of {n_models} models")
 print(df[df.model_id == n_models - 1][['epoch', 'val_avg_loss']].set_index('epoch'))
<define_variables><EOS> 
test_pr_avg /= n_models
df_sub = pd.DataFrame(test_pr_avg, columns=["A", "B", "NEITHER"])
 df_sub["ID"] = test_df.ID
df_sub.to_csv("submission.csv", index=False)