Kaggle competition: TGS Salt Identification Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentation Kaggle data source: tgs-salt-identification-challenge<normalization>
plt.style.use('seaborn-white')
 sns.set_style("white")
scaler = MinMaxScaler().fit(X )<normalization>
img_size_ori = 101
debug=False
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True)
X_scaled = scaler.transform(X )<prepare_x_and_y>
train_df = pd.read_csv(".. /input/train.csv", index_col="id", usecols=[0])
 depths_df = pd.read_csv(".. /input/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]
if debug:
train_df=train_df.iloc[:64*4]
test_df=test_df.iloc[:64*4]
y = trip_duration<import_modules>
train_df["images"] = [np.array(load_img(".. /input/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
from sklearn.model_selection import train_test_split<import_modules>
train_df["masks"] = [np.array(load_img(".. /input/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
from sklearn.model_selection import train_test_split<split>
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2 )
X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, random_state=96 )<split>
def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class )
X_train,X_val,y_train,y_val = \
train_test_split(X_train,y_train, random_state=144 )<import_modules>
def iouMetric(true, pred):
true = K.batch_flatten(true)
 pred = K.batch_flatten(pred)
 pred = K.cast(K.greater(pred, 0.5), K.floatx())
 intersec = true * pred
iou = K.sum(intersec)/(K.sum(true + pred - intersec)+ K.epsilon())
 return iou
import xgboost as xgb<prepare_x_and_y>
ids_train, ids_valid, x_train, x_valid, y_train, y_valid, cov_train, cov_test, depth_train, depth_test = train_test_split(
 train_df.index.values,
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1),
train_df.coverage.values,
train_df.z.values,
test_size=0.2, stratify=train_df.coverage_class, random_state=1337 )
train_xgb = xgb.DMatrix(X_train, label=y_train)
 cv_xgb = xgb.DMatrix(X_val , label=y_val)
 evallist = [(train_xgb, 'train'),(cv_xgb, 'valid')]<train_model>
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
 y_pred = K.cast(y_pred, 'float32')
 y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')
 intersection = y_true_f * y_pred_f
score = 2.* K.sum(intersection)/(K.sum(y_true_f)+ K.sum(y_pred_f))
return score
def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred)
 def bce_logdice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)- K.log(1.- dice_loss(y_true, y_pred))
def weighted_bce_loss(y_true, y_pred, weight):
epsilon = 1e-7
y_pred = K.clip(y_pred, epsilon, 1.- epsilon)
 logit_y_pred = K.log(y_pred /(1.- y_pred))
loss = weight *(logit_y_pred *(1.- y_true)+
K.log(1.+ K.exp(-K.abs(logit_y_pred)))+ K.maximum(-logit_y_pred, 0.))
 return K.sum(loss)/ K.sum(weight)
 def weighted_dice_loss(y_true, y_pred, weight):
smooth = 1.
 w, m1, m2 = weight, y_true, y_pred
intersection =(m1 * m2)
 score =(2.* K.sum(w * intersection)+ smooth)/(K.sum(w * m1)+ K.sum(w * m2)+ smooth)
 loss = 1.- K.sum(score)
 return loss
def weighted_bce_dice_loss(y_true, y_pred):
y_true = K.cast(y_true, 'float32')
 y_pred = K.cast(y_pred, 'float32')
 averaged_mask = K.pool2d(
 y_true, pool_size=(50, 50), strides=(1, 1), padding='same', pool_mode='avg')
 weight = K.ones_like(averaged_mask)
 w0 = K.sum(weight)
 weight = 5.* K.exp(-5.* K.abs(averaged_mask - 0.5))
w1 = K.sum(weight)
 weight *=(w0 / w1)
 loss = weighted_bce_loss(y_true, y_pred, weight)+ dice_loss(y_true, y_pred)
 return loss
param = {'max_depth':10,
'objective':'reg:linear',
'eta' :.1,
'subsample':0.3,
'lambda ' :6,
'colsample_bytree ':0.3,
'colsample_bylevel':1,
'min_child_weight': 20,
'nthread': -1}
model = xgb.train(param, train_xgb, num_boost_round=1000, evals = evallist,
early_stopping_rounds=50, maximize=False,
verbose_eval=50)
 print("score = {:.5f}, n_boost_round = {}".format(model.best_score,
model.best_iteration))<prepare_x_and_y>
def conv_block(m, dim, acti, bn, res, do=0):
	n = Conv2D(dim, 3, activation=acti, padding='same' )(m)
 	n = BatchNormalization()(n)if bn else n
	n = Dropout(do )(n)if do else n
	n = Conv2D(dim, 3, activation=acti, padding='same' )(n)
 	n = BatchNormalization()(n)if bn else n
	return Concatenate()([m, n])if res else n
def level_block(m, dim, depth, inc, acti, do, bn, mp, up, res):
	if depth > 0:
		n = conv_block(m, dim, acti, bn, res)
 		m = MaxPooling2D()(n)if mp else Conv2D(dim, 3, strides=2, padding='same' )(n)
 		m = level_block(m, int(inc*dim), depth-1, inc, acti, do, bn, mp, up, res)
 		if up:
			m = UpSampling2D()(m)
 			m = Conv2D(dim, 2, activation=acti, padding='same' )(m)
 		else:
			m = Conv2DTranspose(dim, 3, strides=2, activation=acti, padding='same' )(m)
 		n = Concatenate()([n, m])
 		m = conv_block(n, dim, acti, bn, res)
 	else:
		m = conv_block(m, dim, acti, bn, res, do)
 	return m
def UNet(img_shape, out_ch=1, start_ch=64, depth=4, inc_rate=2., activation='relu',
		 dropout=0.5, batchnorm=False, maxpool=True, upconv=True, residual=False):
	i = Input(shape=img_shape)
 	o = level_block(i, start_ch, depth, inc_rate, activation, dropout, batchnorm, maxpool, upconv, residual)
 	o = Conv2D(out_ch, 1, activation='sigmoid' )(o)
 	return Model(inputs=i, outputs=o )
X_real_test = test.values<normalization>
img_size_target=128
model = UNet(( img_size_target,img_size_target,1),start_ch=16,depth=5,batchnorm=True )
X_test_scaled = scaler.transform(X_real_test )<predict_on_test>
model.compile(loss=bce_dice_loss, optimizer="adam", metrics=["accuracy",iouMetric] )
test_xgb = xgb.DMatrix(X_test_scaled)
 y_pred = model.predict(test_xgb )<prepare_output>
early_stopping = EarlyStopping(patience=20, verbose=1)
 model_checkpoint = ModelCheckpoint("./keras.model", save_best_only=True, verbose=1)
 reduce_lr = ReduceLROnPlateau(factor=0.2, patience=10, min_lr=0.000005, verbose=1)
 def step_decay(epoch):
if epoch<=60:
return 0.01
elif epoch<=90:
return 0.005
elif epoch<=110:
return 0.001
else:
return 0.0005
epochs = 120
batch_size = 32
if debug:
epochs=3
gen = ImageDataGenerator()
def random_crop_keep_size(img,mask,crop_ratio):
size_x=img.shape[0]
size_y=img.shape[1]
size_x_=int(size_x*crop_ratio)
 size_y_=int(size_y*crop_ratio)
 x = random.randint(0, size_x - size_x_ + 1)
 y = random.randint(0, size_y - size_y_ + 1)
 img_crop=img[x:x+size_x_,y:y+size_y]
mask_crop=mask[x:x+size_x_,y:y+size_y]
img_crop=cv2.resize(img_crop,(size_x,size_y))
mask_crop=cv2.resize(mask_crop,(size_x,size_y))
return img_crop,mask_crop
def reflective_pad(img):
return np.pad(img,(( 13,14),(13,14)) ,'reflect')
 seq = iaa.Sequential([
iaa.Fliplr(0.5),
iaa.OneOf([
iaa.Crop(px=(0,5)) ,
iaa.Noop() ,
iaa.Noop() ,
iaa.Noop() ,
]),
])
 def gen_flow_for_two_inputs(X, y):
genX1 = gen.flow(X,y, batch_size=batch_size)
 while True:
X=genX1.next()
img_mask=np.concatenate([X[0],X[1]],axis=3)
 img_mask_aug=seq.augment_images(img_mask)
 yield np.expand_dims(img_mask_aug[:,:,:,0],axis=3),np.expand_dims(img_mask_aug[:,:,:,1],axis=3)
 print(x_valid.shape)
 gen_flow = gen_flow_for_two_inputs(x_train, y_train)
 history = model.fit_generator(gen_flow,
validation_data=[x_valid, y_valid],
epochs=epochs,
steps_per_epoch=len(x_train)/ batch_size,
callbacks=[early_stopping, model_checkpoint, reduce_lr] )
y_pred = np.exp(y_pred[:])- 1<load_from_csv>
model = load_model("./keras.model",{"bce_dice_loss":bce_dice_loss,'iouMetric':iouMetric} )
submission = pd.read_csv('.. /input/sample_submission.csv', index_col=0, header=0 )<feature_engineering>
preds_valid = model.predict(x_valid ).reshape(-1, img_size_target, img_size_target)
 preds_valid = np.array([downsample(x)for x in preds_valid])
 y_valid_ori = np.array([train_df.loc[idx].masks for idx in ids_valid] )
submission.trip_duration = y_pred.round(0)
 submission.head(10 )<save_to_csv>
thresholds = np.linspace(0, 1, 50)
 ious = np.array([iou_metric_batch(y_valid_ori, np.int32(preds_valid > threshold)) for threshold in tqdm_notebook(thresholds)] )
submission.to_csv('submission.csv' )<install_modules>
threshold_best_index = np.argmax(ious[9:-10])+ 9
iou_best = ious[threshold_best_index]
threshold_best = thresholds[threshold_best_index]
!pip install.. /input/apex01cp37cp37mlinux-x86-64whl/apex-0.1-cp37-cp37m-linux_x86_64.whl
!pip install --upgrade.. /input/tokenizers0011cp37/tokenizers-0.0.11-cp37-cp37m-manylinux1_x86_64.whl
!pip install --upgrade.. /input/sacremoses-repo/sacremoses-0.0.35-py3-none-any.whl
!pip install --upgrade.. /input/transformers-repo/transformers-2.3.0-py3-none-any.whl<import_modules>
def RLenc(img, order='F', format=True):

 bytes = img.reshape(img.shape[0] * img.shape[1], order=order)
 runs = []
r = 0
pos = 1
for c in bytes:
if(c == 0):
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
pos += 1
else:
r += 1
if r != 0:
runs.append(( pos, r))
pos += r
r = 0
if format:
z = ''
for rr in runs:
z += '{} {} '.format(rr[0], rr[1])
 return z[:-1]
else:
return runs
from transformers import BertConfig, BertTokenizer, RobertaConfig, RobertaTokenizer 
<predict_on_test>
preds_test = model.predict(x_test )
def bert_large_predict(data_dict, id_list, id_candidate_len_dict, id_candidate_list_sorted, model_dir, word_len):
class TFQADataset(Dataset):
def __init__(self, id_list):
self.id_list=id_list
def __len__(self):
return len(self.id_list)
 def __getitem__(self, index):
return self.id_list[index]
class Collator(object):
def __init__(self, data_dict, new_token_dict, tokenizer, max_seq_len=384, max_question_len=64):
self.data_dict = data_dict
self.new_token_dict = new_token_dict
self.tokenizer = tokenizer
self.max_seq_len = max_seq_len
self.max_question_len = max_question_len
def _get_input_ids(self, doc_id, candidate_index):
data = self.data_dict[doc_id]
question_tokens = self.tokenizer.tokenize(data['question_text'])[:self.max_question_len]
doc_words = data['document_text'].split()
max_answer_tokens = self.max_seq_len-len(question_tokens)-3
candidate = data['long_answer_candidates'][candidate_index]
candidate_start = candidate['start_token']
candidate_end = candidate['end_token']
candidate_words = doc_words[candidate_start:candidate_end]
for i, word in enumerate(candidate_words):
if re.match(r'<.+>', word):
if word in self.new_token_dict:
candidate_words[i] = self.new_token_dict[word]
else:
candidate_words[i] = '<'
words_to_tokens_index = []
candidate_tokens = []
for i, word in enumerate(candidate_words):
words_to_tokens_index.append(len(candidate_tokens))
tokens = self.tokenizer.tokenize(word)
 if len(candidate_tokens)+len(tokens)> max_answer_tokens:
break
for token in tokens:
candidate_tokens.append(token)
 input_tokens = ['[CLS]'] + question_tokens + ['[SEP]'] + candidate_tokens + ['[SEP]']
input_ids = self.tokenizer.convert_tokens_to_ids(input_tokens)
 return input_ids, words_to_tokens_index, len(input_ids), len(question_tokens)+2
def __call__(self, batch_ids):
batch_size = len(batch_ids)
 batch_input_ids_temp = []
batch_seq_len = []
batch_offset = []
batch_words_to_tokens_index = []
for i,(doc_id, candidate_index)in enumerate(batch_ids):
input_ids, words_to_tokens_index, seq_len, offset = self._get_input_ids(doc_id, candidate_index)
 batch_input_ids_temp.append(input_ids)
 batch_seq_len.append(seq_len)
 batch_offset.append(offset)
 batch_words_to_tokens_index.append(words_to_tokens_index)
 batch_max_seq_len = max(batch_seq_len)
 batch_input_ids = np.zeros(( batch_size, batch_max_seq_len), dtype=np.int64)
 batch_token_type_ids = np.ones(( batch_size, batch_max_seq_len), dtype=np.int64)
 for i in range(batch_size):
input_ids = batch_input_ids_temp[i]
batch_input_ids[i, :len(input_ids)] = input_ids
batch_token_type_ids[i, :len(input_ids)] = [0 if k<=input_ids.index(102)else 1 for k in range(len(input_ids)) ]
batch_attention_mask = batch_input_ids > 0
return torch.from_numpy(batch_input_ids), torch.from_numpy(batch_attention_mask), torch.from_numpy(batch_token_type_ids), batch_words_to_tokens_index, batch_offset, batch_max_seq_len
class BertForQuestionAnswering(BertPreTrainedModel):
def __init__(self, config):
super(BertForQuestionAnswering, self ).__init__(config)
 self.bert = BertModel(config)
 self.qa_outputs = nn.Linear(config.hidden_size, 2)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.classifier = nn.Linear(config.hidden_size, config.num_labels)
 self.init_weights()
def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
outputs = self.bert(input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask)
 sequence_output = outputs[0]
pooled_output = outputs[1]
qa_logits = self.qa_outputs(sequence_output)
 start_logits, end_logits = qa_logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 pooled_output = self.dropout(pooled_output)
 classifier_logits = self.classifier(pooled_output)
 return start_logits, end_logits, classifier_logits
max_seq_len = 360
max_question_len = 64
batch_size = int(384/4)
 model_path = '.. /models/models/bert/'
config = BertConfig.from_pretrained(model_path)
 config.num_labels = 5
config.vocab_size = 30531
tokenizer = BertTokenizer.from_pretrained(model_path, do_lower_case=True)
 model = BertForQuestionAnswering.from_pretrained(model_dir, config=config)
 new_token_dict = {
'<P>':'qw1',
'<Table>':'qw2',
'<Tr>':'qw3',
'<Ul>':'qw4',
'<Ol>':'qw5',
'<Fl>':'qw6',
'<Li>':'qw7',
'<Dd>':'qw8',
'<Dt>':'qw9',
}
new_token_list = [
'qw1',
'qw2',
'qw3',
'qw4',
'qw5',
'qw6',
'qw7',
'qw8',
'qw9',
]
num_added_toks = tokenizer.add_tokens(new_token_list)
 model.resize_token_embeddings(len(tokenizer))
model.cuda()
optimizer = optim.Adam(model.parameters() , lr=1e-5)
 model, optimizer = amp.initialize(model, optimizer, opt_level="O1",verbosity=0)
 if torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
 test_datagen = TFQADataset(id_list=id_candidate_list_sorted)
 test_collate = Collator(data_dict=data_dict,
new_token_dict=new_token_dict,
tokenizer=tokenizer,
max_seq_len=max_seq_len,
max_question_len=max_question_len)
 test_generator = DataLoader(dataset=test_datagen,
collate_fn=test_collate,
batch_size=batch_size,
shuffle=False,
num_workers=16,
pin_memory=True)
 model.eval()
list_offset = []
list_words_to_tokens_index = []
test_prob1 = np.zeros(( len(id_candidate_list_sorted),max_seq_len),dtype=np.float32)
 test_prob2 = np.zeros(( len(id_candidate_list_sorted),max_seq_len),dtype=np.float32)
 test_prob3 = np.zeros(( len(id_candidate_list_sorted),5),dtype=np.float32)
 for j,(batch_input_ids, batch_attention_mask, batch_token_type_ids, batch_words_to_tokens_index, batch_offset, batch_max_seq_len)in tqdm(enumerate(test_generator)) :
with torch.no_grad() :
start = j*batch_size
end = start+batch_size
if j == len(test_generator)-1:
end = len(test_generator.dataset)
 batch_input_ids = batch_input_ids.cuda()
batch_attention_mask = batch_attention_mask.cuda()
batch_token_type_ids = batch_token_type_ids.cuda()
logits1, logits2, logits3 = model(batch_input_ids, batch_attention_mask, batch_token_type_ids)
 test_prob1[start:end, :batch_max_seq_len] += F.softmax(logits1,dim=1 ).cpu().data.numpy()
test_prob2[start:end, :batch_max_seq_len] += F.softmax(logits2,dim=1 ).cpu().data.numpy()
test_prob3[start:end] += F.softmax(logits3,dim=1 ).cpu().data.numpy()
list_words_to_tokens_index += batch_words_to_tokens_index
list_offset += batch_offset
test_word_prob1 = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 test_word_prob2 = np.zeros(( len(id_candidate_list_sorted),word_len),dtype=np.float32)
 for i in range(len(id_candidate_list_sorted)) :
for j in range(len(list_words_to_tokens_index[i])) :
test_word_prob1[i,j] = test_prob1[i, list_words_to_tokens_index[i][j]+list_offset[i]]
test_word_prob2[i,j] = test_prob2[i, list_words_to_tokens_index[i][j]+list_offset[i]]
return test_word_prob1, test_word_prob2, test_prob3 
<categorify>
pred_dict = {idx: RLenc(np.round(downsample(preds_test[i])> threshold_best)) for i, idx in enumerate(tqdm_notebook(test_df.index.values)) }
<define_variables><EOS> 
sub = pd.DataFrame.from_dict(pred_dict,orient='index')
 sub.index.names = ['id']
sub.columns = ['rle_mask']
sub.to_csv('submission.csv' )