Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<split>
iterations_learning = 24
batch_size_learning= 4096
importance = 0.9999
splits_kfold = 4
train_batch_size = 2048
train_epochs = 15
directory_emb = '.. /input/embeddings/'
directory_data = '.. /input/'
def load_glove(word_index):
EMBEDDING_FILE = directory_emb+'glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = directory_emb+'wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = directory_emb + 'paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
print(emb_mean,emb_std,"para")
 nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
maxlen = 256
docs = pad_sequences(sequences=docs, maxlen=max_size)
 print("Samples Number:", len(docs))
print("Sample 1:
{}
Sample 2:
{}".format(docs[0][-40:], docs[1][-40:]))<define_variables>
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D, RepeatVector, Flatten
from keras.layers import Bidirectional, GlobalMaxPool1D, Permute, Lambda, Layer, BatchNormalization, GRU
from keras.models import Model
from keras import initializers, regularizers, constraints, optimizers, layers, callbacks
from keras.callbacks import Callback
input_dim = np.max(docs)+ 1
embedding_dims = 20
input_dim<split>
max_features = 95000
maxlen = 72
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"that's" : "that is",
"there's" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text)
x_train, x_test, y_train, y_test = train_test_split(docs, y, test_size=0.20 )<choose_model_class>
train_df = pd.read_csv(directory_data+"train.csv")
 test_df = pd.read_csv(directory_data+"test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
def create_model(embedding_dims=20, optimizer='adam'):
model = Sequential()
model.add(Embedding(input_dim=input_dim, output_dim=embedding_dims))
model.add(GlobalAveragePooling1D())
 model.add(Dense(3, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
 return model<train_model>
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not",
"didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not",
"he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will",
"how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have",
"I'm": "I am", "I've": "I have", "i'd": "i would",
"i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am",
"i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have",
"it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us",
"ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not",
"mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have",
"needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not",
"oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have",
"she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have",
"she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as",
"this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would",
"there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have",
"they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have",
"wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have",
"we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have",
"what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have",
"where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have",
"who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not",
"won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all",
"y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are",
"y'all've": "you all have","you'd": "you would",
"you'd've": "you would have", "you'll": "you will", "you'll've": "you will have",
"you're": "you are", "you've": "you have" }
punct = "/-'?!.,
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "",
"€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-",
"’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"',
"£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha',
'•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling',
'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2',
'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary',
'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are',
'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I',
'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating',
'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota',
'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization',
'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'trump':'president', 'obama':'president', 'potus':'president'}
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding")
 def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
def unknown_punct(embed, punct):
unknown = ''
for p in punct:
if p not in embed:
unknown += p
unknown += ' '
return unknown
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
def transform_df(dfs):
dfs['question_text'] = dfs['question_text'].apply(lambda x: x.lower())
 dfs['question_text'] = dfs['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))
dfs['question_text'] = dfs['question_text'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))
dfs['question_text'] = dfs['question_text'].apply(lambda x: correct_spelling(x, mispell_dict))
transform_df(train_df)
 transform_df(test_df )
epochs = 500
hist = model.fit(x_train, y_train,
batch_size=16,
validation_data=(x_test, y_test),
epochs=epochs,
callbacks=[EarlyStopping(patience=2, monitor='val_loss')] )<load_from_csv>
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 95000
maxlen = 72
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
test_df = pd.read_csv('.. /input/test.csv')
 docs = create_docs(test_df)
 docs = tokenizer.texts_to_sequences(docs)
 docs = pad_sequences(sequences=docs, maxlen=maxlen)
 y = model.predict_proba(docs)
 result = pd.read_csv('.. /input/sample_submission.csv')
 for a, i in a2c.items() :
result[a] = y[:, i]<save_to_csv>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
result.to_csv('fastText_result_01.csv', index=False )<feature_engineering>
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule)
 class DropConnect(Wrapper):
def __init__(self, layer, prob=1., **kwargs):
self.prob = prob
self.layer = layer
super(DropConnect, self ).__init__(layer, **kwargs)
 if 0.< self.prob < 1.:
self.uses_learning_phase = True
def build(self, input_shape):
if not self.layer.built:
self.layer.build(input_shape)
 self.layer.built = True
super(DropConnect, self ).build()
def compute_output_shape(self, input_shape):
return self.layer.compute_output_shape(input_shape)
 def call(self, x):
if 0.< self.prob < 1.:
self.layer.kernel = K.in_train_phase(K.dropout(self.layer.kernel, self.prob), self.layer.kernel)
 self.layer.bias = K.in_train_phase(K.dropout(self.layer.bias, self.prob), self.layer.bias)
 return self.layer.call(x)
<load_pretrained>
def f1(y_true, y_pred):
y_pred = K.round(y_pred)
 tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)
 tn = K.sum(K.cast(( 1-y_true)*(1-y_pred), 'float'), axis=0)
 fp = K.sum(K.cast(( 1-y_true)*y_pred, 'float'), axis=0)
 fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)
 p = tp /(tp + fp + K.epsilon())
 r = tp /(tp + fn + K.epsilon())
 f1 = 2*p*r /(p+r+K.epsilon())
 f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)
 return K.mean(f1)
 def f1_loss(y_true, y_pred):
tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)
 tn = K.sum(K.cast(( 1-y_true)*(1-y_pred), 'float'), axis=0)
 fp = K.sum(K.cast(( 1-y_true)*y_pred, 'float'), axis=0)
 fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)
 p = tp /(tp + fp + K.epsilon())
 r = tp /(tp + fn + K.epsilon())
 f1 = 2*p*r /(p+r+K.epsilon())
 f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)
 return 1 - K.mean(f1)
 def my_model_lstm(topic,CONTEXT_LENGTH, embedding_size, max_features, embedding_matrix):
L = CONTEXT_LENGTH
NEURONS = 128
inp = Input(shape=(L,))
encoding_input = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 drop_out = Dropout(0.1, name='dropout' )(encoding_input)
 lstm_fwd = LSTM(NEURONS, return_sequences=True, name='lstm_fwd' )(drop_out)
 lstm_bwd = LSTM(NEURONS, return_sequences=True, go_backwards=True, name='lstm_bwd' )(drop_out)
 bilstm = keras.layers.concatenate([lstm_fwd, lstm_bwd], name='bilstm')
 drop_out = Dropout(0.1 )(bilstm)
 topic_input = Input(shape=(topic,), name='top_in')
 attention_topic = Dense(NEURONS, activation='linear' )(topic_input)
 attention_topic = RepeatVector(L )(attention_topic)
 attention = Dense(NEURONS, activation='linear' )(drop_out)
 attention = keras.layers.add([attention,attention_topic])
 attention = Dense(1, activation='tanh' )(attention)
 attention = Flatten()(attention)
 attention = Activation('softmax' )(attention)
 attention = RepeatVector(NEURONS*2 )(attention)
 attention = Permute([2, 1] )(attention)
 sent_representation = keras.layers.multiply([drop_out, attention])
 sent_representation = Lambda(lambda xin: K.sum(xin, axis=-2), output_shape=(NEURONS*2,))(sent_representation)
 sent_representation = BatchNormalization(axis=-1 )(sent_representation)
 bottleneck_features = Dense(NEURONS, activation='relu', name='bottleneck' )(sent_representation)
 out = Dense(1, activation='sigmoid' )(bottleneck_features)
 model = Model(input=[inp,topic_input], output=out)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy',f1])
 print(model.summary())
 return model
def my_small_model(size_intermediate):
inp = Input(shape=(size_intermediate,))
bn = BatchNormalization(axis=-1 )(inp)
 interm = Dense(64, activation='relu' )(bn)
 drop_out = Dropout(0.1 )(interm)
 interm = Dense(32, activation='relu' )(drop_out)
 drop_out = Dropout(0.1 )(interm)
 out = Dense(1, activation='sigmoid' )(drop_out)
 model = Model(input=inp, output=out)
 model.compile(loss=[binary_crossentropy], optimizer='adam', metrics=[f1])
 return model
def build_my_model(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.24 )(x)
 x = Bidirectional(CuDNNLSTM(80,
return_sequences=True,
kernel_initializer=glorot_normal(seed=1029),
recurrent_initializer=orthogonal(gain=1.0, seed=1029)) )(x)
 x_1 = Attention(maxlen )(x)
 x_1 = DropConnect(Dense(32, activation="relu"), prob=0.1 )(x_1)
 x_2 = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True )(x)
 x_2 = Flatten()(x_2)
 x_2 = DropConnect(Dense(32, activation="relu"), prob=0.1 )(x_2)
 conc = concatenate([x_1, x_2], name='bottleneck')
 outp = Dense(1, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer=Adam() , metrics=[f1])
 return model
int_outputs = ".. /input/intermediate-outputs"
with open(int_outputs+"/test_matrices_all.p", "rb")as filehandle:
test_matrices = pickle.load(filehandle )<count_values>
np.add([[1,2,3],[1,2,3]],[[4,5,6],[4,5,6]] )
def get_counts2(column, trees,stem = True):
t1 = time.clock()
stemmer = None
if stem: stemmer = nl.stem.snowball.SnowballStemmer("english")
 counts = []
for i in column.index:
tree = trees[i]
if tree:
counts_i = get_rels_ch3(tree,stemmer)
 counts.append(counts_i)
 else:
counts.append(None)
 t2 = time.clock()
print(i,":",t2 - t1)
 return counts<categorify>
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr() )
def get_probs(train_dict,test_list,ids,weights,smoothing=0.5):
t0=time.clock()
train_cats = {i:v.to_dense().fillna(0)for(i, v)in train_dict.items() if i != "TOTAL"}
train_total = train_dict["TOTAL"].to_dense().fillna(0)
 all_rows = train_total.index
all_columns = train_total.columns
results = pd.DataFrame(index=ids.index, columns=['id']+[i for i in train_cats],dtype=float)
 for(n,test_mat)in enumerate(test_list):
cat_probs = pd.Series({i:1.0 for i in train_cats})
 if test_mat is not None:
test_matrix = test_mat.to_dense().fillna(0)
 nonzeros = test_matrix[test_matrix>0].stack().stack()
for(j, count)in nonzeros.iteritems() :
row=(j[0],j[1] if(j[0],j[1])in all_rows else "_OTHER")
 column =(j[3] if(j[3],j[2])in all_columns else "_OTHER", j[2])
 column_total_nonzeros = train_total[column][train_total[column] >0].size
if column_total_nonzeros == 0:
column =("_OTHER", j[2])
 column_total_nonzeros = train_total[column][train_total[column] >0].size
for(cat,train_matrix)in train_cats.items() :
numerator = smoothing
denominator =(column_total_nonzeros+1)*smoothing
if(column in train_matrix.columns and column_total_nonzeros >=2):
denominator = denominator + train_matrix.loc[:,column].sum()
if(row in train_matrix.index):
numerator = numerator +train_matrix.loc[row,column]
cat_probs[cat] = cat_probs[cat]*pow(numerator / denominator,count)
 cat_probs = cat_probs/cat_probs.sum()
cat_probs=cat_probs * weights
results.loc[ids.index[n]] = cat_probs/cat_probs.sum()
print(n,":",time.clock() - t0)
 results['id'] = ids
return results<predict_on_test>
word_index = tokenizer.word_index
vocab = []
for w,k in word_index.items() :
vocab.append(w)
 if k >= max_features:
break
embedding_matrix = load_glove(word_index)
 embedding_matrix = np.add(embedding_matrix,load_fasttext(word_index))
embedding_matrix = np.add(embedding_matrix, load_para(word_index))
embedding_matrix = embedding_matrix/3
np.shape(embedding_matrix)
def probs_regress(train_dict,test_list,ids,weights,smoothing=0.5, regressors={}):
probs = get_probs(train_dict,test_list,ids,weights,smoothing)
 num_columns = probs.columns.drop('id')
 for i, v in regressors.items() :
probs[i] = regressors[i].predict(probs[i])
 results_sum = probs[num_columns].sum(axis=1)
 probs[num_columns] = probs[num_columns].divide(results_sum, axis=0)
 return probs<load_from_csv>
train_data = pd.read_csv('.. /input/spooky-author-identification/train.csv')
 test_data = pd.read_csv('.. /input/spooky-author-identification/test.csv')
 int_outputs = ".. /input/intermediate-outputs"
with open(int_outputs+"/test_matrices_all.p", "rb")as filehandle:
test_matrices_all = pickle.load(filehandle)
 with open(int_outputs+"/train_matrices_master.p", "rb")as filehandle:
train_matrices_master = pickle.load(filehandle)
 with open(int_outputs+"/train_matrices_calib.p", "rb")as filehandle:
train_matrices_calib = pickle.load(filehandle)
 with open(int_outputs+"/train_matrices_holdout.p", "rb")as filehandle:
train_matrices_holdout = pickle.load(filehandle)
 with open(int_outputs+"/isotonic_regs.p", "rb")as filehandle:
isotonic_regs = pickle.load(filehandle)
 probs = probs_regress(train_dict=train_matrices_master,
test_list = test_matrices_all,
ids = test_data['id'],
weights = train_data['author'][:16000].value_counts() ,
smoothing=0.5,
regressors=isotonic_regs 
)
 probs.to_csv('submission.csv',index=False )<categorify>
bottleneck_layer = model.get_layer(name='bottleneck')
 model_extractor = Model(inputs=model.get_input_at(0), outputs= [bottleneck_layer.get_output_at(0)])
 model_extractor.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
def prep2(train_dict, threshold):
train_cats = {i:v.to_dense().fillna(0)for(i, v)in train_dict.items() if i != "TOTAL"}
train_total = train_dict["TOTAL"].to_dense().fillna(0)
 row_sums=train_total.sum(axis=1)
 all_oth_rows = train_total.loc[row_sums < threshold,:].drop([("LEAF", "_OTHER"),("NODE", "_OTHER")],axis=0)
 all_oth_leaf = all_oth_rows.loc['LEAF']
all_oth_node = all_oth_rows.loc['NODE']
all_oth_leaf_ind = pd.MultiIndex.from_product([["LEAF"],all_oth_leaf.index])
 all_oth_node_ind = pd.MultiIndex.from_product([["NODE"],all_oth_node.index])
 train_total.loc[('LEAF', '_OTHER'),:] = all_oth_leaf.sum(axis=0)
 train_total.loc[('NODE', '_OTHER'),:] = all_oth_node.sum(axis=0)
 train_total = train_total.drop(all_oth_leaf_ind, axis=0)
 train_total = train_total.drop(all_oth_node_ind, axis=0)
 all_oth_l_ind = train_total.columns.intersection(pd.MultiIndex.from_product([all_oth_node.index,["L"]]))
all_oth_r_ind = train_total.columns.intersection(pd.MultiIndex.from_product([all_oth_node.index,["R"]]))
all_oth_c_ind = train_total.columns.intersection(pd.MultiIndex.from_product([all_oth_node.index,["C"]]))
all_oth_l = train_total.loc[:,all_oth_l_ind]
all_oth_r = train_total.loc[:,all_oth_r_ind]
all_oth_c = train_total.loc[:,all_oth_c_ind]
train_total.loc[:,('_OTHER', 'L')] = all_oth_l.sum(axis=1)
 train_total.loc[:,('_OTHER', 'R')] = all_oth_r.sum(axis=1)
 train_total.loc[:,('_OTHER', 'C')] = all_oth_c.sum(axis=1)
 train_total = train_total.drop(all_oth_l_ind, axis=1)
 train_total = train_total.drop(all_oth_r_ind, axis=1)
 train_total = train_total.drop(all_oth_c_ind, axis=1)
 for(i, matrix)in train_cats.items() :
oth_leaf_ind = matrix.index.intersection(all_oth_leaf_ind)
 oth_node_ind = matrix.index.intersection(all_oth_node_ind)
 oth_leaf = matrix.loc[oth_leaf_ind,:]
oth_node = matrix.loc[oth_node_ind,:]
train_cats[i].loc[('LEAF', '_OTHER'),:] = oth_leaf.sum(axis=0)
 train_cats[i].loc[('NODE', '_OTHER'),:] = oth_node.sum(axis=0)
 train_cats[i] = train_cats[i].drop(oth_leaf_ind, axis=0)
 train_cats[i] = train_cats[i].drop(oth_node_ind, axis=0)
 oth_l_ind = matrix.columns.intersection(all_oth_l_ind)
 oth_r_ind = matrix.columns.intersection(all_oth_r_ind)
 oth_c_ind = matrix.columns.intersection(all_oth_c_ind)
 oth_l = matrix.loc[:, oth_l_ind]
oth_r = matrix.loc[:, oth_r_ind]
oth_c = matrix.loc[:, oth_c_ind]
train_cats[i].loc[:,('_OTHER', 'L')] = oth_l.sum(axis=1)
 train_cats[i].loc[:,('_OTHER', 'R')] = oth_r.sum(axis=1)
 train_cats[i].loc[:,('_OTHER', 'C')] = oth_c.sum(axis=1)
 train_cats[i] = train_cats[i].drop(columns=oth_l_ind)
 train_cats[i] = train_cats[i].drop(columns=oth_r_ind)
 train_cats[i] = train_cats[i].drop(columns=oth_c_ind)
 return {'TOTAL':train_total, **train_cats}<define_variables>
clr = CyclicLR(base_lr=0.001, max_lr=0.002,
step_size=300., mode='exp_range',
gamma=0.99994 )
kernel_start = time.time()
LIMIT = 8.8
DO_TRAIN = True
DO_TEST = True
USE_TTA = True
RAND = 12345
!pip install tensorflow-gpu==1.14.0 --quiet
!pip install keras==2.2.4 --quiet
!pip install segmentation-models --quiet 
<feature_engineering>
check_point = ModelCheckpoint('mymodel20.wgt', save_best_only=True)
 early_stop = EarlyStopping(monitor='val_loss', patience=5, mode='min')
 model.fit([train_X], [train_y], batch_size=batch_size_learning,callbacks=[clr,check_point,early_stop], epochs=train_epochs, validation_data=([val_X], [val_y]))
sub = pd.read_csv('.. /input/understanding_cloud_organization/sample_submission.csv')
 sub['Image'] = sub['Image_Label'].map(lambda x: x.split('.')[0])
 sub['Label'] = sub['Image_Label'].map(lambda x: x.split('_')[1])
 sub['p'] = pd.read_csv('.. /input/cloud-classifiers/pred_cls.csv' ).p.values
sub['p'] += np.load('.. /input/cloud-classifiers/pred_cls0.npy' ).reshape(( -1)) * 0.5
sub['p'] += pd.read_csv('.. /input/cloud-classifiers/pred_cls3.csv' ).p.values * 3.0
sub['p'] += np.load('/kaggle/input/cloud-classifiers/pred_cls4b.npy')* 0.6
sub['p'] /= 5.1
train = pd.read_csv('.. /input/cloud-images-resized/train_384x576.csv')
 train['Image'] = train['Image_Label'].map(lambda x: x.split('.')[0])
 train['Label'] = train['Image_Label'].map(lambda x: x.split('_')[1])
 train2 = pd.DataFrame({'Image':train['Image'][::4]})
 train2['e1'] = train['EncodedPixels'][::4].values
train2['e2'] = train['EncodedPixels'][1::4].values
train2['e3'] = train['EncodedPixels'][2::4].values
train2['e4'] = train['EncodedPixels'][3::4].values
train2.set_index('Image',inplace=True,drop=True)
 train2.fillna('',inplace=True); train2.head()
train2[['d1','d2','d3','d4']] =(train2[['e1','e2','e3','e4']]!='' ).astype('int8')
 for k in range(1,5): train2['o'+str(k)] = 0
train2[['o1','o2','o3','o4']] = np.load('.. /input/cloud-classifiers/oof_cls.npy')
 train2[['o1','o2','o3','o4']] += np.load('.. /input/cloud-classifiers/oof_cls0.npy')* 0.5
train2[['o1','o2','o3','o4']] += np.load('.. /input/cloud-classifiers/oof_cls3.npy')* 3.0
train2[['o1','o2','o3','o4']] += np.load('.. /input/cloud-classifiers/oof_cls4b.npy')* 0.6
train2[['o1','o2','o3','o4']] /= 5.1
train2.head()<categorify>
model.load_weights('mymodel20.wgt' )
def mask2rleXXX(img0, shape=(576,384), grow=(525,350)) :

 a =(shape[1]-img0.shape[0])//2
b =(shape[0]-img0.shape[1])//2
img = np.zeros(( shape[1],shape[0]),dtype=np.uint8)
 img[a:a+img0.shape[0],b:b+img0.shape[1]] = img0
img = cv2.resize(img,grow)
 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle2maskX(mask_rle, shape=(2100,1400), shrink=1):
s = mask_rle.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(shape ).T[::shrink,::shrink]
def dice_coef(y_true, y_pred, smooth=1):
y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = K.sum(y_true_f * y_pred_f)
 return(2.* intersection + smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth )<define_variables>
class DataGenerator2(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, batch_size=24, shuffle=False, width=544, height=352, scale=1/128., sub=1., mode='train_seg',
path='.. /input/cloud-images-resized/train_images_384x576/', flips=False, augment=False, shrink1=1,
shrink2=1, dim=(576,384), clean=False):
'Initialization'
self.list_IDs = list_IDs
self.shuffle = shuffle
self.batch_size = batch_size
self.path = path
self.scale = scale
self.sub = sub
self.path = path
self.width = width
self.height = height
self.mode = mode
self.flips = flips
self.augment = augment
self.shrink1 = shrink1
self.shrink2 = shrink2
self.dim = dim
self.clean = clean
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
ct = int(np.floor(len(self.list_IDs)/ self.batch_size))
if len(self.list_IDs)>ct*self.batch_size: ct += 1
return int(ct)
 def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
X, msk = self.__data_generation(indexes)
 if self.augment: X, msk = self.__augment_batch(X, msk)
 if(self.mode=='train_seg')|(self.mode=='validate_seg'): return X, msk
else: return X
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(int(len(self.list_IDs)))
 if self.shuffle: np.random.shuffle(self.indexes)
 def __data_generation(self, indexes):
'Generates data containing batch_size samples'
lnn = len(indexes); ex = self.shrink1; ax = self.shrink2
X = np.empty(( lnn,self.height,self.width,3),dtype=np.float32)
 msk = np.empty(( lnn,self.height//ax,self.width//ax,4),dtype=np.int8)
 for k in range(lnn):
img = cv2.imread(self.path + self.list_IDs[indexes[k]]+'.jpg')
 img = img[::ex,::ex,:]
hflip = False; vflip = False
if(self.flips):
if np.random.uniform(0,1)>0.5: hflip=True
if np.random.uniform(0,1)>0.5: vflip=True
if vflip: img = cv2.flip(img,0)
 if hflip: img = cv2.flip(img,1)
 a = np.random.randint(0,self.dim[0]//ex//ax-self.width//ax+1)
 b = np.random.randint(0,self.dim[1]//ex//ax-self.height//ax+1)
 if(self.mode=='predict'):
a =(self.dim[0]//ex//ax-self.width//ax)//2
b =(self.dim[1]//ex//ax-self.height//ax)//2
img = img[b*ax:self.height+b*ax,a*ax:self.width+a*ax]
X[k,] = img*self.scale - self.sub
if(self.mode!='predict'):
for j in range(1,5):
rle = train2.loc[self.list_IDs[indexes[k]],'e'+str(j)]
if self.clean:
if train2.loc[self.list_IDs[indexes[k]],'o'+str(j)]<0.4:
rle = ''
mask = rle2maskX(rle,shrink=ex*ax,shape=self.dim)
 if vflip: mask = np.flip(mask,axis=0)
 if hflip: mask = np.flip(mask,axis=1)
 msk[k,:,:,j-1] = mask[b:self.height//ax+b,a:self.width//ax+a]
return X, msk
def __random_transform(self, img, masks):
composition = albu.Compose([
albu.ShiftScaleRotate(rotate_limit=30, scale_limit=0.1, p=0.5)
 ])
 composed = composition(image=img, mask=masks)
 aug_img = composed['image']
aug_masks = composed['mask']
return aug_img, aug_masks
def __augment_batch(self, img_batch, masks_batch):
for i in range(img_batch.shape[0]):
img_batch[i, ], masks_batch[i, ] = self.__random_transform(
 img_batch[i, ], masks_batch[i, ])
 return img_batch, masks_batch<choose_model_class>
thresh_saved = 0.1
max_f1 = 0
pred_noemb_val_y = model.predict([val_X], batch_size=1024, verbose=1)
class AdamAccumulate(Optimizer):
def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., amsgrad=False, accum_iters=8, **kwargs):
if accum_iters < 1:
raise ValueError('accum_iters must be >= 1')
 super(AdamAccumulate, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.amsgrad = amsgrad
self.accum_iters = K.variable(accum_iters, K.dtype(self.iterations))
self.accum_iters_float = K.cast(self.accum_iters, K.floatx())
 @interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
completed_updates = K.cast(K.tf.floordiv(self.iterations, self.accum_iters), K.floatx())
 if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * completed_updates))
t = completed_updates + 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) /(1.- K.pow(self.beta_1, t)))
 update_switch = K.equal(( self.iterations + 1)% self.accum_iters, 0)
 update_switch = K.cast(update_switch, K.floatx())
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
else:
vhats = [K.zeros(1)for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):
sum_grad = tg + g
avg_grad = sum_grad / self.accum_iters_float
m_t =(self.beta_1 * m)+(1.- self.beta_1)* avg_grad
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(avg_grad)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 p_t = p - lr_t * m_t /(K.sqrt(vhat_t)+ self.epsilon)
 self.updates.append(K.update(vhat,(1 - update_switch)* vhat + update_switch * vhat_t))
else:
p_t = p - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)
 self.updates.append(K.update(m,(1 - update_switch)* m + update_switch * m_t))
self.updates.append(K.update(v,(1 - update_switch)* v + update_switch * v_t))
self.updates.append(K.update(tg,(1 - update_switch)* sum_grad))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p,(1 - update_switch)* p + update_switch * new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad}
base_config = super(AdamAccumulate, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<compute_test_metric>
pred_noemb_test_y = model.predict([test_X], batch_size=1024, verbose=1)
def kaggle_acc(y_true, y_pred0, pix=0.5, area=24000, dim=(352,544)) :
y_pred = K.cast(K.greater(y_pred0,pix), K.floatx())
 s = K.sum(y_pred, axis=(1,2))
s = K.cast(K.greater(s, area), K.floatx())
 s = K.reshape(s,(-1,1))
s = K.repeat(s,dim[0]*dim[1])
 s = K.reshape(s,(-1,1))
y_pred = K.permute_dimensions(y_pred,(0,3,1,2))
y_pred = K.reshape(y_pred,shape=(-1,1))
y_pred = s*y_pred
y_pred = K.reshape(y_pred,(-1,y_pred0.shape[3],dim[0],dim[1]))
y_pred = K.permute_dimensions(y_pred,(0,2,3,1))
total_y_true = K.sum(y_true, axis=(1,2))
total_y_true = K.cast(K.greater(total_y_true, 0), K.floatx())
 total_y_pred = K.sum(y_pred, axis=(1,2))
total_y_pred = K.cast(K.greater(total_y_pred, 0), K.floatx())
 return 1 - K.mean(K.abs(total_y_pred - total_y_true))
def kaggle_dice(y_true, y_pred0, pix=0.5, area=24000, dim=(352,544)) :
y_pred = K.cast(K.greater(y_pred0,pix), K.floatx())
 s = K.sum(y_pred, axis=(1,2))
s = K.cast(K.greater(s, area), K.floatx())
 s = K.reshape(s,(-1,1))
s = K.repeat(s,dim[0]*dim[1])
 s = K.reshape(s,(-1,1))
y_pred = K.permute_dimensions(y_pred,(0,3,1,2))
y_pred = K.reshape(y_pred,shape=(-1,1))
y_pred = s*y_pred
y_pred = K.reshape(y_pred,(-1,y_pred0.shape[3],dim[0],dim[1]))
y_pred = K.permute_dimensions(y_pred,(0,2,3,1))
intersection = K.sum(y_true * y_pred, axis=(1,2))
total_y_true = K.sum(y_true, axis=(1,2))
total_y_pred = K.sum(y_pred, axis=(1,2))
return K.mean(( 2*intersection+1e-9)/(total_y_true+total_y_pred+1e-9))<define_search_space>
embedding_matrix = None
filters = [256, 128, 64, 32, 16]
REDUCTION = 0; RED = 2**REDUCTION
filters = filters[:5-REDUCTION]
BATCH_SIZE = 16
jaccard_loss = sm.losses.JaccardLoss()
skf = KFold(n_splits=3, shuffle=True, random_state=RAND)
 for k,(idxT0, idxV0)in enumerate(skf.split(train2)) :
train_idx = train2.index[idxT0]
val_idx = train2.index[idxV0]
if k==0: idx_oof_0 = val_idx.copy()
elif k==1: idx_oof_1 = val_idx.copy()
elif k==2: idx_oof_2 = val_idx.copy()
print('
print('
print('
if not DO_TRAIN: continue
train_generator = DataGenerator2(
 train_idx, flips=True, augment=True, shuffle=True, shrink2=RED, batch_size=BATCH_SIZE, 
)
 val_generator = DataGenerator2(
 val_idx, shrink2=RED, batch_size=BATCH_SIZE 
)
 opt = AdamAccumulate(lr=0.001, accum_iters=8)
 model = sm.Unet(
 'efficientnetb2',
classes=4,
encoder_weights='imagenet',
decoder_filters = filters,
input_shape=(None, None, 3),
activation='sigmoid' 
)
 model.compile(optimizer=opt, loss=jaccard_loss, metrics=[dice_coef,kaggle_dice,kaggle_acc])
 checkpoint = ModelCheckpoint('model_'+str(k)+'.h5', save_best_only=True)
 es = EarlyStopping(monitor='val_dice_coef', min_delta=0.001, patience=5, verbose=1, mode='max')
 rlr = ReduceLROnPlateau(monitor='val_dice_coef', factor=0.5, patience=2, verbose=1, mode='max', min_delta=0.001)
 history = model.fit_generator(
 train_generator,
validation_data=val_generator,
callbacks=[rlr, es, checkpoint],
epochs=30,
verbose=2, workers=2 
)
 history_df = pd.DataFrame(history.history)
 history_df.to_csv('history_'+str(k)+'.csv', index=False)
 del train_idx, val_idx, train_generator, val_generator, opt, model, checkpoint, es, rlr, history, history_df
K.clear_session() ; x=gc.collect()<load_pretrained>
new_train_X = model_extractor.predict([train_X], batch_size=1024, verbose=1)
 new_val_X = model_extractor.predict([val_X], batch_size=1024, verbose=1)
 new_test_X= model_extractor.predict([test_X], batch_size=1024, verbose=1)
 scl = StandardScaler()
!pip install tta-wrapper --quiet
if DO_TEST:
model1 = load_model('model_0.h5',custom_objects={'dice_coef':dice_coef,
'jaccard_loss':jaccard_loss,'AdamAccumulate':AdamAccumulate,
'kaggle_dice':kaggle_dice,'kaggle_acc':kaggle_acc})
 if USE_TTA:
model1 = tta_segmentation(model1, h_flip=True, h_shift=(-10, 10), v_flip=True, v_shift=(-10, 10), merge='mean')
 model2 = load_model('model_1.h5',custom_objects={'dice_coef':dice_coef,
'jaccard_loss':jaccard_loss,'AdamAccumulate':AdamAccumulate,
'kaggle_dice':kaggle_dice,'kaggle_acc':kaggle_acc})
 if USE_TTA:
model2 = tta_segmentation(model2, h_flip=True, h_shift=(-10, 10), v_flip=True, v_shift=(-10, 10), merge='mean')
 model3 = load_model('model_2.h5',custom_objects={'dice_coef':dice_coef,
'jaccard_loss':jaccard_loss,'AdamAccumulate':AdamAccumulate,
'kaggle_dice':kaggle_dice,'kaggle_acc':kaggle_acc})
 if USE_TTA:
model3 = tta_segmentation(model3, h_flip=True, h_shift=(-10, 10), v_flip=True, v_shift=(-10, 10), merge='mean' )<define_search_space>
new_train_X_scaled = scl.fit_transform(new_train_X)
 new_val_X_scaled = scl.transform(new_val_X)
 new_test_X_scaled = scl.transform(new_test_X )
print('Computing masks for',len(sub)//4,'test images with 3 models'); sub.EncodedPixels = ''
PTH = '.. /input/cloud-images-resized/test_images_384x576/'; bs = 4
if USE_TTA: bs=1
test_gen = DataGenerator2(sub.Image[::4].values, width=576, height=384, batch_size=bs, mode='predict',path=PTH)
 sz = 20000.*(576/525)*(384/350)/RED/RED
pixt = [0.5,0.5,0.5,0.35]
szt = [25000., 20000., 22500., 15000.]
for k in range(len(szt)) : szt[k] = szt[k]*(576./525.) *(384./350.) /RED/RED
if DO_TEST:
for b,batch in enumerate(test_gen):
btc = model1.predict_on_batch(batch)
 btc += model2.predict_on_batch(batch)
 btc += model3.predict_on_batch(batch)
 btc /= 3.0
for j in range(btc.shape[0]):
for i in range(btc.shape[-1]):
mask =(btc[j,:,:,i]>pixt[i] ).astype(int); rle = ''
if np.sum(mask)>szt[i]: rle = mask2rleXXX(mask ,shape=(576//RED,384//RED))
sub.iloc[4*(bs*b+j)+i,1] = rle
if b%(100//bs)==0: print(b*bs,', ',end='')
 t = np.round(( time.time() - kernel_start)/60,1)
 if t > LIMIT*60:
print('
break
sub[['Image_Label','EncodedPixels']].to_csv('sub_seg.csv',index=False)
 sub.loc[(sub.p<0.5)&(sub.Label=='Fish'),'EncodedPixels'] = ''
sub.loc[(sub.p<0.3)&(sub.Label=='Flower'),'EncodedPixels'] = ''
sub.loc[(sub.p<0.5)&(sub.Label=='Gravel'),'EncodedPixels'] = ''
sub.loc[(sub.p<0.5)&(sub.Label=='Sugar'),'EncodedPixels'] = ''
sub[['Image_Label','EncodedPixels']].to_csv('submission.csv',index=False)
 sub.head(10 )<install_modules>
neg_sample = 500000
pos_sample = 40000
X_smote = None
X_smote_neg = np.array(sample(new_train_X_scaled[train_y==0,:].tolist() ,neg_sample))
X_smote_pos = np.array(sample(new_train_X_scaled[train_y==1,:].tolist() ,pos_sample))
y_neg = np.zeros(X_smote_neg.shape[0])
 y_pos = np.zeros(X_smote_pos.shape[0])+1
to_smote = np.append(X_smote_neg, X_smote_pos,axis=0)
 to_smote_y = np.append(y_neg, y_pos)
 print(np.shape(to_smote),np.shape(to_smote_y))
! pip install git+https://github.com/qubvel/classification_models.git<set_options>
sm = SMOTE(random_state=2)
 X_train_res, y_train_res = sm.fit_sample(to_smote, to_smote_y )
seed(10)
 set_random_seed(10)
 %matplotlib inline<import_modules>
new_x_pos = X_train_res[y_train_res==1,:][pos_sample:]
new_y_pos = y_train_res[y_train_res==1][pos_sample:]
new_train_X_scaled_smoted=np.append(new_train_X_scaled,new_x_pos,axis=0)
 train_y_smoted = np.append(train_y,new_y_pos)
 print(np.shape(new_train_X_scaled_smoted))
print(np.shape(train_y_smoted))
from classification_models.keras import Classifiers<define_variables>
check_point2 = ModelCheckpoint('intermediate.wgt', save_best_only=True)
 def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
def train_pred(model, train_X, train_y, X_val, y_val, epochs=2, callback=None):
model.fit([train_X], [train_y], batch_size=batch_size_learning,callbacks=[clr,early_stop,check_point2], epochs=epochs, validation_data=([X_val], [y_val]))
model.load_weights('intermediate.wgt')
 pred_val_y = model.predict([new_val_X_scaled], batch_size=1024, verbose=0)
 mythres = threshold_search(val_y,pred_val_y)
 thr = mythres['threshold']
best_score = metrics.f1_score(val_y,(pred_val_y > thr ).astype(int))
print("SPLIT Val F1 Score: {:.4f}".format(best_score))
pred_test_y = model.predict([new_test_X_scaled], batch_size=1024, verbose=0)
 print('=' * 60)
 return pred_val_y, pred_test_y, best_score
DATA_SPLIT_SEED = 2018
valid_meta = np.zeros(val_X.shape[0])
 test_meta = np.zeros(test_X.shape[0])
 splits = list(StratifiedKFold(n_splits=splits_kfold, shuffle=True, random_state=DATA_SPLIT_SEED ).split(new_train_X_scaled_smoted, train_y_smoted))
count = 0
for idx,(train_idx, valid_idx)in enumerate(splits):
X_train = new_train_X_scaled_smoted[train_idx]
y_train = train_y_smoted[train_idx]
X_val = new_train_X_scaled_smoted[valid_idx]
y_val = train_y_smoted[valid_idx]
model = my_small_model(64)
 pred_val_y, pred_test_y, best_score = train_pred(model, X_train, y_train, X_val, y_val, epochs = iterations_learning, callback = [clr])
 valid_meta += pred_val_y.reshape(-1)
 test_meta += pred_test_y.reshape(-1)
 count = count +1
test_imgs_folder = '.. /input/understanding_cloud_organization/test_images/'
train_imgs_folder = '.. /input/understanding_cloud_organization/train_images/'
num_cores = multiprocessing.cpu_count()<load_from_csv>
valid_meta = valid_meta/count
test_meta = test_meta/count
threshold_search(val_y,valid_meta)
 mythres = threshold_search(val_y,valid_meta)
 thr = mythres['threshold']
print('-'*60)
 print('thr')
 print(mythres )
train_df = pd.read_csv('.. /input/understanding_cloud_organization/train.csv')
 train_df.head()<feature_engineering>
predict_final =(test_meta>thr ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = predict_final
out_df.to_csv("submission.csv", index=False )
train_df = train_df[~train_df['EncodedPixels'].isnull() ]
train_df['Image'] = train_df['Image_Label'].map(lambda x: x.split('_')[0])
 train_df['Class'] = train_df['Image_Label'].map(lambda x: x.split('_')[1])
 classes = train_df['Class'].unique()
train_df = train_df.groupby('Image')['Class'].agg(set ).reset_index()
for class_name in classes:
train_df[class_name] = train_df['Class'].map(lambda x: 1 if class_name in x else 0)
 train_df.head()<define_variables>
<concatenate><EOS>