Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<normalization>
LSTM, GRU, GlobalMaxPool1D, Dense)
 ReduceLROnPlateau)
 tqdm(tqdm_notebook ).pandas()
print(os.listdir(".. /input"))
datapath='.. /input'
RANDOM_STATE = 2
SHUFFLE = True
TEST_SIZE = 0.8
THRESHOLD = 0.35
cat_dict = dict()
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 for i in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 cat_dict[i] = train3.shape[1]<feature_engineering>
@contextmanager
def timer(name):

 t0 = time.time()
yield
print(f'[{name}] done in {time.time() - t0:.0f} s')
 def load_trained_model(model, weights_path):
model.load_weights(weights_path)
 return model
test['target'] = preds
oof_var = np.zeros(len(train))
preds_var = np.zeros(len(test))
for k in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k]
test2p = test2[(test2['target']<=0.01)|(test2['target']>=0.99)].copy()
test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 pca = PCA(n_components=cat_dict[k], random_state= 1234)
 pca.fit(train2p[cols])
 train3p = pca.transform(train2p[cols])
 train3 = pca.transform(train2[cols])
 test3 = pca.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3p, train2p['target']):
test_index3 = test_index[ test_index<len(train3)]
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_var[idx1[test_index3]] += clf.predict_proba(train3[test_index3,:])[:,1]
preds_var[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'],oof_var)
 print('Pseudo Labeled QDA scores CV =',round(auc,5)) 
<feature_engineering>
class DataReader(object):
def __init__(self,
train_file,
module,
test_file=None):
if not train_file:
raise Exception("DataReader requires a train_file!")
 if not module:
raise Exception("DataReader requires a model that can transform data!")
 self.raw_test = None
if test_file:
print("Loading test_data(%s)into dataframe" % test_file)
 self.test_data = pd.read_csv(test_file)
 self.raw_test = self.test_data[['question_text']]
print("Test data with shape: ", self.test_data.shape)
 print("Loading train_data(%s)into dataframes" % train_file)
 self.train_data = pd.read_csv(train_file)
 self.raw_train = self.train_data[['question_text']]
print("Train data with shape: ", self.train_data.shape)
 train_test_cut = self.train_data.shape[0]
if isinstance(self.raw_test, pd.DataFrame):
df_all = pd.concat([self.raw_train, self.raw_test],
axis=0 ).reset_index(drop=True)
 else:
df_all = self.raw_train
self.df_all = df_all
print("Transforming the data")
 with timer('Transforming data'):
if module:
X_features = module.transform(df_all['question_text'])
 else:
X_features = df_all['question_text']
if isinstance(X_features, list):
self.X_train = [X[:train_test_cut] for X in X_features]
self.X_test = [X[train_test_cut:] for X in X_features]
else:
self.X_train = X_features[:train_test_cut]
self.X_test = X_features[train_test_cut:]
def get_split(self, split=0.8, random_state=2, shuffle_data=True):

 print("Creating validation data by splitting(%s)" % split)
 train_data = self.train_data
X_train = self.X_train
X_t, X_v, y_t, y_v = train_test_split(
 X_train, train_data.target,
test_size=(1 - split), random_state=random_state,
shuffle=shuffle_data, stratify=train_data.target)
 return X_t, X_v, y_t, y_v
def get_kfold(self, k=5, shuffle_data=True, random_state=2):

 print("Creating validation data by kfold(%s)" % k)
 kfold = StratifiedKFold(n_splits=k, shuffle=shuffle_data, random_state=random_state)
 train_data = self.train_data
X_train = self.X_train
folded_data = kfold.split(X_train, train_data.target)
 for i in range(k):
fold = next(folded_data)
 X_t = X_train.iloc[fold[0]]
X_v = train_data.iloc[fold[0]]
y_t = X_train.iloc[fold[1]]
y_v = train_data.iloc[fold[1]]
yield X_t, X_v, y_t, y_v
def get_test(self):
if isinstance(self.test_data, pd.DataFrame):
return self.train_data, self.X_train, self.test_data, self.X_test
raise Exception("No test data provided!")
 def get_all_text(self):
return self.df_all['question_text']
test['target'] = preds_var
oof_var2 = np.zeros(len(train))
preds_var2 = np.zeros(len(test))
for k in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k]
test2p = test2[(test2['target']<=0.01)|(test2['target']>=0.99)].copy()
test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2p[cols])
 train3p = sel.transform(train2p[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3p, train2p['target']):
test_index3 = test_index[ test_index<len(train3)]
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_var2[idx1[test_index3]] += clf.predict_proba(train3[test_index3,:])[:,1]
preds_var2[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'],oof_var2)
 print('Pseudo Labeled QDA scores CV =',round(auc,5))<compute_test_metric>
class PreProcessor(object):
def __init__(self, text):
self.text = text
self.puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$',
'&', '/', '[', ']', '>', '%', '=', '
'~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<',
'→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â',
'█', '½', 'à', '…', '“', '★', '”', '–', '●', 'â', '►', '−', '¢',
'²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥',
'▓', '—', '‹', '─', '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’',
'▀', '¨', '▄', '♫', '☆', 'é', '¯',
'♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³',
'・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√']
self.mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling',
'counselling': 'counseling',
'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor',
'organisation': 'organization',
'wwii': 'world war 2',
'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary',
'Whta': 'What',
'narcisist': 'narcissist',
'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much',
'howmany': 'how many', 'whydo': 'why do',
'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does',
'mastrubation': 'masturbation',
'mastrubate': 'masturbate',
"mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum',
'narcissit': 'narcissist',
'bigdata': 'big data',
'2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend',
'airhostess': 'air hostess', "whst": 'what',
'watsapp': 'whatsapp', 'demonitisation': 'demonetization',
'demonitization': 'demonetization',
'demonetisation': 'demonetization'}
self.mispellings_re = re.compile('(%s)' % '|'.join(self.mispell_dict.keys()))
def get_text(self):
return self.text
def replace_typical_misspell(self):
def replace(match):
return self.mispell_dict[match.group(0)]
self.text = self.mispellings_re.sub(replace, self.text)
 return self
def spacy_tokenize_words(self):
raise NotImplementedError
def normalize_unicode(self):
self.text = unicodedata.normalize('NFKD', self.text)
 return self
def remove_newline(self):

 self.text = ' '.join(self.text.split())
 return self
def decontracted(self):
text = re.sub(r"(W|w)on('|\’)t", "will not", self.text)
 text = re.sub(r"(C|c)an('|\’)t", "can not", text)
 text = re.sub(r"(Y|y )('|\’)all", "you all", text)
 text = re.sub(r"(Y|y)a('|\’)ll", "you all", text)
 text = re.sub(r"(I|i )('|\’)m", "i am", text)
 text = re.sub(r"(A|a)in('|\’)t", "aint", text)
 text = re.sub(r"n('|\’)t", " not", text)
 text = re.sub(r"('|\’)re", " are", text)
 text = re.sub(r"('|\’)s", " is", text)
 text = re.sub(r"('|\’)d", " would", text)
 text = re.sub(r"('|\’)ll", " will", text)
 text = re.sub(r"('|\’)t", " not", text)
 self.text = re.sub(r"('|\’)ve", " have", text)
 return self
def space_punctuation(self):
for punct in self.puncts:
if punct in self.text:
self.text = self.text.replace(punct, f' {punct} ')
 return self
def remove_punctuation(self):
re_tok = re.compile(f'([{string.punctuation}])')
 self.text = re_tok.sub(' ', self.text)
 return self
def clean_numbers(self):
text = self.text
if bool(re.search(r'\d', text)) :
text = re.sub('[0-9]{5,}', '
text = re.sub('[0-9]{4}', '
text = re.sub('[0-9]{3}', '
text = re.sub('[0-9]{2}', '
self.text = text
return self
def clean_and_get_text(self):
self.clean_numbers() \ 
.space_punctuation() \ 
.decontracted() \ 
.normalize_unicode() \ 
.remove_newline() \ 
.replace_typical_misspell()
return self.text
auc = roc_auc_score(train['target'],0.5*(oof_var+ oof_var2))
print('Pseudo Labeled QDA scores CV =',round(auc,5))<save_to_csv>
class NeuralNetworkClassifier:

 def __init__(self, model, batch_size=512, epochs=10, val_score='val_loss',
reduce_lr=True, balancing_class_weight=False, filepath=None):

 self.model = model
self.batch_size = batch_size
self.epochs = epochs
self.val_score = val_score
self.reduce_lr = reduce_lr
self.balancing_class_weight = balancing_class_weight
self.filepath = filepath
self.model.compile(
 loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy', precision_score, recall_score,
f1_score, roc_auc_score])
 def _get_class_weight(self, y):
if self.balancing_class_weight:
return utils.class_weight.compute_class_weight(
 'balanced', np.unique(y), y)
 else:
return None
def _get_callbacks(self):
callbacks = []
callbacks.append(
 EarlyStopping(
 monitor=self.val_score,
patience=2,
verbose=1 
)
)
 if self.filepath:
callbacks.append(
 ModelCheckpoint(
 filepath=self.filepath,
monitor=self.val_score,
save_best_only=True,
save_weights_only=True 
)
)
 if self.reduce_lr:
callbacks.append(
 ReduceLROnPlateau(
 monitor=self.val_score,
factor=0.6,
patience=1,
min_lr=0.0001,
verbose=2 
)
)
 return callbacks
def predict(self, X):
return(self.predict_proba(X)> THRESHOLD ).astype(int)
 def predict_proba(self, X):
return self.model.predict([X], batch_size=1024, verbose=1)
 def train(self, X_train, y_train, X_val, y_val, verbose=1):

 callbacks = self._get_callbacks()
class_weight = self._get_class_weight(y_train)
 self.model.fit(
 X_train, y_train,
batch_size=self.batch_size,
epochs=self.epochs,
verbose=verbose,
callbacks=callbacks,
validation_data=(X_val, y_val),
shuffle=True,
class_weight=class_weight)
 return self
def fit(self, X, y, best_iteration=6, verbose=0):

 class_weight = self._get_class_weight(y)
 self.model.fit(
 X, y,
batch_size=self.batch_size,
epochs=best_iteration,
verbose=verbose,
shuffle=True,
class_weight=class_weight)
 if self.filepath:
self.model.save_weights(self.filepath)
 print('saved fitted model to {}'.format(self.filepath))
return self
@property
def best_param(self):
scores = self.model.history.history[self.val_score]
if 'loss' in self.val_score:
func = min
else:
func = max
best_iteration, _ = func(enumerate(scores), key=operator.itemgetter(1))
return best_iteration + 1
@property
def best_score(self):
scores = self.model.history.history[self.val_score]
if 'loss' in self.val_score:
func = min
else:
func = max
_, best_val_f1 = func(enumerate(scores), key=operator.itemgetter(1))
return best_val_f1

 def recall_score(y_true, y_proba, thres=THRESHOLD):

 y_pred = K.cast(K.greater(y_proba, thres), dtype='float32')
 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision_score(y_true, y_proba, thres=THRESHOLD):

 y_pred = K.cast(K.greater(y_proba, thres), dtype='float32')
 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
def f1_score(y_true, y_proba, thres=THRESHOLD):

 precision = precision_score(y_true, y_proba, thres)
 recall = recall_score(y_true, y_proba, thres)
 return 2 *(( precision * recall)/(precision + recall + K.epsilon()))
def roc_auc_score(y_true, y_proba):

 roc_auc = tf.metrics.auc(y_true, y_proba)[1]
K.get_session().run(tf.local_variables_initializer())
 return roc_auc

 class Attention(Layer):

 def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):

 def __init__(self, num_capsule, dim_capsule, routings=3,
kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule)
 class DropConnect(Wrapper):

 def __init__(self, layer, prob, **kwargs):
self.prob = prob
self.layer = layer
super(DropConnect, self ).__init__(layer, **kwargs)
 if 0.< self.prob < 1.:
self.uses_learning_phase = True
def build(self, input_shape):
if not self.layer.built:
self.layer.build(input_shape)
 self.layer.built = True
super(DropConnect, self ).build()
def compute_output_shape(self, input_shape):
return self.layer.compute_output_shape(input_shape)
 def call(self, x):
if 0.< self.prob < 1.:
self.layer.kernel = K.in_train_phase(
 K.dropout(self.layer.kernel, self.prob),
self.layer.kernel)
 self.layer.bias = K.in_train_phase(
 K.dropout(self.layer.bias, self.prob),
self.layer.bias)
 return self.layer.call(x,)
 def _dropout(x, level, noise_shape=None, seed=None):
x = K.dropout(x, level, noise_shape, seed)
 x *=(1.- level)
 return x
class QRNN(Layer):

 def __init__(self, units, window_size=2, stride=1,
return_sequences=False, go_backwards=False,
stateful=False, unroll=False, activation='tanh',
kernel_initializer='uniform', bias_initializer='zero',
kernel_regularizer=None, bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None, bias_constraint=None,
dropout=0, use_bias=True, input_dim=None, input_length=None,
**kwargs):
self.return_sequences = return_sequences
self.go_backwards = go_backwards
self.stateful = stateful
self.unroll = unroll
self.units = units
self.window_size = window_size
self.strides =(stride, 1)
 self.use_bias = use_bias
self.activation = activations.get(activation)
 self.kernel_initializer = initializers.get(kernel_initializer)
 self.bias_initializer = initializers.get(bias_initializer)
 self.kernel_regularizer = regularizers.get(kernel_regularizer)
 self.bias_regularizer = regularizers.get(bias_regularizer)
 self.activity_regularizer = regularizers.get(activity_regularizer)
 self.kernel_constraint = constraints.get(kernel_constraint)
 self.bias_constraint = constraints.get(bias_constraint)
 self.recurrent_dropout = 0
self.dropout = dropout
self.supports_masking = True
self.input_spec = [InputSpec(ndim=3)]
self.input_dim = input_dim
self.input_length = input_length
if self.input_dim:
kwargs['input_shape'] =(self.input_length, self.input_dim)
 super(QRNN, self ).__init__(**kwargs)
 def build(self, input_shape):
if isinstance(input_shape, list):
input_shape = input_shape[0]
batch_size = input_shape[0] if self.stateful else None
self.input_dim = input_shape[2]
self.input_spec = InputSpec(shape=(batch_size, None, self.input_dim))
self.state_spec = InputSpec(shape=(batch_size, self.units))
self.states = [None]
if self.stateful:
self.reset_states()
kernel_shape =(self.window_size, 1, self.input_dim, self.units * 3)
 self.kernel = self.add_weight(name='kernel',
shape=kernel_shape,
initializer=self.kernel_initializer,
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
 if self.use_bias:
self.bias = self.add_weight(name='bias',
shape=(self.units * 3,),
initializer=self.bias_initializer,
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
 self.built = True
def compute_output_shape(self, input_shape):
if isinstance(input_shape, list):
input_shape = input_shape[0]
length = input_shape[1]
if length:
length = conv_output_length(length + self.window_size - 1,
self.window_size, 'valid',
self.strides[0])
 if self.return_sequences:
return(input_shape[0], length, self.units)
 else:
return(input_shape[0], self.units)
 def compute_mask(self, inputs, mask):
if self.return_sequences:
return mask
else:
return None
def get_initial_states(self, inputs):
initial_state = K.zeros_like(inputs)
 initial_state = K.sum(initial_state, axis=(1, 2))
initial_state = K.expand_dims(initial_state)
 initial_state = K.tile(initial_state, [1, self.units])
 initial_states = [initial_state for _ in range(len(self.states)) ]
return initial_states
def reset_states(self, states=None):
if not self.stateful:
raise AttributeError('Layer must be stateful.')
 if not self.input_spec:
raise RuntimeError('Layer has never been called '
'and thus has no states.')
 batch_size = self.input_spec.shape[0]
if not batch_size:
raise ValueError('If a QRNN is stateful, it needs to know '
'its batch size.Specify the batch size '
'of your input tensors:
'
'- If using a Sequential model, '
'specify the batch size by passing '
'a `batch_input_shape` '
'argument to your first layer.
 '
'- If using the functional API, specify '
'the time dimension by passing a '
'`batch_shape` argument to your Input layer.')
 if self.states[0] is None:
self.states = [K.zeros(( batch_size, self.units))
for _ in self.states]
elif states is None:
for state in self.states:
K.set_value(state, np.zeros(( batch_size, self.units)))
 else:
if not isinstance(states,(list, tuple)) :
states = [states]
if len(states)!= len(self.states):
raise ValueError('Layer ' + self.name + ' expects ' +
str(len(self.states)) + ' states, '
'but it received ' + str(len(states)) +
'state values.Input received: ' +
str(states))
for index,(value, state)in enumerate(zip(states, self.states)) :
if value.shape !=(batch_size, self.units):
raise ValueError('State ' + str(index)+
' is incompatible with layer ' +
self.name + ': expected shape=' +
str(( batch_size, self.units)) +
', found shape=' + str(value.shape))
K.set_value(state, value)
 def __call__(self, inputs, initial_state=None, **kwargs):
if initial_state is not None:
if hasattr(initial_state, '_keras_history'):
input_spec = self.input_spec
state_spec = self.state_spec
if not isinstance(state_spec, list):
state_spec = [state_spec]
self.input_spec = [input_spec] + state_spec
if not isinstance(initial_state,(list, tuple)) :
initial_state = [initial_state]
inputs = [inputs] + list(initial_state)
 output = super(QRNN, self ).__call__(inputs, **kwargs)
 self.input_spec = input_spec
return output
else:
kwargs['initial_state'] = initial_state
return super(QRNN, self ).__call__(inputs, **kwargs)
 def call(self, inputs, mask=None, initial_state=None, training=None):
if isinstance(inputs, list):
initial_states = inputs[1:]
inputs = inputs[0]
elif initial_state is not None:
pass
elif self.stateful:
initial_states = self.states
else:
initial_states = self.get_initial_states(inputs)
 if len(initial_states)!= len(self.states):
raise ValueError('Layer has ' + str(len(self.states)) +
' states but was passed ' +
str(len(initial_states)) +
' initial states.')
 input_shape = K.int_shape(inputs)
 if self.unroll and input_shape[1] is None:
raise ValueError('Cannot unroll a RNN if the '
'time dimension is undefined.
 '
'- If using a Sequential model, '
'specify the time dimension by passing '
'an `input_shape` or `batch_input_shape` '
'argument to your first layer.If your '
'first layer is an Embedding, you can '
'also use the `input_length` argument.
 '
'- If using the functional API, specify '
'the time dimension by passing a `shape` '
'or `batch_shape` argument to your Input layer.')
 constants = self.get_constants(inputs, training=None)
 preprocessed_input = self.preprocess_input(inputs, training=None)
 last_output, outputs, states = K.rnn(self.step, preprocessed_input,
initial_states,
go_backwards=self.go_backwards,
mask=mask,
constants=constants,
unroll=self.unroll,
input_length=input_shape[1])
 if self.stateful:
updates = []
for i in range(len(states)) :
updates.append(( self.states[i], states[i]))
self.add_update(updates, inputs)
 if 0 < self.dropout < 1:
last_output._uses_learning_phase = True
outputs._uses_learning_phase = True
if self.return_sequences:
return outputs
else:
return last_output
def preprocess_input(self, inputs, training=None):
if self.window_size > 1:
inputs = K.temporal_padding(inputs,(self.window_size - 1, 0))
inputs = K.expand_dims(inputs, 2)
 output = K.conv2d(inputs, self.kernel, strides=self.strides,
padding='valid',
data_format='channels_last')
 output = K.squeeze(output, 2)
 if self.use_bias:
output = K.bias_add(output, self.bias, data_format='channels_last')
 if self.dropout is not None and 0.< self.dropout < 1.:
z = output[:, :, :self.units]
f = output[:, :, self.units:2 * self.units]
o = output[:, :, 2 * self.units:]
f = K.in_train_phase(1 - _dropout(1 - f, self.dropout), f, training=training)
 return K.concatenate([z, f, o], -1)
 else:
return output
def step(self, inputs, states):
prev_output = states[0]
z = inputs[:, :self.units]
f = inputs[:, self.units:2 * self.units]
o = inputs[:, 2 * self.units:]
z = self.activation(z)
 f = f if self.dropout is not None and 0.< self.dropout < 1.else K.sigmoid(f)
 o = K.sigmoid(o)
 output = f * prev_output +(1 - f)* z
output = o * output
return output, [output]
def get_constants(self, inputs, training=None):
return []
def get_config(self):
config = {'units': self.units,
'window_size': self.window_size,
'stride': self.strides[0],
'return_sequences': self.return_sequences,
'go_backwards': self.go_backwards,
'stateful': self.stateful,
'unroll': self.unroll,
'use_bias': self.use_bias,
'dropout': self.dropout,
'activation': activations.serialize(self.activation),
'kernel_initializer': initializers.serialize(self.kernel_initializer),
'bias_initializer': initializers.serialize(self.bias_initializer),
'kernel_regularizer': regularizers.serialize(self.kernel_regularizer),
'bias_regularizer': regularizers.serialize(self.bias_regularizer),
'activity_regularizer': regularizers.serialize(self.activity_regularizer),
'kernel_constraint': constraints.serialize(self.kernel_constraint),
'bias_constraint': constraints.serialize(self.bias_constraint),
'input_dim': self.input_dim,
'input_length': self.input_length}
base_config = super(QRNN, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = 0.5* preds_var + 0.5*preds_var2
sub.to_csv('submission.csv',index=False)
 plt.hist(preds,bins=100)
 plt.title('Final Test.csv predictions')
 plt.show()<create_dataframe>
def load_word_embedding(filepath):

 def _get_vec(word, *arr):
return word, np.asarray(arr, dtype='float32')
 print('Load word embeddings...')
 try:
word_embedding = dict(_get_vec(*w.split(' ')) for w in open(filepath))
except UnicodeDecodeError:
word_embedding = dict(_get_vec(*w.split(' ')) for w in open(
 filepath, encoding="utf8", errors='ignore'))
words_to_del = []
for word, vec in word_embedding.items() :
if len(vec)!= 300:
words_to_del.append(word)
 for word in words_to_del:
del word_embedding[word]
return word_embedding
def create_embedding_weights(word_index, word_embedding, max_features):

 print('Create word embeddings weights...')
 mat_embedding = np.stack(word_embedding.values())
 a, b = min(max_features, len(word_index)) , mat_embedding.shape[1]
print('Embeddings weights matrix with shape:({}, {})'.format(a, b))
embedding_mean, embedding_std = mat_embedding.mean() , mat_embedding.std()
embedding_weights = np.random.normal(embedding_mean, embedding_std,(a, b))
for word, idx in word_index.items() :
if idx >= a:
continue
word_vec = word_embedding.get(word, None)
 if word_vec is not None:
embedding_weights[idx] = word_vec
return embedding_weights
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 oof = np.zeros(len(train))
preds = np.zeros(len(test))
for i in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 data = pd.concat([pd.DataFrame(train2[cols]), pd.DataFrame(test2[cols])])
 sel = VarianceThreshold(threshold=1.5 ).fit(data[cols])
 data2 = sel.transform(data[cols])
 train3 = data2[:train2.shape[0]]; test3 = data2[train2.shape[0]:]
skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3, train2['target']):
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train3[train_index,:],train2.loc[train_index]['target'])
 oof[idx1[test_index]] = clf.predict_proba(train3[test_index,:])[:,1]
preds[idx2] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'],oof)
 print('QDA scores CV =',round(auc,5))<normalization>
MAX_FEATURES = int(2.5e5)
 MAX_LEN = 80
RNN_UNITS = 40
DENSE_UNITS_1 = 32
DENSE_UNITS_2 = 16
EMBED_FILEPATH = os.path.join(datapath, 'embeddings/glove.840B.300d/glove.840B.300d.txt')
 EMBED_PICKLEPATH = 'glove.pkl'
MODEL_FILEPATH = 'submission.csv'
cat_dict = dict()
cols = [c for c in train.columns if c not in ['id', 'target']]
cols.remove('wheezy-copper-turtle-magic')
 for i in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==i]
test2 = test[test['wheezy-copper-turtle-magic']==i]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 cat_dict[i] = train3.shape[1]<feature_engineering>
def get_network(embed_filepath):
input_layer = Input(shape=(MAX_LEN,), name='input')
 print('load pre-trained embeddings weights...')
 embed_weights = pd.read_pickle(EMBED_PICKLEPATH)
 input_dim = embed_weights.shape[0]
output_dim = embed_weights.shape[1]
x = Embedding(
 input_dim=input_dim,
output_dim=output_dim,
weights=[embed_weights],
trainable=False,
name='embeddings'
)(input_layer)
 del embed_weights, input_dim, output_dim
gc.collect()
x = SpatialDropout1D(rate=0.15 )(x)
 x = Bidirectional(
 layer=LSTM(RNN_UNITS, return_sequences=True),
name='bidirectional_lstm'
)(x)
 x = GlobalMaxPool1D(name='global_max_pooling1d' )(x)
 x = Dense(units=DENSE_UNITS_1, activation='relu', name='dense_1' )(x)
 x = Dense(units=DENSE_UNITS_2, activation='relu', name='dense_2' )(x)
 output_layer = Dense(units=1, activation='sigmoid', name='output' )(x)
 return Model(inputs=input_layer, outputs=output_layer)
 def get_model() :
print('build network...')
 model = get_network(embed_filepath=EMBED_FILEPATH)
 print(model.summary())
 return NeuralNetworkClassifier(
 model,
balancing_class_weight=True,
filepath=MODEL_FILEPATH)
 def tokenize(df_text):
def _clean(text):
return PreProcessor(text ).clean_and_get_text()
df_text = df_text.progress_apply(_clean)
 tokenizer = Tokenizer(
 num_words=MAX_FEATURES,
filters='',
lower=False,
split=' ')
 tokenizer.fit_on_texts(list(df_text))
sequences = tokenizer.texts_to_sequences(df_text)
 return sequences, tokenizer
def transform(df_text):
seqs, _ = tokenize(df_text)
 X = pad_sequences(seqs, maxlen=MAX_LEN, padding='pre', truncating='post')
 return X
test['target'] = preds
oof_var = np.zeros(len(train))
preds_var = np.zeros(len(test))
for k in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k]
test2p = test2[(test2['target']<=0.01)|(test2['target']>=0.99)].copy()
test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 pca = PCA(n_components=cat_dict[k], random_state= 1234)
 pca.fit(train2p[cols])
 train3p = pca.transform(train2p[cols])
 train3 = pca.transform(train2[cols])
 test3 = pca.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3p, train2p['target']):
test_index3 = test_index[ test_index<len(train3)]
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_var[idx1[test_index3]] += clf.predict_proba(train3[test_index3,:])[:,1]
preds_var[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'],oof_var)
 print('Pseudo Labeled QDA scores CV =',round(auc,5)) 
<feature_engineering>
class fakemodule(object):
@staticmethod
def transform(a):
return transform(a)
 dr = DataReader('%s/train.csv' % datapath, fakemodule, os.path.join(datapath, 'test.csv'))
test['target'] = preds_var
oof_var2 = np.zeros(len(train))
preds_var2 = np.zeros(len(test))
for k in range(512):
train2 = train[train['wheezy-copper-turtle-magic']==k]
train2p = train2.copy() ; idx1 = train2.index
test2 = test[test['wheezy-copper-turtle-magic']==k]
test2p = test2[(test2['target']<=0.01)|(test2['target']>=0.99)].copy()
test2p.loc[ test2p['target']>=0.5, 'target' ] = 1
test2p.loc[ test2p['target']<0.5, 'target' ] = 0
train2p = pd.concat([train2p,test2p],axis=0)
 train2p.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5 ).fit(train2p[cols])
 train3p = sel.transform(train2p[cols])
 train3 = sel.transform(train2[cols])
 test3 = sel.transform(test2[cols])
 skf = StratifiedKFold(n_splits=11, random_state=42, shuffle=True)
 for train_index, test_index in skf.split(train3p, train2p['target']):
test_index3 = test_index[ test_index<len(train3)]
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train3p[train_index,:],train2p.loc[train_index]['target'])
 oof_var2[idx1[test_index3]] += clf.predict_proba(train3[test_index3,:])[:,1]
preds_var2[test2.index] += clf.predict_proba(test3)[:,1] / skf.n_splits
auc = roc_auc_score(train['target'],oof_var2)
 print('Pseudo Labeled QDA scores CV =',round(auc,5))<compute_test_metric>
t0 = time.time()
with timer("Extract Word Index From Train and Test Data"):
print("Loading data...")
 all_text = dr.get_all_text()
print('Tokenizing text...')
 _, tokenizer = tokenize(all_text)
 word_index = tokenizer.word_index
with timer("Create Embedding Weights Matrix"):
print('Loading embeddings file')
 word_embed = load_word_embedding(EMBED_FILEPATH)
 print('Create embeddings weights...')
 embed_weights = create_embedding_weights(
 word_index,
word_embed,
MAX_FEATURES)
 filepath_to_save = '{}.pkl'.format('glove')
 pd.to_pickle(embed_weights, filepath_to_save)
 print('Save embeddings weights to {}'.format(filepath_to_save))
print('Entire program is done and it took {:.2f}s'.format(time.time() - t0))
auc = roc_auc_score(train['target'],0.5*(oof_var+ oof_var2))
print('Pseudo Labeled QDA scores CV =',round(auc,5))<save_to_csv>
def train_and_eval(X_train, y_train, X_val, y_val):

 model = get_model()
print('Training model...')
 model = model.train(X_train, y_train, X_val, y_val)
 best_param = model.best_param
best_score = model.best_score
print("Best param: {:.4f} with best score: {}".format(best_param, best_score))
return pd.DataFrame({'best_param': [best_param], 'best_score': [best_score]})
 t0 = time.time()
with timer("Load and Preprocess"):
X_t, X_v, y_t, y_v = dr.get_split(TEST_SIZE)
 with timer('Training and Tuning'):
filepath = os.path.join(datapath, 'trainer_baseline.csv')
 print('Save CV score file to {}'.format(filepath))
print('Entire program is done and it took {:.2f}s'.format(time.time() - t0))
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = 0.5* preds_var + 0.5*preds_var2
sub.to_csv('submission.csv',index=False)
 plt.hist(preds,bins=100)
 plt.title('Final Test.csv predictions')
 plt.show()<import_modules>
def create_submission(X_train, y_train, X_test, df_test):

 model = get_model()
print('Training model...')
 model = model.fit(X_train, y_train)
 print('Predicting test...')
 y_pred = np.squeeze(model.predict_proba(X_test)> 0.35 ).astype('int')
 return pd.DataFrame({'qid': df_test.qid, 'prediction': y_pred})
 t0 = time.time()
with timer("Load and Preprocess"):
df_train, X_train, df_test, X_test = dr.get_test()
with timer('Trainning and Creating Submission'):
filepath = 'submission.csv'
df_submission = create_submission(
 X_train, df_train.target,
X_test, df_test)
 df_submission.to_csv(filepath, index=False)
 print('Save submission file to {}'.format(filepath))
print('Entire program is done and it took {:.2f}s'.format(time.time() - t0))