Kaggle competition: Quora Insincere Questions Classification
np.zeros(( count_data_items(test_filenames),1))<load_from_csv>
dim = 300
num_words = 50000
max_len = 100
print('Fiting tokenizer')
 tokenizer = Tokenizer(num_words=num_words)
 tokenizer.fit_on_texts(df['question_text'])
 print('spliting data')
 df_train,df_test = train_test_split(df, random_state=1)
 print('text to sequence')
 x_train = tokenizer.texts_to_sequences(df_train['question_text'])
 x_test = tokenizer.texts_to_sequences(df_test['question_text'])
 print('pad sequence')
 x_train = pad_sequences(x_train,maxlen=max_len)
 x_test = pad_sequences(x_test, maxlen=max_len)
 y_train = df_train['target'].values
y_test = df_test['target'].values
print(x_train.shape)
 print(y_train.shape)
train = pd.read_csv('.. /input/siim-isic-melanoma-classification/train.csv')
 print('Examples WITH Melanoma')
 imgs = train.loc[train.target==1].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()
print('Examples WITHOUT Melanoma')
 imgs = train.loc[train.target==0].sample(10 ).image_name.values
plt.figure(figsize=(20,8))
for i,k in enumerate(imgs):
img = cv2.imread('.. /input/jpeg-melanoma-128x128/train/%s.jpg'%k)
 img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
 plt.subplot(2,5,i+1); plt.axis('off')
 plt.imshow(img)
 plt.show()<install_modules>
print('loading word2vec model...')
 word2vec = gensim.models.KeyedVectors.load_word2vec_format('.. /input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin', binary=True)
 print('vocab:',len(word2vec.vocab))
all_embs = word2vec.vectors
emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(emb_mean,emb_std)
 print(num_words,' from ',len(tokenizer.word_index.items()))
embedding_matrix = np.random.normal(emb_mean, emb_std,(num_words, dim))
count = 0
for word, i in tokenizer.word_index.items() :
if i>=num_words:
break
if word in word2vec.vocab:
embedding_matrix[i] = word2vec.word_vec(word)
 else:
count += 1
print('embedding matrix size:',embedding_matrix.shape)
 print('Number of words not in vocab:',count)
 del word2vec
gc.collect()
!pip install -q efficientnet >> /dev/null<import_modules>
print('Glove...')
 def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_glov = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_glov[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
import pandas as pd, numpy as np
from kaggle_datasets import KaggleDatasets
import tensorflow as tf, re, math
import tensorflow.keras.backend as K
import efficientnet.tfkeras as efn
from sklearn.model_selection import KFold
from sklearn.metrics import roc_auc_score
import matplotlib.pyplot as plt<define_variables>
print('Para...')
 EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_para = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_para[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
DEVICE = "TPU"
SEED = 42
FOLDS = 5
IMG_SIZES = [384,384,384,384,384]
INC2019 = [0]*FOLDS
INC2018 = [1]*FOLDS
BATCH_SIZES = [32]*FOLDS
EPOCHS = [12]*FOLDS
EFF_NETS = [6,6,6,6,6]
WGTS = [1/FOLDS]*FOLDS
TTA = 20<choose_model_class>
print('Wiki...')
 EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
word_index = tokenizer.word_index
embedding_matrix_wiki = np.random.normal(emb_mean, emb_std,(num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_wiki[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_wiki.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<feature_engineering>
matrixes = [embedding_matrix,embedding_matrix_glov,embedding_matrix_wiki,embedding_matrix_para]
matrix = np.mean(matrixes,axis=0)
 del embedding_matrix,embedding_matrix_glov,embedding_matrix_wiki,embedding_matrix_para
gc.collect()
GCS_PATH = [None]*FOLDS; GCS_PATH2 = [None]*FOLDS
for i,k in enumerate(IMG_SIZES):
GCS_PATH[i] = KaggleDatasets().get_gcs_path('melanoma-%ix%i'%(k,k))
GCS_PATH2[i] = KaggleDatasets().get_gcs_path('isic2019-%ix%i'%(k,k))
files_train = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/train*.tfrec')))
 files_test = np.sort(np.array(tf.io.gfile.glob(GCS_PATH[0] + '/test*.tfrec')) )<define_variables>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
def f1(y_true, y_pred):

 def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 adam = optimizers.Adam()
print('LR:',K.eval(adam.lr))
batch_size = int(x_train.shape[0]*K.eval(adam.lr))
print('Batch size = ',batch_size)
 inp1 = Input(shape=(max_len,))
x = Embedding(num_words, dim, weights=[matrix],trainable = False, )(inp1)
 x = SpatialDropout1D(0.1 )(x)
 x = Bidirectional(CuDNNLSTM(50, return_sequences=True))(x)
 y = Bidirectional(CuDNNGRU(50, return_sequences=True))(x)
 atten_1 = Attention(max_len )(x)
 atten_2 = Attention(max_len )(y)
 avg_pool = GlobalAveragePooling1D()(y)
 max_pool = GlobalMaxPooling1D()(y)
 x = concatenate([atten_1, atten_2, avg_pool, max_pool])
 x = Dense(100, activation="relu",kernel_regularizer=regularizers.l2(0.0001))(x)
 x = Dropout(0.2 )(x)
 x = Dense(1, activation='sigmoid',kernel_regularizer=regularizers.l2(0.0001))(x)
 model = Model(inputs=inp1, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy',f1])
 print(model.summary())
 patience = 10
history = model.fit(x_train,y_train,
batch_size=batch_size,
validation_split=0.33,
epochs=100,
callbacks=[EarlyStopping(patience=patience,monitor='f1'),])
 print('training finished...' )
ROT_ = 180.0
SHR_ = 2.0
HZOOM_ = 8.0
WZOOM_ = 8.0
HSHIFT_ = 8.0
WSHIFT_ = 8.0<normalization>
y_pred = model.predict(x_train,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_train, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON TRAINING SET:
',classification_report(y_train,y_pred))
y_pred = model.predict(x_test,batch_size=batch_size, verbose=1)
 search_result = threshold_search(y_test, y_pred)
 print(search_result)
 y_pred = y_pred>search_result['threshold']
y_pred = y_pred.astype(int)
 print('RESULTS ON TEST SET:
',classification_report(y_test,y_pred))
def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):
rotation = math.pi * rotation / 180.
 shear = math.pi * shear / 180.
 def get_3x3_mat(lst):
return tf.reshape(tf.concat([lst],axis=0), [3,3])
 c1 = tf.math.cos(rotation)
 s1 = tf.math.sin(rotation)
 one = tf.constant([1],dtype='float32')
 zero = tf.constant([0],dtype='float32')
 rotation_matrix = get_3x3_mat([c1, s1, zero,
-s1, c1, zero,
zero, zero, one])
 c2 = tf.math.cos(shear)
 s2 = tf.math.sin(shear)
 shear_matrix = get_3x3_mat([one, s2, zero,
zero, c2, zero,
zero, zero, one])
 zoom_matrix = get_3x3_mat([one/height_zoom, zero, zero,
zero, one/width_zoom, zero,
zero, zero, one])
 shift_matrix = get_3x3_mat([one, zero, height_shift,
zero, one, width_shift,
zero, zero, one])
 return K.dot(K.dot(rotation_matrix, shear_matrix),
K.dot(zoom_matrix, shift_matrix))
def transform(image, DIM=256):
XDIM = DIM%2
rot = ROT_ * tf.random.normal([1], dtype='float32')
 shr = SHR_ * tf.random.normal([1], dtype='float32')
 h_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ HZOOM_
w_zoom = 1.0 + tf.random.normal([1], dtype='float32')/ WZOOM_
h_shift = HSHIFT_ * tf.random.normal([1], dtype='float32')
 w_shift = WSHIFT_ * tf.random.normal([1], dtype='float32')
 m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift)
 x = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)
 y = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])
 z = tf.ones([DIM*DIM], dtype='int32')
 idx = tf.stack([x,y,z])
 idx2 = K.dot(m, tf.cast(idx, dtype='float32'))
idx2 = K.cast(idx2, dtype='int32')
 idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)
 idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])
 d = tf.gather_nd(image, tf.transpose(idx3))
return tf.reshape(d,[DIM, DIM,3] )<prepare_x_and_y>
print('fiting final model...')
 n_epochs = len(history.history['loss'])- patience
history = model.fit(x_train,y_train, batch_size=batch_size, epochs=n_epochs)
 print('fitting on full data done...' )
<create_dataframe><EOS> 
print('Loading test data...')
 df_final = pd.read_csv('.. /input/test.csv')
 df_final["question_text"].fillna("_
x_final=tokenizer.texts_to_sequences(df_final['question_text'])
 x_final = pad_sequences(x_final,maxlen=max_len)
 y_pred = model.predict(x_final,batch_size=batch_size,verbose=1)
 y_pred = y_pred > search_result['threshold']
y_pred = y_pred.astype(int)
 print(y_pred[:5])
 df_subm = pd.DataFrame()
df_subm['qid'] = df_final.qid
df_subm['prediction'] = y_pred
print(df_subm.head())
 df_subm.to_csv('submission.csv', index=False )