Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<load_from_csv>

 pyLDAvis.enable_notebook()
warnings.filterwarnings('ignore' )
test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 if test_features.shape[0] == 3982:
print("Running for ONE SEED...")
 SEED = [0]
else:
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions 
<merge>
sample = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv')
 sample.head(3 )
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )<save_to_csv>
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv')
 train.head(3 )
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<import_modules>
test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv')
 test.head(3 )
%matplotlib inline 
<import_modules>
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]
PadIfNeeded,
HorizontalFlip,
VerticalFlip,
CenterCrop,
Crop,
Compose,
Transpose,
RandomRotate90,
ElasticTransform,
GridDistortion,
OpticalDistortion,
RandomSizedCrop,
OneOf,
CLAHE,
RandomBrightnessContrast,
RandomGamma,
RandomContrast,
InvertImg,
RandomContrast
)<feature_engineering>
stopwords=stopwords.words('english')
 train['que_stopwords']=train['question_body'].apply(lambda x : [x for x in x.split() if x in stopwords])
 train['ans_stopwords']=train['answer'].apply(lambda x: [x for x in x.split() if x in stopwords] )
os.listdir('/kaggle/input/severstal-steel-defect-detection')
 train_df = pd.read_csv('/kaggle/input/severstal-steel-defect-detection/train.csv')
 train_df['Image_id']= train_df[train_df.columns[0]].apply(lambda x : x.split('_')[0])
 train_df['class_id']= train_df[train_df.columns[0]].apply(lambda x : x.split('_')[1])
 train_df['mask']= ~train_df[train_df.columns[1]].isna()
print(train_df.shape)
 train_df.head(5 )<groupby>
def common_ngrams(col,common=10):
corpus=[]
for question in train[col].values:
words=[str(x[0]+' '+x[1])for x in ngrams(question.split() ,2)]
corpus.append(words)
 flatten=[x for one in corpus for x in one]
counter=Counter(flatten)
 most_common=counter.most_common(common)
 string,value=zip(*(most_common))
return string,value
mask_count=train_df.groupby(by='Image_id' ).agg(np.sum ).reset_index()
print(mask_count.shape)
 mask_count.head(3 )<sort_values>
np.set_printoptions(suppress=True )
mask_count.sort_values(by='mask',ascending=False,inplace=True)
 mask_count.head()<feature_engineering>
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = FullTokenizer(BERT_PATH+'/assets/vocab.txt', True)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )
sub_df = pd.read_csv('/kaggle/input/severstal-steel-defect-detection/sample_submission.csv')
 sub_df['Image_id']=sub_df[sub_df.columns[0]].apply(lambda x : x.split('_')[0])
 test_imgs = pd.DataFrame(sub_df['Image_id'].unique() ,columns=['Image_id'])
 print(test_imgs.shape)
 test_imgs.head()<define_variables>
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
train_path = '/kaggle/input/severstal-steel-defect-detection/train_images/'
train_dir=os.listdir(train_path)
 train_dir[:3]
test_path = '/kaggle/input/severstal-steel-defect-detection/test_images/'
test_dir = os.listdir(test_path )<categorify>
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.nanmean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback
def mask2rle(img):
pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle2mask(rle, input_shape):
width, height = input_shape[:2]
mask= np.zeros(width*height ).astype(np.uint8)
 array = np.asarray([int(x)for x in rle.split() ])
 starts = array[0::2]
lengths = array[1::2]
current_position = 0
for index, start in enumerate(starts):
mask[int(start):int(start+lengths[index])] = 1
current_position += lengths[index]
return mask.reshape(height, width ).T
def build_masks(rles, input_shape):
depth = len(rles)
 masks = np.zeros(( *input_shape, depth))
for i, rle in enumerate(rles):
if type(rle)is str:
masks[:, :, i] = rle2mask(rle, input_shape)
 return masks
def build_rles(masks):
width, height, depth = masks.shape
rles = [mask2rle(masks[:, :, i])
 for i in range(depth)]
return rles 
<choose_model_class>
gkf = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body)
 test_inputs = compute_input_arays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )
pa='/kaggle/input/efficientnet'
sys.path.append(os.path.abspath('/kaggle/input/efficientnet-models/efficientnet-master/'))
N_CLASSES=4
def ClsModel(n_classes=1, input_shape=(254,600,3)) :
base_model = EfficientNetB2(weights=None, include_top=False, input_shape=input_shape)
 x = AveragePooling2D(pool_size=(5,5), name='avg_pool' )(base_model.output)
 x = Flatten()(x)
 x = Dense(1600, activation='relu', name='dense_post_pool' )(x)
 x = Dropout(0.5 )(x)
 output = Dense(n_classes, activation='sigmoid', name='predictions' )(x)
 model = Model(inputs=base_model.input, output=output)
 return model
model = ClsModel(n_classes=N_CLASSES)
 N_CLASSES=4
def ClsModel(n_classes=1, input_shape=(254,254,3)) :
base_model = EfficientNetB3(weights=None, include_top=False, input_shape=input_shape)
 x = AveragePooling2D(pool_size=(3,3), name='avg_pool' )(base_model.output)
 x = Flatten()(x)
 x = Dense(1024, activation='relu', name='dense_post_pool' )(x)
 x = Dropout(0.5 )(x)
 output = Dense(n_classes, activation='sigmoid', name='predictions' )(x)
 model = Model(inputs=base_model.input, output=output)
 return model
model_class = ClsModel(n_classes=N_CLASSES)
<train_model>
models = []
for i in range(5):
model_path = f'.. /input/bertuned-f{i}/bertuned_f{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model)
class DataGenerator1(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path=train_path,val ='no',augmentaion =[1],
batch_size=1, dim=(254, 600), n_channels=3,
n_classes=4, random_state=1994, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.val=val
self.augmentaion=augmentaion
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[self.batch_size*index:self.batch_size*(index+1)]
ID = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(ID)
 if self.mode == 'fit':
y = self.__generate_y(ID)
 return X,y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __generate_X(self, ID):
'Generates data containing batch_size samples'
img_i = np.empty(( self.batch_size,*self.dim,self.n_channels),dtype=np.uint8)
 for i,loc in enumerate(ID):
im_name = self.df['Image_id'].loc[loc]
img_path = f"{self.base_path}/{im_name}"
img=cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img=cv2.resize(img ,(600,254))
if self.augmentaion[0]==1 :
img_i[i,] = img
continue;
aug_no =0
aug = self.augmentaion[aug_no](p=1)
 img = aug(image=img)['image']
img_i[i,] = img
return img_i
def __generate_y(self, ID):
y = np.empty(( self.batch_size,self.n_classes))
for i,loc in enumerate(ID):
y[i,] = self.df['mask'].loc[loc]
return y
def __load_rgb(self, img_path):
return cv2.imread(img_path)
<train_model>
model_path = f'.. /input/bertf1e15/Full-0.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
class DataGenerator2(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path=train_path,val ='no',augmentaion =[1],
batch_size=1, dim=(254, 254), n_channels=3,
n_classes=4, random_state=1994, shuffle=True):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.val=val
self.augmentaion=augmentaion
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[self.batch_size*index:self.batch_size*(index+1)]
ID = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(ID)
 if self.mode == 'fit':
y = self.__generate_y(ID)
 return X,y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
 def __generate_X(self, ID):
'Generates data containing batch_size samples'
img_i = np.empty(( self.batch_size,*self.dim,self.n_channels),dtype=np.uint8)
 for i,loc in enumerate(ID):
im_name = self.df['Image_id'].loc[loc]
img_path = f"{self.base_path}/{im_name}"
img=cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img=cv2.resize(img ,(254,254))
if self.augmentaion[0]==1 :
img_i[i,] = img
continue;
aug_no =0
aug = self.augmentaion[aug_no](p=1)
 img = aug(image=img)['image']
img_i[i,] = img
return img_i
def __generate_y(self, ID):
y = np.empty(( self.batch_size,self.n_classes))
for i,loc in enumerate(ID):
y[i,] = self.df['mask'].loc[loc]
return y
def __load_rgb(self, img_path):
return cv2.imread(img_path)
<load_pretrained>
model_path = f'.. /input/bertmodelpretrained/bert-0.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
model_class_2 = load_model('/kaggle/input/efficent-model-and-weights-975/efficent_class_model.h5')
<load_pretrained>
model_path = f'.. /input/bertmodelpretrained/bert-1.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
all_classifer_path=['/kaggle/input/efficent-b2-600/EfficientNetB3_efficenent_254_600_exp_4.h5',
'/kaggle/input/efficient977/EfficientNetB3_efficenent_exp_2.h5']
class_models =[model,model_class]
pred_class_i=[]
pred=[]
DataGenerator=[DataGenerator1,DataGenerator2]
for i in range(2):
augmentaion=[1,HorizontalFlip,VerticalFlip]
for aug in augmentaion:
class_models[i].load_weights(all_classifer_path[i])
 test_generator = DataGenerator[i](
 test_imgs.index,
df=test_imgs,
shuffle=False,
mode='predict',
base_path='.. /input/severstal-steel-defect-detection/test_images',
target_df=test_imgs,
augmentaion=[aug],
batch_size=1,
n_classes=4 
)
 batch_pred_class = class_models[i].predict_generator(
 test_generator,
workers=1,
verbose=1,
use_multiprocessing=False 
)
 pred.append(batch_pred_class.round().astype(int)) 
<create_dataframe>
for i in range(7):
model_path = f'.. /input/pretrained-bert/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
test=pred.copy()<feature_engineering>
test_predictions = []
test_1=np.array(test)
 test_1=test_1[0]+test_1[1]+test_1[2]+test_1[3]+test_1[4]+test_1[5]
test_1[np.where(test_1<=2)]=0
test_1[np.where(test_1>2)]=1
np.sum(test_1,axis=0 )<data_type_conversions>
for model in models:
test_predictions.append(model.predict(test_inputs, batch_size=8))
np.sum(test_1.round().astype(int),axis=0 )<define_variables>
final_predictions = np.mean(test_predictions, axis=0 )
<count_values><EOS> 
df_sub.iloc[:, 1:] = final_predictions
df_sub.to_csv('submission.csv', index=False )