Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<import_modules>
!pip install.. /input/huggingface-transformers/sacremoses-master/sacremoses-master
!pip install.. /input/huggingface-transformers/transformers-master/transformers-master
')
 ')
 ')
<install_modules>
tqdm.pandas()
warnings.filterwarnings('ignore')
!pip install.. /input/cassava-models/Keras_Applications-1.0.8-py3-none-any.whl
!pip install.. /input/cassava-models/efficientnet-1.1.0-py3-none-any.whl 
<define_variables>
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer_google_qa = tokenization.FullTokenizer(BERT_PATH+'/assets/vocab.txt', True)
 tokenizer2 = BertTokenizer.from_pretrained(BERT_PATH+'/assets/vocab.txt', do_lower_case=True,)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv',header=0,encoding='utf-8')
 df_test = pd.read_csv(PATH+'test.csv',header=0,encoding='utf-8')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 df = pd.concat([df_train,df_test],axis=0,ignore_index=True)
 print(f)
 output_categories = ['question_asker_intent_understanding', 'question_body_critical', 'question_conversational', 'question_expect_short_answer', 'question_fact_seeking', 'question_has_commonly_accepted_answer', 'question_interestingness_others', 'question_interestingness_self', 'question_multi_intent', 'question_not_really_a_question', 'question_opinion_seeking', 'question_type_choice', 'question_type_compare', 'question_type_consequence', 'question_type_definition', 'question_type_entity', 'question_type_instructions', 'question_type_procedure', 'question_type_reason_explanation', 'question_type_spelling', 'question_well_written', 'answer_helpful', 'answer_level_of_information', 'answer_plausible', 'answer_relevance', 'answer_satisfaction', 'answer_type_instructions', 'answer_type_procedure', 'answer_type_reason_explanation', 'answer_well_written']
input_categories = ['question_title', 'question_body', 'answer']
print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 20
BATCH_SIZE = 32 * strategy.num_replicas_in_sync
IMAGE_SIZE = [512, 512]
SEED = 123
LR = 0.0001
TTA = 10
VERBOSE = 2
N_CLASSES = 5
TEST_FILENAMES = '.. /input/cassava-leaf-disease-classification/test_images/*.jpg'<define_search_space>
tree_tokenizer = TreebankWordTokenizer()
def get_tree_tokens(x):
x = tree_tokenizer.tokenize(x)
 x = ' '.join(x)
 return x
def data_augment(image, image_name):
p_spatial = tf.random.uniform([], 0, 1.0, dtype = tf.float32)
 p_rotate = tf.random.uniform([], 0, 1.0, dtype = tf.float32)
 p_pixel_1 = tf.random.uniform([], 0, 1.0, dtype = tf.float32)
 p_pixel_2 = tf.random.uniform([], 0, 1.0, dtype = tf.float32)
 p_pixel_3 = tf.random.uniform([], 0, 1.0, dtype = tf.float32)
 p_crop = tf.random.uniform([], 0, 1.0, dtype = tf.float32)
 image = tf.image.random_flip_left_right(image)
 image = tf.image.random_flip_up_down(image)
 if p_spatial > 0.75:
image = tf.image.transpose(image)
 if p_rotate > 0.75:
image = tf.image.rot90(image, k = 3)
 elif p_rotate > 0.5:
image = tf.image.rot90(image, k = 2)
 elif p_rotate > 0.25:
image = tf.image.rot90(image, k = 1)
 if p_pixel_1 >= 0.4:
image = tf.image.random_saturation(image, lower = 0.7, upper = 1.3)
 if p_pixel_2 >= 0.4:
image = tf.image.random_contrast(image, lower = 0.8, upper = 1.2)
 if p_pixel_3 >= 0.4:
image = tf.image.random_brightness(image, max_delta = 0.1)
 if p_crop > 0.7:
if p_crop > 0.9:
image = tf.image.central_crop(image, central_fraction = 0.7)
 elif p_crop > 0.8:
image = tf.image.central_crop(image, central_fraction = 0.8)
 else:
image = tf.image.central_crop(image, central_fraction = 0.9)
 elif p_crop > 0.4:
crop_size = tf.random.uniform([], int(IMAGE_SIZE[0] * 0.8), IMAGE_SIZE[0], dtype = tf.int32)
 image = tf.image.random_crop(image, size = [crop_size, crop_size, 3])
 image = tf.image.resize(image, size = IMAGE_SIZE)
 image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image, image_name
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels = 3)
 image = tf.image.resize(image, IMAGE_SIZE)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [*IMAGE_SIZE, 3])
 return image
def get_image_name(file_path):
parts = tf.strings.split(file_path, os.path.sep)
 image_name = parts[-1]
return image_name
def read_image(file_path):
image_name = get_image_name(file_path)
 image = tf.io.read_file(file_path)
 image = decode_image(image)
 return image, image_name
def get_test_dataset(filenames, tta = False):
dataset = tf.data.Dataset.list_files(filenames, shuffle = False)
 dataset = dataset.map(read_image, num_parallel_calls = AUTO)
 dataset = dataset.map(data_augment, num_parallel_calls = AUTO)
 if tta:
dataset = dataset.repeat()
dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
NUM_TESTING_IMAGES = len(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))<choose_model_class>
for col in input_categories:
df_train[f'treated_{col}'] = df_train[col].progress_apply(lambda x: get_tree_tokens(x))
df_test[f'treated_{col}'] = df_test[col].progress_apply(lambda x: get_tree_tokens(x))
df[f'treated_{col}'] = df[col].progress_apply(lambda x: get_tree_tokens(x))
def get_model() :
with strategy.scope() :
inp = tf.keras.layers.Input(shape =(*IMAGE_SIZE, 3))
x = efn.EfficientNetB5(weights = None, include_top = False )(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 output = tf.keras.layers.Dense(N_CLASSES, activation = 'softmax' )(x)
 model = tf.keras.models.Model(inputs = [inp], outputs = [output])
 opt = tf.keras.optimizers.Adam(learning_rate = LR)
 model.compile(
 optimizer = opt,
loss = [tf.keras.losses.CategoricalCrossentropy(label_smoothing = 0.4)],
metrics = [tf.keras.metrics.CategoricalAccuracy() ] 
)
 return model
def inference(model_paths):
prediction = np.zeros(( NUM_TESTING_IMAGES, N_CLASSES))
print('Extracting test image names...')
 test_dataset = get_test_dataset(TEST_FILENAMES, tta = False)
 image_name = test_dataset.map(lambda image, image_name: image_name ).unbatch()
image_name = next(iter(image_name.batch(NUM_TESTING_IMAGES)) ).numpy().astype('U')
 print('Test image names completed...')
 for fold, model_path in enumerate(model_paths):
print('
')
 print('-'*50)
 print(f'Predicting fold {fold + 1}')
 K.clear_session()
model = get_model()
model.load_weights(model_path)
 steps = TTA *(( NUM_TESTING_IMAGES / BATCH_SIZE)+ 1)
 test_dataset = get_test_dataset(TEST_FILENAMES, tta = True)
 image = test_dataset.map(lambda image, image_name: image)
 probabilities = model.predict(image, steps = steps)[: TTA * NUM_TESTING_IMAGES]
probabilities = np.mean(probabilities.reshape(( NUM_TESTING_IMAGES, TTA, N_CLASSES), order = 'F'), axis = 1)
 prediction += probabilities / len(model_paths)
 sub = pd.DataFrame({'image_id': image_name, 'label': np.argmax(prediction, axis = -1)})
 sub.to_csv('submission.csv', index = False)
 return image_name, prediction, sub
model_paths = glob.glob('.. /input/cassava-models/*.h5')
 image_name, prediction, sub = inference(model_paths)
 sub<define_variables>
tokenizer = text.Tokenizer(lower=False )
package_path = '.. /input/pytorch-image-models/pytorch-image-models-master' 
<import_modules>
X_train_question = df_train['question_body']
X_train_title = df_train['question_title']
X_train_answer = df_train['answer']
X_test_question = df_test['question_body']
X_test_title = df_test['question_title']
X_test_answer = df_test['answer']
from glob import glob
from sklearn.model_selection import GroupKFold, StratifiedKFold
import cv2
from skimage import io
import torch
from torch import nn
import os
from datetime import datetime
import time
import random
import cv2
import torchvision
from torchvision import transforms
import pandas as pd
import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt
from torch.utils.data import Dataset,DataLoader
from torch.utils.data.sampler import SequentialSampler, RandomSampler
from torch.cuda.amp import autocast, GradScaler
import sklearn
import warnings
import joblib
from sklearn.metrics import roc_auc_score, log_loss
from sklearn import metrics
import warnings
import cv2
import pydicom
import timm
from scipy.ndimage.interpolation import zoom
from sklearn.metrics import log_loss<init_hyperparams>
tokenizer.fit_on_texts(list(X_train_title)+list(X_train_question)+list(X_train_answer)+list(X_test_title)+list(X_test_question)+list(X_test_answer))
CFG = {
'fold_num': 5,
'seed': 719,
'model_arch': 'vit_base_patch16_384',
'img_size': 384,
'epochs': 10,
'train_bs': 32,
'valid_bs': 32,
'lr': 1e-4,
'num_workers': 4,
'accum_iter': 1,
'verbose_step': 1,
'device': 'cuda:0',
'used_folds':[0,2,3],
'used_epochs': [7,8,9],
'tta': 3
}<load_from_csv>
nlp = English()
sentencizer = nlp.create_pipe('sentencizer')
 nlp.add_pipe(sentencizer )
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv')
 train.head()<count_values>
def split_document(texts):
all_sents = []
max_num_sentences = 0.0
for text in texts:
doc = nlp(text)
 sents=[]
for i,sent in enumerate(doc.sents):
sents.append(sent.text)
 all_sents.append(sents)
 return all_sents
X_train_question = split_document(X_train_question)
 X_train_answer = split_document(X_train_answer)
 X_test_question = split_document(X_test_question)
 X_test_answer = split_document(X_test_answer)
train.label.value_counts()<load_from_csv>
def add_question_metadata_features(text):
doc=nlp(text)
 indirect = 0
choice_words=0
reason_explanation_words = 0
question_count = 0
for sent in doc.sents:
if '?' in sent.text and '?' == sent.text[-1]:
question_count += 1
for token in sent:
if token.text.lower() =='why':
reason_explanation_words+=1
elif token.text.lower() =='or':
choice_words+=1
if question_count==0:
indirect+=1
return np.array([indirect, question_count, reason_explanation_words, choice_words] )
submission = pd.read_csv('.. /input/cassava-leaf-disease-classification/sample_submission.csv')
 submission.head()<categorify>
ans_user_and_category=df_train[df_train[['answer_user_name', 'category']].duplicated() ][['answer_user_name', 'category']].values
ans_user_and_category.shape
class CassavaDataset(Dataset):
def __init__(
 self, df, data_root, transforms=None, output_label=True 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df.iloc[index]['label']
path = "{}/{}".format(self.data_root, self.df.iloc[index]['image_id'])
 img = get_img(path)
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
return img, target
else:
return img<import_modules>
def question_answer_author_same(df):
q_username = df['question_user_name']
a_username = df['answer_user_name']
author_same=[]
for i in range(len(df)) :
if q_username[i] == a_username[i]:
author_same.append(int(1))
else:
author_same.append(int(0))
return author_same
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomResizedCrop,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize 
)
 def get_inference_transforms() :
return Compose([
RandomResizedCrop(CFG['img_size'], CFG['img_size']),
Transpose(p=0.5),
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(p=0.5),
HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),
RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),
Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),
ToTensorV2(p=1.0),
], p=1.)<choose_model_class>
def add_external_features(df):
df['question_body'] = df['question_body'].progress_apply(lambda x: str(x))
df['question_body_num_words'] = df['question_body'].str.count('\S+')
 df['answer'] = df['answer'].progress_apply(lambda x: str(x))
df['answer_num_words'] = df['answer'].str.count('\S+')
 df['question_vs_answer_length'] = df['question_body_num_words']/df['answer_num_words']
df['q_a_author_same'] = question_answer_author_same(df)
 answer_user_cat = []
for i in tqdm(df[['answer_user_name', 'category']].values):
if i in ans_user_and_category:
answer_user_cat.append(int(1))
else:
answer_user_cat.append(int(0))
df['answer_user_cat'] = answer_user_cat
handmade_features=[]
for text in df['question_body'].values:
handmade_features.append(add_question_metadata_features(text))
return df, np.array(handmade_features)
class ViTBase16(nn.Module):
def __init__(self, n_classes, pretrained=False):
super(ViTBase16, self ).__init__()
self.model = timm.create_model(CFG['model_arch'], pretrained=pretrained)
 self.model.head = nn.Linear(self.model.head.in_features, n_classes)
 def forward(self, x):
x = self.model(x)
 return x<create_dataframe>
df_train, handmade_features = add_external_features(df_train)
 df_test, handmade_features_test = add_external_features(df_test)
 df_train = pd.concat([df_train,pd.DataFrame(handmade_features, columns=['indirect', 'question_count', 'reason_explanation_words', 'choice_words'])],axis=1)
 df_test = pd.concat([df_test,pd.DataFrame(handmade_features_test, columns=['indirect', 'question_count', 'reason_explanation_words', 'choice_words'])],axis=1 )
if __name__ == '__main__':
tst_preds_avg = []
seed_everything(CFG['seed'])
 for fold in CFG['used_folds']:
print('Inference fold {} started'.format(fold))
test = pd.DataFrame()
test['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_ds = CassavaDataset(test, '.. /input/cassava-leaf-disease-classification/test_images/', transforms=get_inference_transforms() , output_label=False)
 tst_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=CFG['valid_bs'],
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False, 
)
 device = torch.device(CFG['device'])
 model = ViTBase16(n_classes=5 ).to(device)
 tst_preds = []
for i, epoch in enumerate(CFG['used_epochs']):
model.load_state_dict(torch.load('.. /input/vit384-pytorch-weights-exp-1/{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))
 with torch.no_grad() :
for _ in range(CFG['tta']):
tst_preds += [1/CFG['tta']*inference_one_epoch(model, tst_loader, device)]
tst_preds_avg.append(np.mean(tst_preds, axis=0))
del model
torch.cuda.empty_cache()<feature_engineering>
num_words_scaler = MinMaxScaler()
df_train[['question_body_num_words', 'answer_num_words']] = num_words_scaler.fit_transform(df_train[['question_body_num_words', 'answer_num_words']].values)
 df_test[['question_body_num_words', 'answer_num_words']] = num_words_scaler.transform(df_test[['question_body_num_words', 'answer_num_words']].values )
test['label'] = np.argmax(np.mean(tst_preds_avg, axis=0), axis=-1)
 test.head()<save_to_csv>
df=pd.concat([df,pd.get_dummies(df['host'], drop_first=False, prefix='host')],axis=1)
 df=pd.concat([df,pd.get_dummies(df['category'], drop_first=False, prefix='cat')],axis=1 )
test.to_csv('submission.csv', index=False )<define_variables>
len(['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')] )
package_paths = ['.. /input/imgclsmob/imgclsmob-master',
'.. /input/efficientnetpytorch/EfficientNet-PyTorch-master',
'.. /input/timm-pytorch-image-models/pytorch-image-models-master']
for package_path in package_paths:
sys.path.append(package_path )<import_modules>
df_train=pd.merge(df_train, df[['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]], how='inner', on='qa_id')
 df_test = pd.merge(df_test, df[['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]], how='inner', on='qa_id' )
import numpy as np
import pandas as pd
import os
import torch
import random
from albumentations import *
from albumentations.pytorch import ToTensorV2
from torch.utils.data import Dataset, DataLoader
import timm
import torch.nn as nn
import torch.nn.functional as F
from efficientnet_pytorch import EfficientNet
from pytorch.pytorchcv.model_provider import get_model as ptcv_get_model
from tqdm import tqdm<set_options>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True<categorify>
class CasDataset(Dataset):
def __init__(self, df, path, transforms):
self.df = df
self.path = path
self.transforms = transforms
def __len__(self):
return self.df.shape[0]
def __getitem__(self, idx):
image = cv2.imread(self.path+self.df.loc[idx, 'image_id'])
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 if self.transforms:
transformed = self.transforms(image=image)
 image = transformed['image']
return image<choose_model_class>
df_train.drop(['host', 'category'], inplace=True, axis=1)
 df_test.drop(['host', 'category'], inplace=True, axis=1 )
class CasModel(nn.Module):
def __init__(self, num_classes=5, model="resnet34"):
super().__init__()
if model == "resnet16":
self.backbone = ptcv_get_model("resnet16", pretrained=False)
 self.backbone.features.final_pool = nn.AdaptiveAvgPool2d(1)
 self.backbone.output = nn.Sequential(nn.Linear(512, num_classes))
elif model == "resnet34":
self.backbone = ptcv_get_model("resnet34", pretrained=False)
 self.backbone.features.final_pool = nn.AdaptiveAvgPool2d(1)
 self.backbone.output = nn.Sequential(nn.Linear(512, num_classes))
elif model == "efficientnet-b2":
self.backbone = EfficientNet.from_pretrained("efficientnet-b2")
 self.backbone._fc = nn.Linear(self.backbone._fc.in_features, num_classes)
 elif model == "efficientnet-b4":
self.backbone = EfficientNet.from_pretrained("efficientnet-b4")
 self.backbone._fc = nn.Linear(self.backbone._fc.in_features, num_classes)
 elif model == "efficientnet-b7":
self.backbone = EfficientNet.from_pretrained("efficientnet-b7")
 self.backbone._fc = nn.Linear(self.backbone._fc.in_features, num_classes)
 elif model == "vit-b":
self.backbone = timm.create_model("vit_base_patch16_384", pretrained=False)
 self.backbone.head = nn.Linear(self.backbone.head.in_features, num_classes)
 elif model == "vit-l":
self.backbone = timm.create_model("vit_large_patch16_384", pretrained=False)
 self.backbone.head = nn.Linear(self.backbone.head.in_features, num_classes)
 elif model == "vit-r":
self.backbone = timm.create_model("vit_base_resnet50d_224", pretrained=False)
 self.backbone.head = nn.Linear(self.backbone.head.in_features, num_classes)
 elif model == "efficientnet-b4-timm":
self.backbone = timm.create_model("tf_efficientnet_b4_ns", pretrained=False)
 self.backbone.classifier = nn.Linear(self.backbone.classifier.in_features, num_classes)
 else:
raise NotImplementedError
def forward(self, x):
x = self.backbone(x)
 return x<predict_on_test>
def _get_masks(tokens, max_seq_length):
if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+[0]*(max_seq_length-len(tokens))
def _get_segments(tokens, max_seq_length):
if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments=[]
first_sep=True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == '[SEP]':
current_segment_id=1
return segments+[0]*(max_seq_length-len(tokens))
def _trim_input_questions(title, question, max_sequence_length, t_max_len=30, q_max_len=479):
t = tokenizer2.tokenize(title)
 q = tokenizer2.tokenize(question)
 t_len = len(t)
 q_len = len(q)
 if(t_len+q_len+3)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if q_max_len > q_len:
q_new_len = q_len
else:
q_new_len = q_max_len
if t_new_len+q_new_len+3 > max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+q_new_len+3)))
 t = t[:t_new_len]
q = q[:q_new_len]
return t, q
def _get_ids(tokens, tokenizer, max_seq_length):
token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0]*(max_seq_length - len(token_ids))
return input_ids
def _convert_to_bert_inputs_questions(title, question,tokenizer, max_seq_length):
stoken = ['[CLS]']+title+['[SEP]']+question+['[SEP]']
input_ids = _get_ids(tokens=stoken, tokenizer=tokenizer,max_seq_length=max_seq_length)
 input_masks = _get_masks(tokens=stoken, max_seq_length=max_seq_length)
 input_segments = _get_segments(tokens=stoken, max_seq_length=max_seq_length)
 return [input_ids, input_masks, input_segments]
def compute_input_array_questions(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, col in tqdm(df[columns].iterrows()):
t, q = col['treated_question_title'], col['treated_question_body']
t,q = _trim_input_questions(t,q, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_questions(t,q, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.array(input_ids, dtype=np.int32), np.array(input_masks, dtype=np.int32),
np.array(input_segments, dtype=np.int32)]
def _trim_input_answers(answer, max_sequence_length, a_max_len=510):
a = tokenizer2.tokenize(answer)
 a_len = len(a)
 if(a_len+2)> max_sequence_length:
if a_max_len > a_len:
a_new_len = a_len
else:
a_new_len = a_max_len
if a_new_len+2 > max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(a_new_len+2)))
 a = a[:a_new_len]
return a
def _convert_to_bert_inputs_answers(answer, tokenizer, max_seq_length):
stoken = ['[CLS]']+answer+['[SEP]']
input_ids = _get_ids(tokens=stoken, tokenizer=tokenizer,max_seq_length=max_seq_length)
 input_masks = _get_masks(tokens=stoken, max_seq_length=max_seq_length)
 input_segments = _get_segments(tokens=stoken, max_seq_length=max_seq_length)
 return [input_ids, input_masks, input_segments]
def compute_input_array_answers(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, col in tqdm(df[columns].iterrows()):
a = col['treated_answer']
a = _trim_input_answers(a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_answers(a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.array(input_ids, dtype=np.int32), np.array(input_masks, dtype=np.int32),
np.array(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns])
 def compute_spearmanr(trues, preds):
rhos=[]
for t, p in zip(trues.T, preds.T):
rhos.append(
 spearmanr(t,p ).correlation 
)
 return np.nanmean(rhos)
 test_predictions=[]
class CustomCallback(keras.callbacks.Callback):
def __init__(self, valid_data, test_data, test_predictions=test_predictions, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.test_predictions = test_predictions
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions=[]
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(self.valid_outputs, np.average(self.valid_predictions, axis=0))
print(f"
validation rho: {round(rho_val,4)}")
 if(epoch+1)%4==0:
self.model.save_weights(f'/kaggle/working/bert-base-{fold}-{epoch}.hdf5')
 self.test_predictions.append(self.model.predict(self.test_inputs, batch_size=self.batch_size))
def inference_one_epoch(model, data_loader, device):
model.eval()
image_preds_all = []
for step,(imgs)in enumerate(data_loader):
imgs = imgs.to(device ).float()
image_preds = model(imgs)
 image_preds_all += [torch.softmax(image_preds, 1 ).detach().cpu().numpy() ]
image_preds_all = np.concatenate(image_preds_all, axis=0)
 return image_preds_all<create_dataframe>
bert_config=BertConfig(unk_token="[QBODY]", pad_token="[ANS]" ).from_pretrained('.. /input/bert-tensorflow/bert-base-uncased-config.json',output_hidden_states=True)
 def bertModel() :
input_ids_q = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_word_ids_q')
 input_mask_q = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_masks_q')
 input_segments_q = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_segments_q')
 input_categories = keras.layers.Input(( len([i for i in df.columns if i.startswith('host_')or i.startswith('cat_')])) ,dtype=tf.float32, name='input_categorical')
 input_new_features = keras.layers.Input(( 9,), dtype=tf.float32, name='input_new_features')
 input_ids_a = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_word_ids_a')
 input_mask_a = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_masks_a')
 input_segments_a = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_segments_a')
 bert_model_q = TFBertModel.from_pretrained(pretrained_model_name_or_path='.. /input/bert-tensorflow/bert-base-uncased-tf_model.h5',config = bert_config)
 bert_model_a = TFBertModel.from_pretrained(pretrained_model_name_or_path='.. /input/bert-tensorflow/bert-base-uncased-tf_model.h5',config = bert_config)
 _, _, hidden_states_q = bert_model_q([input_ids_q,input_mask_q, input_segments_a])
 _, _, hidden_states_a = bert_model_a([input_ids_a,input_mask_a, input_segments_a])
 h12_q = tf.reshape(hidden_states_q[-1][:,0],(-1,1,768))
h11_q = tf.reshape(hidden_states_q[-2][:,0],(-1,1,768))
concat_hidden_q = keras.layers.Concatenate(axis=2 )([h12_q, h11_q])
 h12_a = tf.reshape(hidden_states_a[-1][:,0],(-1,1,768))
h11_a = tf.reshape(hidden_states_a[-2][:,0],(-1,1,768))
concat_hidden_a = keras.layers.Concatenate(axis=2 )([h12_a, h11_a])
 x = keras.layers.GlobalAveragePooling1D()(concat_hidden_q)
 x_a = keras.layers.GlobalAveragePooling1D()(concat_hidden_a)
 x = keras.layers.Concatenate()([x, input_new_features, input_categories])
 x_a = keras.layers.Concatenate()([x_a,x, input_new_features, input_categories])
 out = keras.layers.Dense(len([i for i in output_categories if i.startswith('question')]), activation='sigmoid', name='final_dense_output_q' )(x)
 out_a = keras.layers.Dense(len([i for i in output_categories if i.startswith('answer')]), activation='sigmoid', name='final_dense_output_a' )(x_a)
 out = keras.layers.Concatenate()([out,out_a])
 model = keras.models.Model(inputs=[input_ids_q, input_mask_q, input_segments_q,input_categories, input_new_features, input_ids_a, input_mask_a, input_segments_a]
, outputs=out)
 model.compile(loss='binary_crossentropy', optimizer = keras.optimizers.Adam(lr=3e-5))
return model
seed_everything(960630)
 test_df = pd.DataFrame()
test_df['image_id'] = list(os.listdir('.. /input/cassava-leaf-disease-classification/test_images/'))
test_data = CasDataset(test_df, '.. /input/cassava-leaf-disease-classification/test_images/', transforms_test )<load_pretrained>
gkf = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body )
test_loader = DataLoader(
 test_data,
batch_size=32,
num_workers=8,
shuffle=False,
pin_memory=False,
)<load_pretrained>
outputs = compute_output_arrays(df_train, output_categories)
 inputs_q = compute_input_array_questions(df_train, ['treated_question_title','treated_question_body'], tokenizer2, MAX_SEQUENCE_LENGTH)
 inputs_a = compute_input_array_answers(df_train, ['treated_answer'], tokenizer2, MAX_SEQUENCE_LENGTH)
 test_inputs_q = compute_input_array_questions(df_test, ['treated_question_title','treated_question_body'], tokenizer2, MAX_SEQUENCE_LENGTH)
 test_inputs_a = compute_input_array_answers(df_test, ['treated_answer'], tokenizer2, MAX_SEQUENCE_LENGTH )
test_preds = []
device = torch.device('cuda:0')
 model = CasModel(model="resnet34" ).to(device)
 for fold in range(5):
model.load_state_dict(torch.load(f'.. /input/casresnet34/resnet34_f_{fold}.pth'))
with torch.no_grad() :
for t in range(tta):
test_preds += [inference_one_epoch(model, test_loader, device)]<feature_engineering>
histories = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold<2:
keras.backend.clear_session()
model = bertModel()
train_inputs_q = [inputs_q[i][train_idx] for i in range(3)]
train_inputs_a = [inputs_a[i][train_idx] for i in range(3)]
train_outputs = outputs[train_idx]
valid_inputs_q = [inputs_q[i][valid_idx] for i in range(3)]
valid_inputs_a = [inputs_a[i][valid_idx] for i in range(3)]
valid_outputs = outputs[valid_idx]
custom_callback = CustomCallback(valid_data=(valid_inputs_q + [df_train[[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]].iloc[valid_idx,:].values]+
[df_train[['question_body_num_words', 'answer_num_words','question_vs_answer_length', 'q_a_author_same', 'answer_user_cat','indirect', 'question_count', 'reason_explanation_words','choice_words']].iloc[valid_idx,:].values]+ valid_inputs_a,
valid_outputs),
test_data=test_inputs_q + [df_test[[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]].values]+
[df_test[['question_body_num_words', 'answer_num_words','question_vs_answer_length', 'q_a_author_same', 'answer_user_cat','indirect', 'question_count', 'reason_explanation_words','choice_words']].values]+test_inputs_a,
batch_size=8, fold=fold)
 H = model.fit(train_inputs_q+
[df_train[[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]].iloc[train_idx,:].values]+
[df_train[['question_body_num_words', 'answer_num_words','question_vs_answer_length', 'q_a_author_same', 'answer_user_cat','indirect', 'question_count', 'reason_explanation_words','choice_words']].iloc[train_idx,:].values] + train_inputs_a,
train_outputs, batch_size=4, epochs=3, callbacks=[custom_callback])
 histories.append(H )
test_preds = np.mean(test_preds, axis=0)
 test_df['label'] = np.argmax(test_preds, axis=1)
 test_df.head()<save_to_csv>
def _get_masks_google_qa(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments_google_qa(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids_google_qa(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input_google_qa(title, question, answer, max_sequence_length,
t_max_len=60, q_max_len=224, a_max_len=224):
t = tokenizer_google_qa.tokenize(title)
 q = tokenizer_google_qa.tokenize(question)
 a = tokenizer_google_qa.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs_google_qa(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids_google_qa(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks_google_qa(stoken, max_sequence_length)
 input_segments = _get_segments_google_qa(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays_google_qa(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance['treated_question_title'], instance['treated_question_body'], instance['treated_answer']
t, q, a = _trim_input_google_qa(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_google_qa(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)
 ]
def compute_output_arrays_google_qa(df, columns):
return np.asarray(df[columns])
 def bert_model_google_qa(learning_rate=4e-5,loss_function='binary_crossentropy'):
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[[input_word_ids, input_masks, input_segments]], outputs=out)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 return model
test_df.to_csv('submission.csv', index=False )<import_modules>
gkf_google_qa = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body)
 outputs_google_qa = compute_output_arrays_google_qa(df_train, output_categories)
 inputs_google_qa = compute_input_arays_google_qa(df_train, ['treated_question_title','treated_question_body','treated_answer'], tokenizer_google_qa, MAX_SEQUENCE_LENGTH)
 test_inputs_google_qa = compute_input_arays_google_qa(df_test, ['treated_question_title','treated_question_body','treated_answer'],tokenizer_google_qa, MAX_SEQUENCE_LENGTH)
print(tf.__version__)
<load_from_csv>
test_predictions_google_qa=[]
for fold,(train_idx, valid_idx)in enumerate(gkf_google_qa):
if fold<3:
keras.backend.clear_session()
model_qa = bert_model_google_qa()
print(f'/kaggle/input/google-qa-bert-trained-tfbert-hiddenl-preprocess/bert-base-{fold}-4.hdf5')
 model_qa.load_weights(f'/kaggle/input/google-qa-bert-trained-tfbert-hiddenl-preprocess/bert-base-{fold}-4.hdf5')
 preds_qa = model_qa.predict(test_inputs_google_qa , batch_size=8, verbose=1)
 test_predictions_google_qa.append(preds_qa)
 print(f'Fold:{fold} Completed' )
train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv" )<init_hyperparams>
test_preds_google_qa = [test_predictions_google_qa[i] for i in range(len(test_predictions_google_qa)) ]
test_preds_google_qa = [np.average(test_preds_google_qa, axis=0)for i in range(len(test_preds_google_qa)) ]
test_preds_google_qa = np.mean(test_preds_google_qa, axis=0)
 test_preds_google_qa.shape
cut=0
nets = 15
epochs = 45 
<prepare_x_and_y>
final_preds = np.average(np.array([test_preds, test_preds_google_qa]),axis=0)
 final_preds.shape
<create_dataframe><EOS> 
df_sub.iloc[:, 1:] = final_preds
df_sub.to_csv('submission.csv', index=False )