Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<categorify>
!pip install.. /input/huggingface-transformers/sacremoses-master/sacremoses-master
!pip install.. /input/huggingface-transformers/transformers-master/transformers-master
test_ = process_data(test )<define_variables>
tqdm.pandas()
warnings.filterwarnings('ignore')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1500<normalization>
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer2 = BertTokenizer.from_pretrained(BERT_PATH+'/assets/vocab.txt', do_lower_case=True,)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv',header=0,encoding='utf-8')
 df_test = pd.read_csv(PATH+'test.csv',header=0,encoding='utf-8')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 df = pd.concat([df_train,df_test],axis=0,ignore_index=True)
 print(f)
 output_categories = ['question_asker_intent_understanding', 'question_body_critical', 'question_conversational', 'question_expect_short_answer', 'question_fact_seeking', 'question_has_commonly_accepted_answer', 'question_interestingness_others', 'question_interestingness_self', 'question_multi_intent', 'question_not_really_a_question', 'question_opinion_seeking', 'question_type_choice', 'question_type_compare', 'question_type_consequence', 'question_type_definition', 'question_type_entity', 'question_type_instructions', 'question_type_procedure', 'question_type_reason_explanation', 'question_type_spelling', 'question_well_written', 'answer_helpful', 'answer_level_of_information', 'answer_plausible', 'answer_relevance', 'answer_satisfaction', 'answer_type_instructions', 'answer_type_procedure', 'answer_type_reason_explanation', 'answer_well_written']
input_categories = ['question_title', 'question_body', 'answer']
print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(Dropout_Model)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(Dropout_Model)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))<prepare_x_and_y>
tree_tokenizer = TreebankWordTokenizer()
def get_tree_tokens(x):
x = tree_tokenizer.tokenize(x)
 x = ' '.join(x)
 return x
def run_training(fold, seed):
seed_everything(seed)
 train = folds
test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions 
<categorify>
for col in input_categories:
df_train[f'treated_{col}'] = df_train[col].apply(lambda x: get_tree_tokens(x))
df_test[f'treated_{col}'] = df_test[col].apply(lambda x: get_tree_tokens(x))
df[f'treated_{col}'] = df_test[col].apply(lambda x: get_tree_tokens(x))
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<categorify>
tokenizer = text.Tokenizer(lower=False )
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions 
<save_to_csv>
X_train_question = df_train['question_body']
X_train_title = df_train['question_title']
X_train_answer = df_train['answer']
X_test_question = df_test['question_body']
X_test_title = df_test['question_title']
X_test_answer = df_test['answer']
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<import_modules>
tokenizer.fit_on_texts(list(X_train_title)+list(X_train_question)+list(X_train_answer)+list(X_test_title)+list(X_test_question)+list(X_test_answer))
BaseShuffleSplit, _validate_shuffle_split
def IterativeStratification(labels, r, random_state):

 n_samples = labels.shape[0]
test_folds = np.zeros(n_samples, dtype=int)
 c_folds = r * n_samples
c_folds_labels = np.outer(r, labels.sum(axis=0))
labels_not_processed_mask = np.ones(n_samples, dtype=bool)
 while np.any(labels_not_processed_mask):
num_labels = labels[labels_not_processed_mask].sum(axis=0)
 if num_labels.sum() == 0:
sample_idxs = np.where(labels_not_processed_mask)[0]
for sample_idx in sample_idxs:
fold_idx = np.where(c_folds == c_folds.max())[0]
if fold_idx.shape[0] > 1:
fold_idx = fold_idx[random_state.choice(fold_idx.shape[0])]
test_folds[sample_idx] = fold_idx
c_folds[fold_idx] -= 1
break
label_idx = np.where(num_labels == num_labels[np.nonzero(num_labels)].min())[0]
if label_idx.shape[0] > 1:
label_idx = label_idx[random_state.choice(label_idx.shape[0])]
sample_idxs = np.where(np.logical_and(labels[:, label_idx].flatten() , labels_not_processed_mask)) [0]
for sample_idx in sample_idxs:
label_folds = c_folds_labels[:, label_idx]
fold_idx = np.where(label_folds == label_folds.max())[0]
if fold_idx.shape[0] > 1:
temp_fold_idx = np.where(c_folds[fold_idx] ==
c_folds[fold_idx].max())[0]
fold_idx = fold_idx[temp_fold_idx]
if temp_fold_idx.shape[0] > 1:
fold_idx = fold_idx[random_state.choice(temp_fold_idx.shape[0])]
test_folds[sample_idx] = fold_idx
labels_not_processed_mask[sample_idx] = False
c_folds_labels[fold_idx, labels[sample_idx]] -= 1
c_folds[fold_idx] -= 1
return test_folds
class MultilabelStratifiedKFold(_BaseKFold):

 def __init__(self, n_splits=3, shuffle=False, random_state=None):
super(MultilabelStratifiedKFold, self ).__init__(n_splits=n_splits, shuffle=shuffle, random_state=random_state)
 def _make_test_folds(self, X, y):
y = np.asarray(y, dtype=bool)
 type_of_target_y = type_of_target(y)
 if type_of_target_y != 'multilabel-indicator':
raise ValueError(
 'Supported target type is: multilabel-indicator.Got {!r} instead.'.format(type_of_target_y))
num_samples = y.shape[0]
rng = check_random_state(self.random_state)
 indices = np.arange(num_samples)
 if self.shuffle:
rng.shuffle(indices)
 y = y[indices]
r = np.asarray([1 / self.n_splits] * self.n_splits)
 test_folds = IterativeStratification(labels=y, r=r, random_state=rng)
 return test_folds[np.argsort(indices)]
def _iter_test_masks(self, X=None, y=None, groups=None):
test_folds = self._make_test_folds(X, y)
 for i in range(self.n_splits):
yield test_folds == i
def split(self, X, y, groups=None):

 y = check_array(y, ensure_2d=False, dtype=None)
 return super(MultilabelStratifiedKFold, self ).split(X, y, groups)
 class RepeatedMultilabelStratifiedKFold(_RepeatedSplits):

 def __init__(self, n_splits=5, n_repeats=10, random_state=None):
super(RepeatedMultilabelStratifiedKFold, self ).__init__(
 MultilabelStratifiedKFold, n_repeats=n_repeats, random_state=random_state,
n_splits=n_splits)
 class MultilabelStratifiedShuffleSplit(BaseShuffleSplit):

 def __init__(self, n_splits=10, test_size="default", train_size=None,
random_state=None):
super(MultilabelStratifiedShuffleSplit, self ).__init__(
 n_splits=n_splits, test_size=test_size, train_size=train_size, random_state=random_state)
 def _iter_indices(self, X, y, groups=None):
n_samples = _num_samples(X)
 y = check_array(y, ensure_2d=False, dtype=None)
 y = np.asarray(y, dtype=bool)
 type_of_target_y = type_of_target(y)
 if type_of_target_y != 'multilabel-indicator':
raise ValueError(
 'Supported target type is: multilabel-indicator.Got {!r} instead.'.format(
 type_of_target_y))
n_train, n_test = _validate_shuffle_split(n_samples, self.test_size,
self.train_size)
 n_samples = y.shape[0]
rng = check_random_state(self.random_state)
 y_orig = y.copy()
r = np.array([n_train, n_test])/(n_train + n_test)
 for _ in range(self.n_splits):
indices = np.arange(n_samples)
 rng.shuffle(indices)
 y = y_orig[indices]
test_folds = IterativeStratification(labels=y, r=r, random_state=rng)
 test_idx = test_folds[np.argsort(indices)] == 1
test = np.where(test_idx)[0]
train = np.where(~test_idx)[0]
yield train, test
def split(self, X, y, groups=None):

 y = check_array(y, ensure_2d=False, dtype=None)
 return super(MultilabelStratifiedShuffleSplit, self ).split(X, y, groups )<import_modules>
nlp = English()
sentencizer = nlp.create_pipe('sentencizer')
 nlp.add_pipe(sentencizer )
import numpy as np
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt<define_variables>
def split_document(texts):
all_sents = []
max_num_sentences = 0.0
for text in texts:
doc = nlp(text)
 sents=[]
for i,sent in enumerate(doc.sents):
sents.append(sent.text)
 all_sents.append(sents)
 return all_sents
X_train_question = split_document(X_train_question)
 X_train_answer = split_document(X_train_answer)
 X_test_question = split_document(X_test_question)
 X_test_answer = split_document(X_test_answer)
TEST_FEATURES_PATH = "/kaggle/input/lish-moa/test_features.csv"
TRAIN_FEATURES_PATH = "/kaggle/input/lish-moa/train_features.csv"
TRAIN_TARGETS_PATH = "/kaggle/input/lish-moa/train_targets_scored.csv"
TRAIN_TARGETS_NONSCORED_PATH = "/kaggle/input/lish-moa/train_targets_nonscored.csv"
SAMPLE_SUB_PATH = "/kaggle/input/lish-moa/sample_submission.csv"<load_from_csv>
def add_question_metadata_features(text):
doc=nlp(text)
 indirect = 0
choice_words=0
reason_explanation_words = 0
question_count = 0
for sent in doc.sents:
if '?' in sent.text and '?' == sent.text[-1]:
question_count += 1
for token in sent:
if token.text.lower() =='why':
reason_explanation_words+=1
elif token.text.lower() =='or':
choice_words+=1
if question_count==0:
indirect+=1
return np.array([indirect, question_count, reason_explanation_words, choice_words] )
test_features_df = pd.read_csv(TEST_FEATURES_PATH ).sort_values(by='sig_id')
 train_features_df = pd.read_csv(TRAIN_FEATURES_PATH ).sort_values(by='sig_id')
 train_targets_df = pd.read_csv(TRAIN_TARGETS_PATH ).sort_values(by='sig_id')
 train_targets_nonscored_df = pd.read_csv(TRAIN_TARGETS_NONSCORED_PATH)
 sample_sub_df = pd.read_csv(SAMPLE_SUB_PATH ).sort_values(by='sig_id' )<count_values>
ans_user_and_category=df_train[df_train[['answer_user_name', 'category']].duplicated() ][['answer_user_name', 'category']].values
ans_user_and_category.shape
train_targets_df['5-alpha_reductase_inhibitor'].value_counts()<count_values>
def question_answer_author_same(df):
q_username = df['question_user_name']
a_username = df['answer_user_name']
author_same=[]
for i in range(len(df)) :
if q_username[i] == a_username[i]:
author_same.append(int(1))
else:
author_same.append(int(0))
return author_same
value_counts_arr = np.sort([train_targets_df[col].value_counts() [1] for col in train_targets_df.columns])
 print(value_counts_arr )<categorify>
def add_external_features(df):
df['question_body'] = df['question_body'].progress_apply(lambda x: str(x))
df['question_body_num_words'] = df['question_body'].str.count('\S+')
 df['answer'] = df['answer'].progress_apply(lambda x: str(x))
df['answer_num_words'] = df['answer'].str.count('\S+')
 df['question_vs_answer_length'] = df['question_body_num_words']/df['answer_num_words']
df['q_a_author_same'] = question_answer_author_same(df)
 answer_user_cat = []
for i in tqdm(df[['answer_user_name', 'category']].values):
if i in ans_user_and_category:
answer_user_cat.append(int(1))
else:
answer_user_cat.append(int(0))
df['answer_user_cat'] = answer_user_cat
handmade_features=[]
for text in df['question_body'].values:
handmade_features.append(add_question_metadata_features(text))
return df, np.array(handmade_features)
<feature_engineering>
df_train, handmade_features = add_external_features(df_train)
 df_test, handmade_features_test = add_external_features(df_test)
 df_train = pd.concat([df_train,pd.DataFrame(handmade_features, columns=['indirect', 'question_count', 'reason_explanation_words', 'choice_words'])],axis=1)
 df_test = pd.concat([df_test,pd.DataFrame(handmade_features_test, columns=['indirect', 'question_count', 'reason_explanation_words', 'choice_words'])],axis=1 )
<prepare_x_and_y>
num_words_scaler = MinMaxScaler()
df_train[['question_body_num_words', 'answer_num_words']] = num_words_scaler.fit_transform(df_train[['question_body_num_words', 'answer_num_words']].values)
 df_test[['question_body_num_words', 'answer_num_words']] = num_words_scaler.transform(df_test[['question_body_num_words', 'answer_num_words']].values )
X = train_features_df.drop(columns=['sig_id'])
 X_test = test_features_df.drop(columns=['sig_id'])
 y = train_targets_df.drop(columns=['sig_id'] )<normalization>
df=pd.concat([df,pd.get_dummies(df['host'], drop_first=False, prefix='host')],axis=1)
 df=pd.concat([df,pd.get_dummies(df['category'], drop_first=False, prefix='cat')],axis=1 )
transformer = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution="normal")
 transformer.fit(X[genes+cells] )<categorify>
len(['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')] )
X.loc[:,genes+cells] = transformer.transform(X[genes+cells])
 X_test.loc[:,genes+cells] = transformer.transform(X_test[genes+cells])
<import_modules>
df_train=pd.merge(df_train, df[['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]], how='inner', on='qa_id')
 df_test = pd.merge(df_test, df[['qa_id']+[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]], how='inner', on='qa_id' )
from sklearn.decomposition import PCA<concatenate>
def pca_number(X_genes, X_test_genes, X_cells, X_test_cells, X, X_test, n_gene=200, n_cells=10):
pca = PCA(n_components=n_gene)
 data = pca.fit_transform(pd.concat([X_genes,X_test_genes]))
X_temp = pd.DataFrame(data=data, columns = ['pca_gene'+str(i)for i in range(n_gene)])
 X_genes = X_temp[:len(X_genes)]
X_test_genes = X_temp[len(X_genes):].reset_index(drop=True)
 pca = PCA(n_components=n_cells)
 data = pca.fit_transform(pd.concat([X_cells,X_test_cells]))
X_temp = pd.DataFrame(data=data, columns = ['pca_cells'+str(i)for i in range(n_cells)])
 X_cells = X_temp[:len(X_cells)]
X_test_cells = X_temp[len(X_cells.index):].reset_index(drop=True)
 New_x = pd.concat([X, X_genes, X_cells], axis=1)
 New_xtest = pd.concat([X_test, X_test_genes, X_test_cells], axis=1)
 return New_x, New_xtest<define_variables>
features = X.columns
genes = [col for col in features if col.startswith('g-')]<define_variables>
df_train.drop(['host', 'category'], inplace=True, axis=1)
 df_test.drop(['host', 'category'], inplace=True, axis=1 )
cells = [col for col in features if col.startswith('c-')]<categorify>
def _get_masks(tokens, max_seq_length):
if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+[0]*(max_seq_length-len(tokens))
def _get_segments(tokens, max_seq_length):
if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments=[]
first_sep=True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == '[ANS]':
current_segment_id=1
return segments+[0]*(max_seq_length-len(tokens))
def _trim_input(title, question, answer, max_sequence_length, t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer2.tokenize(title)
 q = tokenizer2.tokenize(question)
 a = tokenizer2.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _get_ids(tokens, tokenizer, max_seq_length):
token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0]*(max_seq_length - len(token_ids))
return input_ids
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_seq_length):
stoken = ['[CLS]']+title+['[QBODY]']+question+['[ANS]']+answer+['[SEP]']
input_ids = _get_ids(tokens=stoken, tokenizer=tokenizer,max_seq_length=max_seq_length)
 input_masks = _get_masks(tokens=stoken, max_seq_length=max_seq_length)
 input_segments = _get_segments(tokens=stoken, max_seq_length=max_seq_length)
 return [input_ids, input_masks, input_segments]
def compute_input_array(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, col in tqdm(df[columns].iterrows()):
t, q, a = col['treated_question_title'], col['treated_question_body'], col['treated_answer']
t,q,a = _trim_input(t,q,a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t,q,a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.array(input_ids, dtype=np.int32), np.array(input_masks, dtype=np.int32),
np.array(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
n_genes=35
n_cells=5
kmeans = KMeans(n_genes)
 data = kmeans.fit(X[genes].append(X_test[genes]))
X['kmeans_g'] = data.labels_[:X.shape[0]]
X_test['kmeans_g'] = data.labels_[X.shape[0]:]
kmeans = KMeans(n_cells)
 data = kmeans.fit(X[cells].append(X_test[cells]))
X['kmeans_c'] = data.labels_[:X.shape[0]]
X_test['kmeans_c'] = data.labels_[X.shape[0]:] 
<feature_engineering>
def compute_spearmanr(trues, preds):
rhos=[]
for t, p in zip(trues.T, preds.T):
rhos.append(
 spearmanr(t,p ).correlation 
)
 return np.nanmean(rhos)
 test_predictions=[]
class CustomCallback(keras.callbacks.Callback):
def __init__(self, valid_data, test_data, test_predictions=test_predictions, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.test_predictions = test_predictions
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions=[]
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(self.valid_outputs, np.average(self.valid_predictions, axis=0))
print(f"
validation rho: {round(rho_val,4)}")
 if(epoch+1)%4==0:
self.model.save_weights(f'/kaggle/working/bert-base-{fold}-{epoch}.hdf5')
 self.test_predictions.append(self.model.predict(self.test_inputs, batch_size=self.batch_size))
for df in X, X_test:
df['g_sum'] = df[genes].sum(axis = 1)
 df['g_mean'] = df[genes].mean(axis = 1)
 df['g_std'] = df[genes].std(axis = 1)
 df['g_kurt'] = df[genes].kurtosis(axis = 1)
 df['g_skew'] = df[genes].skew(axis = 1)
 df['c_sum'] = df[cells].sum(axis = 1)
 df['c_mean'] = df[cells].mean(axis = 1)
 df['c_std'] = df[cells].std(axis = 1)
 df['c_kurt'] = df[cells].kurtosis(axis = 1)
 df['c_skew'] = df[cells].skew(axis = 1)
 df['gc_sum'] = df[genes + cells].sum(axis = 1)
 df['gc_mean'] = df[genes + cells].mean(axis = 1)
 df['gc_std'] = df[genes + cells].std(axis = 1)
 df['gc_kurt'] = df[genes + cells].kurtosis(axis = 1)
 df['gc_skew'] = df[genes + cells].skew(axis = 1)
 df['g_sum_cp_dose'] = df['cp_dose'].map(dict(zip(['D1','D2'],df[genes+['cp_dose']].groupby(['cp_dose'] ).sum().values.sum(axis=1))))
df['g_mean_cp_dose'] = df['cp_dose'].map(dict(zip(['D1','D2'],df[genes+['cp_dose']].groupby(['cp_dose'] ).mean().values.mean(axis=1))))
df['g_std_cp_dose'] = df['cp_dose'].map(dict(zip(['D1','D2'],df[genes+['cp_dose']].groupby(['cp_dose'] ).std().values.std(axis=1))))
df['c_sum_cp_dose'] = df['cp_dose'].map(dict(zip(['D1','D2'],df[cells+['cp_dose']].groupby(['cp_dose'] ).sum().values.sum(axis=1))))
df['c_mean_cp_dose'] = df['cp_dose'].map(dict(zip(['D1','D2'],df[cells+['cp_dose']].groupby(['cp_dose'] ).mean().values.mean(axis=1))))
df['c_std_cp_dose'] = df['cp_dose'].map(dict(zip(['D1','D2'],df[cells+['cp_dose']].groupby(['cp_dose'] ).std().values.std(axis=1)))) 
<categorify>
bert_config=BertConfig(unk_token="[QBODY]", pad_token="[ANS]" ).from_pretrained('.. /input/bert-tensorflow/bert-base-uncased-config.json',output_hidden_states=True)
 def bertModel() :
input_ids = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_word_ids')
 input_mask = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_masks')
 input_segments = keras.layers.Input(( MAX_SEQUENCE_LENGTH), dtype = tf.int32, name = 'input_segments')
 input_categories = keras.layers.Input(( len([i for i in df.columns if i.startswith('host_')or i.startswith('cat_')])) ,dtype=tf.float32, name='input_categorical')
 input_new_features = keras.layers.Input(( 9,), dtype=tf.float32, name='input_new_features')
 bert_model = TFBertModel.from_pretrained(pretrained_model_name_or_path='.. /input/bert-tensorflow/bert-base-uncased-tf_model.h5',config = bert_config)
 sequence_output, pooler_output, hidden_states = bert_model([input_ids,input_mask, input_segments])
 h12 = tf.reshape(hidden_states[-1][:,0],(-1,1,768))
h11 = tf.reshape(hidden_states[-2][:,0],(-1,1,768))
h10 = tf.reshape(hidden_states[-3][:,0],(-1,1,768))
h09 = tf.reshape(hidden_states[-4][:,0],(-1,1,768))
concat_hidden = keras.layers.Concatenate(axis=2 )([h12, h11, h10, h09])
 x = keras.layers.GlobalAveragePooling1D()(concat_hidden)
 x = keras.layers.Concatenate()([x, input_new_features, input_categories])
 x = keras.layers.Dropout(0.2 )(x)
 out = keras.layers.Dense(len(output_categories), activation='sigmoid', name='final_dense_output' )(x)
 model = keras.models.Model(inputs=[input_ids, input_mask, input_segments,input_categories, input_new_features], outputs=out)
 model.compile(loss='binary_crossentropy', optimizer = keras.optimizers.Adam(lr=3e-5))
return model
X = pd.get_dummies(X, columns=['cp_dose', 'cp_time'])
 X_test = pd.get_dummies(X_test, columns=['cp_dose', 'cp_time'] )<define_variables>
gkf = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body )
X_genes = X[genes].copy()
X_test_genes = X_test[genes].copy()
X_cells = X[cells].copy()
X_test_cells = X_test[cells].copy()<set_options>
outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_array(df_train, ['treated_question_title','treated_question_body','treated_answer'], tokenizer2, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_array(df_test, ['treated_question_title','treated_question_body','treated_answer'], tokenizer2, MAX_SEQUENCE_LENGTH )
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<compute_test_metric>
histories = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold<3:
keras.backend.clear_session()
model = bertModel()
train_inputs = [inputs[i][train_idx] for i in range(3)]
train_inputs
train_outputs = outputs[train_idx]
valid_inputs = [inputs[i][valid_idx] for i in range(3)]
valid_outputs = outputs[valid_idx]
custom_callback = CustomCallback(valid_data=(valid_inputs + [df_train[[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]].iloc[valid_idx,:].values]+
[df_train[['question_body_num_words', 'answer_num_words','question_vs_answer_length', 'q_a_author_same', 'answer_user_cat','indirect', 'question_count', 'reason_explanation_words','choice_words']].iloc[valid_idx,:].values],
valid_outputs),
test_data=test_inputs + [df_test[[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]].values]+
[df_test[['question_body_num_words', 'answer_num_words','question_vs_answer_length', 'q_a_author_same', 'answer_user_cat','indirect', 'question_count', 'reason_explanation_words','choice_words']].values],
batch_size=8, fold=fold)
 H = model.fit(train_inputs+
[df_train[[i for i in df.columns if i.startswith('host_')or i.startswith('cat_')]].iloc[train_idx,:].values]+
[df_train[['question_body_num_words', 'answer_num_words','question_vs_answer_length', 'q_a_author_same', 'answer_user_cat','indirect', 'question_count', 'reason_explanation_words','choice_words']].iloc[train_idx,:].values],
train_outputs, batch_size=8, epochs=4, callbacks=[custom_callback])
 histories.append(H )
X, X_test = pca_number(X_genes, X_test_genes, X_cells, X_test_cells, X, X_test, n_gene=650, n_cells=50 )<define_variables>
test_preds = [test_predictions[i] for i in range(len(test_predictions)) ]
test_preds = [np.average(test_preds[i], axis=0)for i in range(len(test_preds)) ]
test_preds = np.mean(test_predictions, axis=0)
 df_sub.iloc[:, 1:] = test_preds
df_sub.to_csv('submission.csv', index=False )
<data_type_conversions><EOS> 
len([i for i in df_test.columns if i.startswith('cat_')or i.startswith('host_')] )