Kaggle competition: Google QUEST Q&A Labeling
def create_mlp(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate):
inp = tf.keras.layers.Input(shape =(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i+1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation('sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = learning_rate),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = label_smoothing),
metrics = tf.keras.metrics.AUC(name = 'AUC'), 
)
 return model<init_hyperparams>
data_dir = '.. /input/'
NETLOC_PATH = '.. /input/bert-use-netloc/unique_netloc.pickle'
USE_URL = '.. /input/universalsentenceencoder4/'
TRAIN_CSV_PATH = path_join(data_dir,'google-quest-challenge', 'train.csv')
 TEST_CSV_PATH = path_join(data_dir, 'google-quest-challenge','test.csv')
 POST_PROCESS = False
batch_size = 4096
hidden_units = [384, 896, 896, 394]
dropout_rates = [0.10143786981358652, 0.19720339053599725, 0.2703017847244654, 0.23148340929571917, 0.2357768967777311]
label_smoothing = 1e-2
learning_rate = 1e-3<load_pretrained>
BertTokenizer, BertModel, BertForSequenceClassification, BertConfig,
WEIGHTS_NAME, CONFIG_NAME, AdamW, get_linear_schedule_with_warmup,
get_cosine_schedule_with_warmup, BertPreTrainedModel, 
)
 print(transformers.__version__)
 def set_seeds(SEED=42):
os.environ["PYTHONHASHSEED"] = str(SEED)
 np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
SEED=2019
set_seeds(SEED)
 HEAD_TAIL = True
num_models = 2
models = []
for i in range(num_models):
clf = create_mlp(len(features), 1, hidden_units, dropout_rates, label_smoothing, learning_rate)
 clf.load_weights(f'.. /input/js-nn-models/JSModel_{i}.hdf5')
 models.append(clf )<load_pretrained>
test = pd.read_csv(TEST_CSV_PATH ).fillna(' ')
 train = pd.read_csv(TRAIN_CSV_PATH ).fillna(' ' )
f_mean = np.load('.. /input/js-nn-models/f_mean.npy' )<split>
train = clean_data(train, ['answer', 'question_body', 'question_title'])
 test = clean_data(test, ['answer', 'question_body', 'question_title'])
 tokenizer = Tokenizer()
full_text = list(train['question_body'])+ \
list(train['answer'])+ \
list(train['question_title'])+ \
list(test['question_body'])+ \
list(test['answer'])+ \
list(test['question_title'])
 tokenizer.fit_on_texts(full_text)
 embed_size=300
embedding_path = "/kaggle/input/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl"
embedding_matrix, nb_words, unknown_words = build_matrix(embedding_path, path_join(data_dir,'wiki-news-300d-1M.vec'), tokenizer.word_index,
100000, embed_size )
env = janestreet.make_env()
env_iter = env.iter_test()<feature_engineering>
seed(SEED)
 random.seed(SEED )
opt_th = 0.502
for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = 0.
 for clf in models:
pred += clf(x_tt, training = False ).numpy().item() / num_models
pred_df.action = np.where(pred >= opt_th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<load_from_csv>
train = pd.read_csv(TRAIN_CSV_PATH)
 test = pd.read_csv(TEST_CSV_PATH)
 print(train.shape, test.shape )
train_data=pd.read_csv(".. /input/train.csv",usecols=['Producto_ID','Demanda_uni_equil'])
<feature_engineering>
find = re.compile(r"^[^.]*")
 train['netloc'] = train['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 test['netloc'] = test['url'].apply(lambda x: re.findall(find, urlparse(x ).netloc)[0])
 with open(NETLOC_PATH, "rb")as input_file:
unique_netloc = pickle.load(input_file)
 diff1 = list(set(test.netloc.unique() ).difference(set(unique_netloc)))
 unknownlist = ['unknown1','unknown2','unknown3', 'unknown4', 'unknown5', 'unknown6', 'unknown7', 'unknown8', 'unknown9',
'unknown10', 'unknown11', 'unknown12', 'unknown13', 'unknown14', 'unknown15', 'unknown16', 'unknown17', 'unknown18',
'unknown19', 'unknown20', 'unknown21', 'unknown22', 'unknown23', 'unknown24', 'unknown25', 'unknown26', 'unknown27',
'unknown28', 'unknown29', 'unknown30', 'unknown31', 'unknown32', 'unknown33', 'unknown34', 'unknown35', 'unknown36',
'unknown37', 'unknown38','unknown39', 'unknown40']
if len(diff1)> 50:
exit(1)
 for i,k in enumerate(diff1):
idx = test[test.netloc==k].index
test.iloc[idx,11] = unknownlist[i]
tmp = pd.DataFrame([['unknown1', 'LIFE_ARTS'],['unknown2', 'CULTURE'],['unknown3', 'SCIENCE'],
['unknown4', 'STACKOVERFLOW'],['unknown5', 'TECHNOLOGY'],['unknown6', 'LIFE_ARTS'],
['unknown7', 'CULTURE'],['unknown8', 'SCIENCE'],['unknown9', 'STACKOVERFLOW'],
['unknown10', 'TECHNOLOGY'],['unknown11', 'CULTURE'],['unknown12', 'SCIENCE'],
['unknown13', 'STACKOVERFLOW'],['unknown14', 'TECHNOLOGY'],['unknown15', 'LIFE_ARTS'],
['unknown16', 'LIFE_ARTS'],['unknown17', 'CULTURE'],['unknown18', 'SCIENCE'],
['unknown19', 'STACKOVERFLOW'],['unknown20', 'TECHNOLOGY'],['unknown21', 'LIFE_ARTS'],
['unknown22', 'CULTURE'],['unknown23', 'SCIENCE'],['unknown24', 'STACKOVERFLOW'],
['unknown25', 'TECHNOLOGY'],['unknown26', 'CULTURE'],['unknown27', 'SCIENCE'],
['unknown28', 'STACKOVERFLOW'],['unknown29', 'TECHNOLOGY'],['unknown30', 'LIFE_ARTS'] ,
['unknown31', 'LIFE_ARTS'],['unknown32', 'CULTURE'],['unknown33', 'SCIENCE'],
['unknown34', 'STACKOVERFLOW'],['unknown35', 'TECHNOLOGY'],['unknown36', 'LIFE_ARTS'],
['unknown37', 'CULTURE'],['unknown38', 'SCIENCE'],['unknown39', 'STACKOVERFLOW'],
['unknown40', 'TECHNOLOGY']
],columns = ['netloc', 'category'] )
train_data['log_Dem']=np.log(np.array(train_data['Demanda_uni_equil'].tolist())+1)
<groupby>
features = ['netloc', 'category']
merged = pd.concat([train[features], test[features]])
 merged = merged.append(tmp[-(40 - len(diff1)) :])
 ohe = OneHotEncoder()
ohe.fit(merged)
 features_train = ohe.transform(train[features] ).toarray()
features_test = ohe.transform(test[features] ).toarray()
print(features_train.shape, features_test.shape)
mean_data=train_data.groupby(train_data['Producto_ID'] ).mean()
print(mean_data )<load_from_csv>
module_url = USE_URL
embed = hub.load(module_url )
test_data=pd.read_csv(".. /input/test.csv",usecols=['id','Producto_ID'])
 target=np.zeros(test_data.shape[0])
 log_target=np.zeros(test_data.shape[0])
 for pid in mean_data.index:
target[test_data[test_data['Producto_ID']==pid]['id'].values]=mean_data.ix[pid]['Demanda_uni_equil']
log_target[test_data[test_data['Producto_ID']==pid]['id'].values]=mean_data.ix[pid]['log_Dem'] 
<save_to_csv>
input_columns = ['question_title', 'question_body', 'answer']
embeddings_train = {}
embeddings_test = {}
for text in input_columns:
print(text)
 test_text = test[text].str.replace('?', '.' ).str.replace('!', '.' ).tolist()
curr_test_emb = []
batch_size = 4
ind = 0
ind = 0
while ind*batch_size < len(test_text):
curr_test_emb.append(embed(test_text[ind*batch_size:(ind + 1)*batch_size])["outputs"].numpy())
 ind += 1
embeddings_test[text + '_embedding'] = np.vstack(curr_test_emb)
 del embed
gc.collect()
test_data['Demanda_uni_equil']=np.exp(log_target)-1
print(test_data)
 test_data.to_csv('result_groupmean_log.csv',index=False,columns=['id','Demanda_uni_equil'] )<filter>
torch.cuda.empty_cache()
gc.collect()
test_data[test_data['Producto_ID']==41]['id']<import_modules>
cuda.select_device(0)
 cuda.close()
cuda.select_device(0)
 cuda.close()
cuda.select_device(0 )
sys.path.append('.. /input/iterativestratification')
 warnings.filterwarnings('ignore')
 os.listdir('.. /input/lish-moa')
 pd.set_option('max_columns', 2000 )<define_variables>
l2_dist = lambda x, y: np.power(x - y, 2 ).sum(axis=1)
 cos_dist = lambda x, y:(x*y ).sum(axis=1)
 dist_features_test = np.array([
l2_dist(embeddings_test['question_title_embedding'], embeddings_test['answer_embedding']),
l2_dist(embeddings_test['question_body_embedding'], embeddings_test['answer_embedding']),
l2_dist(embeddings_test['question_body_embedding'], embeddings_test['question_title_embedding']),
cos_dist(embeddings_test['question_title_embedding'], embeddings_test['answer_embedding']),
cos_dist(embeddings_test['question_body_embedding'], embeddings_test['answer_embedding']),
cos_dist(embeddings_test['question_body_embedding'], embeddings_test['question_title_embedding'])
 ] ).T
X_test = np.hstack([item for k, item in embeddings_test.items() ] + [features_test, dist_features_test])
n_comp_GENES = 450
n_comp_CELLS = 2
VarianceThreshold_for_FS = 0.9
Dropout_Model = 0.25
QT_n_quantile_min=50,
QT_n_quantile_max=1000,
print('n_comp_GENES', n_comp_GENES, 'n_comp_CELLS', n_comp_CELLS, 'total', n_comp_GENES + n_comp_CELLS )<load_from_csv>
max_len = 500
max_len_title = 30
train_question_tokenized = pad_sequences(tokenizer.texts_to_sequences(train['question_body']), maxlen = max_len)
 train_answer_tokenized = pad_sequences(tokenizer.texts_to_sequences(train['answer']), maxlen = max_len)
 train_title_tokenized = pad_sequences(tokenizer.texts_to_sequences(train['question_title']), maxlen = max_len_title)
 test_question_tokenized = pad_sequences(tokenizer.texts_to_sequences(test['question_body']), maxlen = max_len)
 test_answer_tokenized = pad_sequences(tokenizer.texts_to_sequences(test['answer']), maxlen = max_len)
 test_title_tokenized = pad_sequences(tokenizer.texts_to_sequences(test['question_title']), maxlen = max_len_title )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 train_drugs = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<feature_engineering>
LEN_USE_FEATURES = 1646
class BertForSequenceClassification_v2Answer(BertPreTrainedModel):
r
def __init__(self, config,embedding_matrix=None):
super(BertForSequenceClassification_v2Answer,self ).__init__(config)
 self.num_labels = config.num_labels
self.embedding = nn.Embedding(*embedding_matrix.shape)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.linear_q_add = nn.Linear(300, 128)
 self.linear_q_add1 = nn.Linear(128, self.config.num_labels)
 self.bert = BertModel(config)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.linear1 = nn.Linear(config.hidden_size+LEN_USE_FEATURES + self.config.num_labels ,config.hidden_size+LEN_USE_FEATURES + self.config.num_labels)
 self.classifier = nn.Linear(config.hidden_size+LEN_USE_FEATURES + self.config.num_labels , self.config.num_labels)
 self.init_weights()
def forward(
 self,
input_ids=None,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
inputs_embeds=None,
labels=None,
usefeatures = None,
question_data = None,
answer_data = None,
title_data = None 
):
outputs = self.bert(
 input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds, 
)
 last_hidden_state = outputs[0][:,0,:].reshape(( -1, 1, 768))
pooled_output = outputs[1]
hidden_states = outputs[2]
h12 = hidden_states[-1][:, 0].reshape(( -1, 1, 768))
h11 = hidden_states[-2][:, 0].reshape(( -1, 1, 768))
h10 = hidden_states[-3][:, 0].reshape(( -1, 1, 768))
h9 = hidden_states[-4][:, 0].reshape(( -1, 1, 768))
all_h = torch.cat([ h9, h10, h11, h12], 1)
 all_h_mean = torch.mean(all_h, 1 ).reshape(( -1,1,768))
mean_pool = torch.mean(torch.cat([last_hidden_state,all_h_mean],1),1)
 pooled_output = self.dropout(mean_pool)
 h_embedding_q = self.embedding(question_data)
 h_embedding_q = self.embedding_dropout(h_embedding_q)
 h_embedding_a = self.embedding(answer_data)
 h_embedding_a = self.embedding_dropout(h_embedding_a)
 h_embedding_t = self.embedding(title_data)
 h_embedding_t = self.embedding_dropout(h_embedding_t)
 add = torch.cat(( h_embedding_t, h_embedding_q, h_embedding_a), 1)
 add = self.linear_q_add(torch.mean(add, 1))
add = self.linear_q_add1(add)
 x = torch.nn.ELU()(self.linear1(torch.cat([pooled_output,usefeatures,add],1)))
 logits = self.classifier(x)
 outputs =(logits,)+ outputs[2:]
if labels is not None:
if self.num_labels == 1:
loss_fct = MSELoss()
loss = loss_fct(logits.view(-1), labels.view(-1))
else:
loss_fct = CrossEntropyLoss()
loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
outputs =(loss,)+ outputs
return outputs
<define_search_space>
LEN_USE_FEATURES = 1646
class BertForSequenceClassification_v2Question(BertPreTrainedModel):
r
def __init__(self, config,embedding_matrix=None):
super(BertForSequenceClassification_v2Question,self ).__init__(config)
 self.num_labels = config.num_labels
self.embedding = nn.Embedding(*embedding_matrix.shape)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.linear_q_add = nn.Linear(300, 128)
 self.linear_q_add1 = nn.Linear(128, self.config.num_labels)
 self.bert = BertModel(config)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.linear1 = nn.Linear(config.hidden_size+LEN_USE_FEATURES + self.config.num_labels ,config.hidden_size+LEN_USE_FEATURES + self.config.num_labels)
 self.classifier = nn.Linear(config.hidden_size+LEN_USE_FEATURES + self.config.num_labels , self.config.num_labels)
 self.init_weights()
def forward(
 self,
input_ids=None,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
inputs_embeds=None,
labels=None,
usefeatures = None,
question_data = None,
answer_data = None,
title_data = None 
):
outputs = self.bert(
 input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds, 
)
 last_hidden_state = outputs[0][:,0,:].reshape(( -1, 1, 768))
pooled_output = outputs[1]
hidden_states = outputs[2]
h12 = hidden_states[-1][:, 0].reshape(( -1, 1, 768))
h11 = hidden_states[-2][:, 0].reshape(( -1, 1, 768))
h10 = hidden_states[-3][:, 0].reshape(( -1, 1, 768))
h9 = hidden_states[-4][:, 0].reshape(( -1, 1, 768))
all_h = torch.cat([ h9, h10, h11, h12], 1)
 all_h_mean = torch.mean(all_h, 1 ).reshape(( -1,1,768))
mean_pool = torch.mean(torch.cat([last_hidden_state,all_h_mean],1),1)
 pooled_output = self.dropout(mean_pool)
 h_embedding_q = self.embedding(question_data)
 h_embedding_q = self.embedding_dropout(h_embedding_q)
 h_embedding_t = self.embedding(title_data)
 h_embedding_t = self.embedding_dropout(h_embedding_t)
 add = torch.cat(( h_embedding_t, h_embedding_q), 1)
 add = self.linear_q_add(torch.mean(add,1))
add = self.linear_q_add1(add)
 x = torch.nn.ELU()(self.linear1(torch.cat([pooled_output,usefeatures,add],1)))
 logits = self.classifier(x)
 outputs =(logits,)+ outputs[2:]
if labels is not None:
if self.num_labels == 1:
loss_fct = MSELoss()
loss = loss_fct(logits.view(-1), labels.view(-1))
else:
loss_fct = CrossEntropyLoss()
loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
outputs =(loss,)+ outputs
return outputs
train = pd.read_csv(TRAIN_CSV_PATH)
 test = pd.read_csv(TEST_CSV_PATH)
 target_cols_questions = ['question_asker_intent_understanding', 'question_body_critical',
'question_conversational', 'question_expect_short_answer',
'question_fact_seeking', 'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written']
target_cols_answers = ['answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']
def calc_QT_par_kurt(QT_n_quantile_min=10, QT_n_quantile_max=200):
a = np.array([[1.53655,1], [30.4929,1]])
 b = np.array([QT_n_quantile_min, QT_n_quantile_max])
 return np.linalg.solve(a, b )<compute_train_metric>
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)> max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
def n_quantile_for_kurt(kurt, calc_QT_par_kurt_transform):
return int(calc_QT_par_kurt_transform[0]*kurt + calc_QT_par_kurt_transform[1] )<feature_engineering>
def _trim_input_answers(title, question, answer, tokenizer, max_sequence_length=512, t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"%(max_sequence_length,(t_new_len + a_new_len + q_new_len + 4)))
 q_len_head = round(q_new_len/2)
 q_len_tail = -1*(q_new_len -q_len_head)
 a_len_head = round(a_new_len/2)
 a_len_tail = -1*(a_new_len -a_len_head)
 t = t[:t_new_len]
if HEAD_TAIL :
q = q[:q_len_head]+q[q_len_tail:]
a = a[:a_len_head]+a[a_len_tail:]
else:
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs_answers(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays_answers(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows() ,position=0, leave=True):
t, q, a = instance.question_title,instance.question_body, instance.answer
t, q, a = _trim_input_answers(t, q, a, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_answers(t,q, a,tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<set_options>
def _trim_input(title, question, tokenizer, max_sequence_length=512, t_max_len=30, q_max_len=479):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 t_len = len(t)
 q_len = len(q)
 if(t_len+q_len+3)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
q_max_len = q_max_len +(t_max_len - t_len)
 q_new_len = q_max_len
else:
t_new_len = t_max_len
q_new_len = q_max_len
if q_max_len > q_len:
q_new_len = q_len
if t_new_len+q_new_len+3 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"%(max_sequence_length,(t_new_len + q_new_len + 3)))
 t = t[:t_new_len]
q = q[:q_new_len]
return t, q
def _convert_to_bert_inputs(title, question, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows() ,position=0, leave=True):
t, q = instance.question_title, instance.question_body
t, q = _trim_input(t, q, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q,tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<concatenate>
class QuestDataset(torch.utils.data.Dataset):
def __init__(self, inputs, usefeatures,question_data, answer_data, title_data, lengths, labels = None):
self.inputs = inputs
self.usefeatures = usefeatures
self.question_data = question_data
self.answer_data = answer_data
self.title_data = title_data
if labels is not None:
self.labels = labels
else:
self.labels = None
self.lengths = lengths
def __getitem__(self, idx):
input_ids = self.inputs[0][idx]
input_masks = self.inputs[1][idx]
input_segments = self.inputs[2][idx]
lengths = self.lengths[idx]
usef = self.usefeatures[idx,:]
question_data = self.question_data[idx,:]
answer_data = self.answer_data[idx,:]
title_data = self.title_data[idx,:]
if self.labels is not None:
labels = self.labels[idx]
return input_ids, input_masks, input_segments, usef, question_data, answer_data, title_data, labels, lengths
return input_ids, input_masks, input_segments, usef, question_data,answer_data, title_data, lengths
def __len__(self):
return len(self.inputs[0] )
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(TruncatedSVD(n_components=n_comp_GENES, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<create_dataframe>
def predict_result_question(model, test_loader, batch_size=32):
test_preds = np.zeros(( len(test), 21))
model.eval() ;
tk0 = tqdm(enumerate(test_loader),position=0, leave=True)
 for idx, x_batch in tk0:
with torch.no_grad() :
outputs = model(input_ids = x_batch[0].to(device),
labels = None,
attention_mask = x_batch[1].to(device),
token_type_ids = x_batch[2].to(device),
usefeatures = x_batch[3].to(device),
question_data = x_batch[4].to(device),
answer_data = x_batch[5].to(device),
title_data = x_batch[6].to(device)
)
 predictions = outputs[0]
test_preds[idx*batch_size :(idx+1)*batch_size] = predictions.detach().cpu().squeeze().numpy()
output = torch.sigmoid(torch.tensor(test_preds)).numpy()
return output
def predict_result_answer(model, test_loader, batch_size=32):
test_preds = np.zeros(( len(test), 9))
model.eval() ;
tk0 = tqdm(enumerate(test_loader),position=0, leave=True)
 for idx, x_batch in tk0:
with torch.no_grad() :
outputs = model(input_ids = x_batch[0].to(device),
labels = None,
attention_mask = x_batch[1].to(device),
token_type_ids = x_batch[2].to(device),
usefeatures = x_batch[3].to(device),
question_data = x_batch[4].to(device),
answer_data = x_batch[5].to(device),
title_data = x_batch[6].to(device)
)
 predictions = outputs[0]
test_preds[idx*batch_size :(idx+1)*batch_size] = predictions.detach().cpu().squeeze().numpy()
output = torch.sigmoid(torch.tensor(test_preds)).numpy()
return output
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(TruncatedSVD(n_components=n_comp_CELLS, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )<concatenate>
set_seeds(SEED)
 question_tokenizer = BertTokenizer.from_pretrained(".. /input/bert-vocab1/vocab.txt")
 input_categories_q = list(train.columns[[1,2]]);
bert_model_config_question = '.. /input/pretrained-bert-models-for-pytorch/bert-base-uncased/bert_config.json'
bert_config_q = BertConfig.from_json_file(bert_model_config_question)
 bert_config_q.output_hidden_states = True
bert_config_q.num_labels = 21
answer_tokenizer = BertTokenizer.from_pretrained(".. /input/bert-vocab1/vocab.txt")
 input_categories_a = list(train.columns[[1,2,5]]);
bert_model_config_answer = '.. /input/pretrained-bert-models-for-pytorch/bert-base-uncased/bert_config.json'
bert_config_a = BertConfig.from_json_file(bert_model_config_answer)
 bert_config_a.output_hidden_states = True
bert_config_a.num_labels = 9
device = 'cuda' if torch.cuda.is_available() else 'cpu'
test_inputs_q = compute_input_arays(df = test, columns = input_categories_q, tokenizer = question_tokenizer, max_sequence_length=512)
 lengths_test_q = np.argmax(test_inputs_q[0] == 0, axis=1)
 lengths_test_q[lengths_test_q == 0] = test_inputs_q[0].shape[1]
test_inputs_a = compute_input_arays_answers(df = test, columns = input_categories_a, tokenizer = answer_tokenizer, max_sequence_length=512)
 lengths_test_a = np.argmax(test_inputs_a[0] == 0, axis=1)
 lengths_test_a[lengths_test_a == 0] = test_inputs_a[0].shape[1]
data = train_features.append(test_features)
 data<create_dataframe>
NUM_FOLDS = 5
BATCH_SIZE = 8
epochs = 4
test_predictions_q = []
test_predictions_a = []
test_predictions = []
test_set_q = QuestDataset(inputs=test_inputs_q,
usefeatures=torch.FloatTensor(X_test),
question_data = torch.LongTensor(test_question_tokenized),
answer_data = torch.LongTensor(test_answer_tokenized),
title_data = torch.LongTensor(test_title_tokenized),
lengths=lengths_test_q, labels=None)
 test_loader_q = DataLoader(test_set_q, batch_size=32, shuffle=False)
 result_q = np.zeros(( len(test), 21))
test_set_a = QuestDataset(inputs=test_inputs_a,
usefeatures=torch.FloatTensor(X_test),
question_data = torch.LongTensor(test_question_tokenized),
answer_data = torch.LongTensor(test_answer_tokenized),
title_data = torch.LongTensor(test_title_tokenized),
lengths=lengths_test_a, labels=None)
 test_loader_a = DataLoader(test_set_a, batch_size=32, shuffle=False)
 result_a = np.zeros(( len(test), 9))
set_seeds(SEED )
var_thresh = VarianceThreshold(VarianceThreshold_for_FS)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape<merge>
class_labels_dict = {'question_conversational': 2, 'question_expect_short_answer': 3, 'question_fact_seeking':4,
'question_has_commonly_accepted_answer':5,
'question_multi_intent' : 8,
'question_not_really_a_question':9,
'question_opinion_seeking':10,
'question_type_choice' :11,
'question_type_compare':12,
'question_type_consequence': 13,
'question_type_definition':14,
'question_type_entity':15,
'question_type_instructions':16,
'question_type_procedure':17,
'question_type_reason_explanation':18,
'question_type_spelling':19,
'answer_type_instructions':26,
'answer_type_procedure':27,
'answer_type_reason_explanation':28}
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]<drop_column>
BERT_SAVED_MODEL_ANSWERS = ['.. /input/bert-use-crawl2-dataset-answers/',
'.. /input/bert-use-crawl-seed42-answers/',
'.. /input/bert-use-crawl-seed101-answers/',
'.. /input/bert-use-crawl2-dataset-seed1029-answers',
'.. /input/bert-use-crawl2-dataset-seed1977-answers']
BERT_SAVED_MODEL_QUESTIONS = ['.. /input/bert-use-crawl2-dataset-questions/' ,
'.. /input/bert-use-crawl-seed42-questions/',
'.. /input/bert-use-crawl-seed101-questions/',
'.. /input/bert-use-crawl2-dataset-seed1029-questions',
'.. /input/bert-use-crawl2-dataset-seed1977-questions'
]
for qa in range(len(BERT_SAVED_MODEL_ANSWERS)) :
for f in range(NUM_FOLDS):
model_q = BertForSequenceClassification_v2Question.from_pretrained('.. /input/pretrained-bert-models-for-pytorch/bert-base-uncased/', config=bert_config_q , embedding_matrix=embedding_matrix)
 model_q.zero_grad() ;
model_q.to(device);
torch.cuda.empty_cache()
dict_load = torch.load(path_join(BERT_SAVED_MODEL_QUESTIONS[qa], 'Q_model_save_fold_{}.pt'.format(f)))
 dict_load['embedding.weight'] = torch.from_numpy(embedding_matrix)
 model_q.load_state_dict(dict_load)
 out_q = predict_result_question(model_q, test_loader_q)
 model_a = BertForSequenceClassification_v2Answer.from_pretrained('.. /input/pretrained-bert-models-for-pytorch/bert-base-uncased/',config=bert_config_a , embedding_matrix=embedding_matrix)
 model_a.zero_grad() ;
model_a.to(device);
torch.cuda.empty_cache()
dict_load = torch.load(path_join(BERT_SAVED_MODEL_ANSWERS[qa], 'A_model_save_fold_{}.pt'.format(f)))
 dict_load['embedding.weight'] = torch.from_numpy(embedding_matrix)
 model_a.load_state_dict(dict_load)
 out_a = predict_result_answer(model_a, test_loader_a)
 preds = np.hstack(( out_q,out_a))
if POST_PROCESS:
for l in class_labels_dict.keys() :
preds[:,class_labels_dict[l]] = pd.cut(preds[:,class_labels_dict[l]], train[l].nunique() , labels=train[l].value_counts().index.sort_values())
 test_predictions.append(preds)
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )<drop_column>
test_preds_bert = np.mean(test_predictions,axis=0)
 idx = np.argwhere(test.category!='CULTURE' ).flatten()
arr = 0.0001 * np.clip(np.random.normal(size=len(idx)) ,0.1,0.99)
 test_preds_bert[idx,19] = arr
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<split>
ROBERTA_PRETRAINED = '.. /input/roberta-transformers-pytorch/roberta-base'
sys.path.insert(0, data_dir)
 RobertaForSequenceClassification, RobertaConfig, AdamW,
get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP)
 print(transformers.__version__)
 def set_seeds(SEED=42):
os.environ["PYTHONHASHSEED"] = str(SEED)
 np.random.seed(SEED)
 torch.manual_seed(SEED)
 torch.cuda.manual_seed(SEED)
 torch.backends.cudnn.deterministic = True
SEED=2019
set_seeds(SEED)
 HEAD_TAIL = True
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds<count_values>
LEN_USE_FEATURES = 1646
class RobertaClassificationHead(nn.Module):

 def __init__(self, config):
super().__init__()
self.dense = nn.Linear(config.hidden_size, config.hidden_size)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
 def forward(self, features, **kwargs):
x = features[:, 0, :]
x = self.dropout(x)
 x = self.dense(x)
 x = torch.tanh(x)
 x = self.dropout(x)
 x = self.out_proj(x)
 return x
class RobertaForSequenceClassification_v2(BertPreTrainedModel):
config_class = RobertaConfig
pretrained_model_archive_map = ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP
base_model_prefix = "roberta"
def __init__(self, config):
super().__init__(config)
 self.num_labels = config.num_labels
self.roberta = RobertaModel(config)
 self.classifier = RobertaClassificationHead(config)
 def forward(
 self,
input_ids=None,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
inputs_embeds=None,
labels=None,
usefeatures = None 
):
r
outputs = self.roberta(
 input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds, 
)
 sequence_output = outputs[0]
pooled_output = outputs[1]
hidden_states = outputs[2]
logits = self.classifier(sequence_output)
 outputs =(logits,)+ outputs[2:]
if labels is not None:
if self.num_labels == 1:
loss_fct = MSELoss()
loss = loss_fct(logits.view(-1), labels.view(-1))
else:
loss_fct = CrossEntropyLoss()
loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
outputs =(loss,)+ outputs
return outputs
max(train_drugs['sig_id'].value_counts().values)
<merge>
train = pd.read_csv(TRAIN_CSV_PATH)
 test = pd.read_csv(TEST_CSV_PATH)
 target_cols_questions = ['question_asker_intent_understanding', 'question_body_critical',
'question_conversational', 'question_expect_short_answer',
'question_fact_seeking', 'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written']
target_cols_answers = ['answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']
merged_drug = folds.merge(train_drugs)
<count_values>
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)> max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "
":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
merged_drug['drug_id'].value_counts()<groupby>
def _trim_input_answers(title, question, answer, tokenizer, max_sequence_length=512, t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"%(max_sequence_length,(t_new_len + a_new_len + q_new_len + 4)))
 q_len_head = round(q_new_len/2)
 q_len_tail = -1*(q_new_len -q_len_head)
 a_len_head = round(a_new_len/2)
 a_len_tail = -1*(a_new_len -a_len_head)
 t = t[:t_new_len]
if HEAD_TAIL :
q = q[:q_len_head]+q[q_len_tail:]
a = a[:a_len_head]+a[a_len_tail:]
else:
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs_answers(title, question, answer, tokenizer, max_sequence_length):

 stoken = ['
 '] + title + ['
 
 '] + question + ['
 
 '] + answer + ['
']
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays_answers(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows() ,position=0, leave=True):
t, q, a = instance.question_title,instance.question_body, instance.answer
t, q, a = _trim_input_answers(t, q, a, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs_answers(t,q, a,tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]
merge_grouped = merged_drug.groupby(by = 'drug_id')[target_cols].sum() 
<define_variables>
def _trim_input(title, question, tokenizer, max_sequence_length=512, t_max_len=30, q_max_len=479):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 t_len = len(t)
 q_len = len(q)
 if(t_len+q_len+3)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
q_max_len = q_max_len +(t_max_len - t_len)
 q_new_len = q_max_len
else:
t_new_len = t_max_len
q_new_len = q_max_len
if q_max_len > q_len:
q_new_len = q_len
if t_new_len+q_new_len+3 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"%(max_sequence_length,(t_new_len + q_new_len + 3)))
 t = t[:t_new_len]
q = q[:q_new_len]
return t, q
def _convert_to_bert_inputs(title, question, tokenizer, max_sequence_length):

 stoken = ['
'] + title + ['
 
 '] + question + ['
']
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows() ,position=0, leave=True):
t, q = instance.question_title, instance.question_body
t, q = _trim_input(t, q, tokenizer, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q,tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [
torch.from_numpy(np.asarray(input_ids, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_masks, dtype=np.int32)).long() ,
torch.from_numpy(np.asarray(input_segments, dtype=np.int32)).long() ,
]
drug_ids = list(merge_grouped[merge_grouped['sum_tail_enders'] >0].index)
<count_values>
class QuestDataset(torch.utils.data.Dataset):
def __init__(self, inputs, usefeatures, lengths, labels = None):
self.inputs = inputs
self.usefeatures = usefeatures
if labels is not None:
self.labels = labels
else:
self.labels = None
self.lengths = lengths
def __getitem__(self, idx):
input_ids = self.inputs[0][idx]
input_masks = self.inputs[1][idx]
input_segments = self.inputs[2][idx]
lengths = self.lengths[idx]
usef = self.usefeatures[idx,:]
if self.labels is not None:
labels = self.labels[idx]
return input_ids, input_masks, input_segments, usef, labels, lengths
return input_ids, input_masks, input_segments, usef, lengths
def __len__(self):
return len(self.inputs[0] )
x = merged_drug['drug_id'].value_counts() 
<count_values>
def predict_result_question(model, test_loader, batch_size=32):
test_preds = np.zeros(( len(test), 21))
model.eval() ;
tk0 = tqdm(enumerate(test_loader),position=0, leave=True)
 for idx, x_batch in tk0:
with torch.no_grad() :
outputs = model(input_ids = x_batch[0].to(device),
labels = None,
attention_mask = x_batch[1].to(device),
token_type_ids = x_batch[2].to(device),
usefeatures = x_batch[3].to(device)
)
 predictions = outputs[0]
test_preds[idx*batch_size :(idx+1)*batch_size] = predictions.detach().cpu().squeeze().numpy()
output = torch.sigmoid(torch.tensor(test_preds)).numpy()
return output
def predict_result_answer(model, test_loader, batch_size=32):
test_preds = np.zeros(( len(test), 9))
model.eval() ;
tk0 = tqdm(enumerate(test_loader),position=0, leave=True)
 for idx, x_batch in tk0:
with torch.no_grad() :
outputs = model(input_ids = x_batch[0].to(device),
labels = None,
attention_mask = x_batch[1].to(device),
token_type_ids = x_batch[2].to(device),
usefeatures = x_batch[3].to(device)
)
 predictions = outputs[0]
test_preds[idx*batch_size :(idx+1)*batch_size] = predictions.detach().cpu().squeeze().numpy()
output = torch.sigmoid(torch.tensor(test_preds)).numpy()
return output
all_drugs = merged_drug['drug_id'].value_counts()<define_variables>
set_seeds(SEED)
 question_tokenizer = RobertaTokenizer.from_pretrained(ROBERTA_PRETRAINED)
 input_categories_q = list(train.columns[[1,2]]);
robert_config_question = RobertaConfig.from_pretrained(ROBERTA_PRETRAINED)
 robert_config_question.num_labels = 21
robert_config_question.output_hidden_states = True
answer_tokenizer = RobertaTokenizer.from_pretrained(ROBERTA_PRETRAINED)
 input_categories_a = list(train.columns[[1,2,5]]);
robert_config_answer = RobertaConfig.from_pretrained(ROBERTA_PRETRAINED)
 robert_config_answer.num_labels = 9
robert_config_answer.output_hidden_states = True
device = 'cuda' if torch.cuda.is_available() else 'cpu'
samp_strat = {}<init_hyperparams>
test_inputs_q = compute_input_arays(df = test, columns = input_categories_q, tokenizer = question_tokenizer, max_sequence_length=512)
 lengths_test_q = np.argmax(test_inputs_q[0] == 0, axis=1)
 lengths_test_q[lengths_test_q == 0] = test_inputs_q[0].shape[1]
test_inputs_a = compute_input_arays_answers(df = test, columns = input_categories_a, tokenizer = answer_tokenizer, max_sequence_length=512)
 lengths_test_a = np.argmax(test_inputs_a[0] == 0, axis=1)
 lengths_test_a[lengths_test_a == 0] = test_inputs_a[0].shape[1]
samp_strat = {}
for i in list(all_drugs.index):
if i in drug_ids:
samp_strat.update({i:150})
 else:
if(( all_drugs[i]+1)< 7):
samp_strat.update({i:7})
 else:
samp_strat.update({i:all_drugs[i]+1})
<categorify>
NUM_FOLDS = 5
BATCH_SIZE = 8
epochs = 5
test_predictions_q = []
test_predictions_a = []
test_predictions = []
test_set_q = QuestDataset(inputs=test_inputs_q,
usefeatures=torch.FloatTensor(X_test),
lengths=lengths_test_q, labels=None)
 test_loader_q = DataLoader(test_set_q, batch_size=32, shuffle=False)
 result_q = np.zeros(( len(test), 21))
test_set_a = QuestDataset(inputs=test_inputs_a,
usefeatures=torch.FloatTensor(X_test),
lengths=lengths_test_a, labels=None)
 test_loader_a = DataLoader(test_set_a, batch_size=32, shuffle=False)
 result_a = np.zeros(( len(test), 9))
set_seeds(SEED )
drug_smote = pd.get_dummies(merged_drug, columns=['cp_time','cp_dose'] )<prepare_x_and_y>
ROBERTA_SAVED_MODEL_ANSWERS = ['.. /input/roberta-dataset-answers/',
'.. /input/roberta-seed-42-answers/' ,
'.. /input/roberta-dataset-seed101-answers/',
'.. /input/roberta-dataset-seed1029-answers'
]
ROBERTA_SAVED_MODEL_QUESTIONS = ['.. /input/roberta-dataset-questions/',
'.. /input/roberta-seed-42-questions/',
'.. /input/roberta-dataset-seed101-questions/',
'.. /input/roberta-dataset-seed1029-questions']
for qa in range(len(ROBERTA_SAVED_MODEL_ANSWERS)) :
for f in range(NUM_FOLDS):
model_q = RobertaForSequenceClassification_v2.from_pretrained(ROBERTA_PRETRAINED, config=robert_config_question)
 model_q.zero_grad() ;
model_q.to(device);
torch.cuda.empty_cache()
dict_load = torch.load(path_join(ROBERTA_SAVED_MODEL_QUESTIONS[qa], 'Q_model_save_fold_{}.pt'.format(f)))
 model_q.load_state_dict(dict_load)
 out_q = predict_result_question(model_q, test_loader_q)
 model_a = RobertaForSequenceClassification_v2.from_pretrained(ROBERTA_PRETRAINED,config=robert_config_answer)
 model_a.zero_grad() ;
model_a.to(device);
torch.cuda.empty_cache()
dict_load = torch.load(path_join(ROBERTA_SAVED_MODEL_ANSWERS[qa], 'A_model_save_fold_{}.pt'.format(f)))
 model_a.load_state_dict(dict_load)
 out_a = predict_result_answer(model_a, test_loader_a)
 preds = np.hstack(( out_q,out_a))
if POST_PROCESS:
for l in class_labels_dict.keys() :
preds[:,class_labels_dict[l]] = pd.cut(preds[:,class_labels_dict[l]], train[l].nunique() , labels=train[l].value_counts().index.sort_values())
 test_predictions.append(preds)
y_smote = merged_drug['drug_id']
x_smote = drug_smote.drop(target_cols+['sig_id' , 'kfold', 'drug_id'] , axis =1 )<choose_model_class>
test_preds_roberta = np.mean(test_predictions,axis=0)
 idx = np.argwhere(test.category!='CULTURE' ).flatten()
arr = 0.0001 * np.clip(np.random.normal(size=len(idx)) ,0.1,0.99)
 test_preds_roberta[idx,19] = arr
smote = RandomOverSampler(sampling_strategy= samp_strat , random_state=42)
<train_model>
class_labels = ['question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation']
submission = pd.read_csv('.. /input/google-quest-challenge/sample_submission.csv')
 submission.iloc[:,1:] = test_preds_bert * 0.70 + test_preds_roberta*0.30
for l in class_labels:
submission[l] = pd.cut(submission[l], train[l].nunique() , labels=train[l].value_counts().index.sort_values())
 submission.to_csv('submission.csv', index=False)
 submission.head()