Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<choose_model_class>
!pip install.. /input/sacremoses/sacremoses-master/
!pip install.. /input/transformers/transformers-master/
def get_model() :
input_cat = Input(( len(cat_features_hash),))
input_num = Input(( len(num_features),))
input_words = Input(( maxlen,))
x_cat = Embedding(max_size, 10 )(input_cat)
 x_cat = Flatten()(x_cat)
 x_cat = Dropout(0.5 )(x_cat)
 x_words = Embedding(max_features, 100 )(input_words)
 x_words = Convolution1D(100, 3, activation="relu" )(x_words)
 x_words = GlobalMaxPool1D()(x_words)
 x_words = Dropout(0.5 )(x_words)
 x_cat = Dense(100, activation="relu" )(x_cat)
 x_num = Dense(100, activation="relu" )(input_num)
 x_num = Dropout(0.5 )(x_num)
 x = concatenate([x_cat, x_num, x_words])
 x = Dense(50, activation="relu" )(x)
 x = Dropout(0.5 )(x)
 predictions = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[input_cat, input_num, input_words], outputs=predictions)
 model.compile(optimizer=optimizers.Adam(0.001, decay=1e-6),
loss='binary_crossentropy',
metrics=['accuracy'])
 return model<find_best_params>
DATA_DIR = '.. /input/google-quest-challenge'
model = get_model()<predict_on_test>
sub = pd.read_csv(f'{DATA_DIR}/sample_submission.csv')
 sub.head()
pred_test = model.predict([X_test_cat, X_test_num, X_test_words] )<save_to_csv>
train = pd.read_csv(f'{DATA_DIR}/train.csv')
 train.head()
test["project_is_approved"] = pred_test
test[['id', 'project_is_approved']].to_csv("baseline_submission.csv", index=False )<define_variables>
test = pd.read_csv(f'{DATA_DIR}/test.csv')
 test.head()
seed = 72
np.random.seed(seed )<load_from_csv>
MAX_LEN = 512
SEP_TOKEN_ID = 102
class QuestDataset(torch.utils.data.Dataset):
def __init__(self, df, model_type="bert-base-cased", max_len=512, content="Question_Answer", train_mode=True, labeled=True):
self.df = df
self.train_mode = train_mode
self.labeled = labeled
self.max_len = max_len
self.content = content
bert_tokenizer_path = '.. /input/pretrained-bert-models-for-pytorch/' + model_type + '-vocab.txt'
xlnet_tokenizer_path = '.. /input/xlnet-pretrained-models-pytorch/' + model_type + '-spiece.model'
roberta_tokenizer_path = '.. /input/roberta-transformers-pytorch/roberta-base/vocab.json'
roberta_tokenizer_merges_file = '.. /input/roberta-transformers-pytorch/roberta-base/merges.txt'
if model_type == "bert-base-uncased":
self.tokenizer = BertTokenizer.from_pretrained(bert_tokenizer_path)
 elif model_type == "bert-base-cased":
self.tokenizer = BertTokenizer.from_pretrained(bert_tokenizer_path)
 elif model_type == "xlnet-base-cased":
self.tokenizer = XLNetTokenizer.from_pretrained(xlnet_tokenizer_path)
 elif model_type == "roberta-base":
self.tokenizer = RobertaTokenizer(vocab_file=roberta_tokenizer_path, merges_file=roberta_tokenizer_merges_file)
 def __getitem__(self, index):
row = self.df.iloc[index]
token_ids, seg_ids = self.get_token_ids(row)
 if self.labeled:
labels = self.get_label(row)
 return token_ids, seg_ids, labels
else:
return token_ids, seg_ids
def __len__(self):
return len(self.df)
 def select_tokens(self, tokens, max_num):
if len(tokens)<= max_num:
return tokens
if self.train_mode:
num_remove = len(tokens)- max_num
remove_start = random.randint(0, len(tokens)-num_remove-1)
 return tokens[:remove_start] + tokens[remove_start + num_remove:]
else:
return tokens[:max_num//2] + tokens[-(max_num - max_num//2):]
def trim_input_single_content(self, title, content, max_sequence_length=512,
t_max_len=30, c_max_len=512-30-4, num_token=3):
t = self.tokenizer.tokenize(title)
 c = self.tokenizer.tokenize(content)
 t_len = len(t)
 c_len = len(c)
 if(t_len+c_len+num_token)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
c_max_len = c_max_len + floor(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if c_max_len > c_len:
c_new_len = c_len
else:
c_new_len = c_max_len
if t_new_len+c_new_len+num_token > max_sequence_length:
raise ValueError("New sequence length should be less or equal than %d, but is %d"
%(max_sequence_length,(t_new_len+c_new_len+num_token)))
 if len(t)- t_new_len > 0:
t = t[:t_new_len//4] + t[len(t)-t_new_len+t_new_len//4:]
else:
t = t[:t_new_len]
if len(c)- c_new_len > 0:
c = c[:c_new_len//4] + c[len(c)-c_new_len+c_new_len//4:]
else:
c = c[:c_new_len]
if(len(t)+ len(c)+ num_token > max_sequence_length):
more_token = len(t)+ len(c)+ num_token - max_sequence_length
c = c[:(len(c)-more_token)]
return t, c
def trim_input(self, title, question, answer, max_sequence_length=MAX_LEN,
t_max_len=30, q_max_len=239, a_max_len=239, num_token=4):
t = self.tokenizer.tokenize(title)
 q = self.tokenizer.tokenize(question)
 a = self.tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+num_token)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+num_token > max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+num_token)))
 if len(t)- t_new_len > 0:
t = t[:t_new_len//4] + t[len(t)-t_new_len+t_new_len//4:]
else:
t = t[:t_new_len]
if len(q)- q_new_len > 0:
q = q[:q_new_len//4] + q[len(q)-q_new_len+q_new_len//4:]
else:
q = q[:q_new_len]
if len(a)- a_new_len > 0:
a = a[:a_new_len//4] + a[len(a)-a_new_len+a_new_len//4:]
else:
a = a[:a_new_len]
return t, q, a
def get_token_ids(self, row):
num_token = 4
if self.content == "Question":
num_token -= 1
elif self.content == "Answer":
num_token -= 1
if self.content == "Question_Answer":
t_max_len=30
q_max_len=int(( self.max_len-t_max_len-num_token)/2)
 a_max_len=(self.max_len-t_max_len - num_token - int(( self.max_len-t_max_len-num_token)/2))
elif self.content == "Question":
t_max_len=30
q_max_len=self.max_len-t_max_len-num_token
a_max_len=0
elif self.content == "Answer":
t_max_len=30
q_max_len=0
a_max_len=self.max_len-t_max_len-num_token
else:
raise NotImplementedError
if self.content == "Question_Answer":
t_tokens, q_tokens, a_tokens = self.trim_input(row.question_title, row.question_body, row.answer, max_sequence_length=self.max_len, \
t_max_len=t_max_len, q_max_len=q_max_len, a_max_len=a_max_len, num_token=num_token)
 elif self.content == "Question":
t_tokens, c_tokens = self.trim_input_single_content(row.question_title, row.question_body, max_sequence_length=self.max_len, \
t_max_len=t_max_len, c_max_len=q_max_len, num_token=num_token)
 elif self.content == "Answer":
t_tokens, c_tokens = self.trim_input_single_content(row.question_title, row.answer, max_sequence_length=self.max_len, \
t_max_len=t_max_len, c_max_len=a_max_len, num_token=num_token)
 else:
raise NotImplementedError
if self.content == "Question_Answer":
tokens = ['[CLS]'] + t_tokens + ['[SEP]'] + q_tokens + ['[SEP]'] + a_tokens + ['[SEP]']
elif(( self.content == "Question")or(self.content == "Answer")) :
tokens = ['[CLS]'] + t_tokens + ['[SEP]'] + c_tokens + ['[SEP]']
else:
raise NotImplementedError
token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
 if len(token_ids)< self.max_len:
token_ids += [0] *(self.max_len - len(token_ids))
ids = torch.tensor(token_ids)
 seg_ids = self.get_seg_ids(ids)
 return ids, seg_ids
def get_seg_ids(self, ids):
seg_ids = torch.zeros_like(ids)
 seg_idx = 0
first_sep = True
for i, e in enumerate(ids):
seg_ids[i] = seg_idx
if e == self.tokenizer.sep_token_id:
if first_sep:
first_sep = False
else:
seg_idx = 1
pad_idx = torch.nonzero(ids == 0)
 seg_ids[pad_idx] = 0
return seg_ids
def get_label(self, row):
return torch.tensor(row[TARGET_COLUMNS].values.astype(np.float32))
def collate_fn(self, batch):
token_ids = torch.stack([x[0] for x in batch])
 seg_ids = torch.stack([x[1] for x in batch])
 if self.labeled:
labels = torch.stack([x[2] for x in batch])
 return token_ids, seg_ids, labels
else:
return token_ids, seg_ids
def get_test_loader(model_type="bert-base-cased", max_len=512, content="Question_Answer", batch_size=4):
df = pd.read_csv(f'{DATA_DIR}/test.csv')
 ds_test = QuestDataset(df, model_type, max_len=max_len, content=content, train_mode=False, labeled=False)
 loader = torch.utils.data.DataLoader(ds_test, batch_size=batch_size, shuffle=False, num_workers=2, collate_fn=ds_test.collate_fn, drop_last=False)
 loader.num = len(df)
 return loader, ds_test.tokenizer
def get_train_val_loaders(model_type="bert-base-cased", max_len=512, content="Question_Answer", batch_size=4, val_batch_size=4, ifold=0):
df = pd.read_csv(f'{DATA_DIR}/train.csv')
 df = shuffle(df, random_state=42)
 gkf = GroupKFold(n_splits=5 ).split(X=df.question_body, groups=df.question_body)
 for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold == ifold:
df_train = df.iloc[train_idx]
df_val = df.iloc[valid_idx]
break
print(df_train.shape)
 print(df_val.shape)
 ds_train = QuestDataset(df_train, model_type, max_len=max_len, content=content)
 train_loader = torch.utils.data.DataLoader(ds_train, batch_size=batch_size, shuffle=True, num_workers=2, collate_fn=ds_train.collate_fn, drop_last=True)
 train_loader.num = len(df_train)
 ds_val = QuestDataset(df_val, model_type, max_len=max_len, content=content, train_mode=False)
 val_loader = torch.utils.data.DataLoader(ds_val, batch_size=val_batch_size, shuffle=False, num_workers=2, collate_fn=ds_val.collate_fn, drop_last=False)
 val_loader.num = len(df_val)
 val_loader.df = df_val
return train_loader, val_loader, ds_train.tokenizer
def test_train_loader() :
loader, _, _ = get_train_val_loaders("xlnet-base-cased", 512, "Question", 4, 4, 1)
 for ids, seg_ids, labels in loader:
print(ids)
 print(seg_ids.numpy())
 print(labels)
 break
def test_test_loader() :
loader, _ = get_test_loader("roberta-base", 512, "Question", 4)
 for ids, seg_ids in loader:
print(ids)
 print(seg_ids)
 break
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 train_features.shape, train_targets_scored.shape<drop_column>
test_test_loader()
kp_cl = train_features[train_features['cp_type'] != 'ctl_vehicle'].index
train_features = train_features.loc[kp_cl]
train_targets_scored = train_targets_scored.loc[kp_cl]
train_features.drop(['sig_id', 'cp_type'], axis=1, inplace=True)
 train_targets_scored.drop('sig_id', axis=1, inplace=True)
 test_features.drop(['sig_id'], axis=1, inplace=True )<feature_engineering>
test_train_loader()
cols = train_features.columns
cl_g = cols[cols.str.contains('g-')]
cl_c = cols[cols.str.contains('c-')]
def tra_cp_dose(val):
if val == 'D1':
return 1.
 elif val == 'D2':
return 0.
 def tra_cp_time(val):
return val / 24.
 def process1() :
train = train_features.copy()
test = test_features.drop('cp_type', axis=1)
 train['cp_dose'] = train['cp_dose'].apply(tra_cp_dose)
 train['cp_time'] = train['cp_time'].apply(tra_cp_time)
 test['cp_dose'] = test['cp_dose'].apply(tra_cp_dose)
 test['cp_time'] = test['cp_time'].apply(tra_cp_time)
 return train, test
def process2(n_c=60, n_g=480, seed=seed, scale=True):
pca_c = PCA(n_components=n_c, random_state=seed)
 pca_g = PCA(n_components=n_g, random_state=seed)
 SS = StandardScaler()
train_pca_g = pd.DataFrame(pca_g.fit_transform(train_features[cl_g]),
columns=[f'g-pca{i}' for i in range(n_g)],
index=train_features.index)
 train_pca_c = pd.DataFrame(pca_c.fit_transform(train_features[cl_c]),
columns=[f'c-pca{i}' for i in range(n_c)],
index=train_features.index)
 train = pd.concat([train_pca_g, train_pca_c], axis=1)
 train['g-mean'] = np.mean(train_features[cl_g], axis=1)
 train['g-std'] = np.std(train_features[cl_g], axis=1)
 train['c-mean'] = np.mean(train_features[cl_c], axis=1)
 train['c-std'] = np.std(train_features[cl_c], axis=1)
 test_pca_g = pd.DataFrame(pca_g.transform(test_features[cl_g]),
columns=[f'g-pca{i}' for i in range(n_g)],
index=test_features.index)
 test_pca_c = pd.DataFrame(pca_c.fit_transform(test_features[cl_c]),
columns=[f'c-pca{i}' for i in range(n_c)],
index=test_features.index)
 test = pd.concat([test_pca_g, test_pca_c], axis=1)
 test['g-mean'] = np.mean(test_features[cl_g], axis=1)
 test['g-std'] = np.std(test_features[cl_g], axis=1)
 test['c-mean'] = np.mean(test_features[cl_c], axis=1)
 test['c-std'] = np.std(test_features[cl_c], axis=1)
 if scale:
train = pd.DataFrame(SS.fit_transform(train), columns=train.columns, index=train.index)
 test = pd.DataFrame(SS.transform(test), columns=test.columns, index=test.index)
 return train, test
def process3(n_c=80, n_g=660, tr=.8):
trn1, tst1 = process1()
trn2 ,tst2 = process2(n_c, n_g, scale=False)
 train = pd.concat([trn1, trn2], axis=1)
 test = pd.concat([tst1, tst2], axis=1)
 QS = QuantileTransformer()
VT = VarianceThreshold(tr)
 train = pd.DataFrame(QS.fit_transform(VT.fit_transform(train)) , index=train.index)
 test = pd.DataFrame(QS.transform(VT.transform(test)) , index=test.index)
 return train, test 
<randomize_order>
class QuestModel(nn.Module):
def __init__(self, model_type="xlnet-base-cased", tokenizer=None, n_classes=30, hidden_layers=[-1, -3, -5, -7, -9]):
super(QuestModel, self ).__init__()
self.model_name = 'QuestModel'
self.model_type = model_type
self.hidden_layers = hidden_layers
if model_type == "bert-base-uncased":
bert_model_config = '.. /input/pretrained-bert-models-for-pytorch/bert-base-uncased/bert_config.json'
bert_config = BertConfig.from_json_file(bert_model_config)
 bert_config.output_hidden_states = True
if n_classes == 30:
bert_config.hidden_dropout_prob = 0.1
else:
bert_config.hidden_dropout_prob = 0
model_path = os.path.join('.. /input/pretrained-bert-models-for-pytorch/' + model_type)
 self.bert_model = BertModel.from_pretrained(model_path, config=bert_config)
 elif model_type == "bert-base-cased":
bert_model_config = '.. /input/pretrained-bert-models-for-pytorch/bert-base-cased/bert_config.json'
bert_config = BertConfig.from_json_file(bert_model_config)
 bert_config.output_hidden_states = True
model_path = os.path.join('.. /input/pretrained-bert-models-for-pytorch/' + model_type)
 self.bert_model = BertModel.from_pretrained(model_path, config=bert_config)
 elif model_type == "xlnet-base-cased":
xlnet_model_config = '.. /input/xlnet-pretrained-models-pytorch/xlnet-base-cased-config.json'
xlnet_config = XLNetConfig.from_json_file(xlnet_model_config)
 xlnet_config.output_hidden_states = True
xlnet_config.hidden_dropout_prob = 0
model_path = os.path.join('.. /input/xlnet-pretrained-models-pytorch/' + model_type + '-pytorch_model.bin')
 self.xlnet_model = XLNetModel.from_pretrained(model_path, config=xlnet_config)
 elif model_type == "xlnet-large-cased":
xlnet_model_config = '.. /input/xlnet-pretrained-models-pytorch/xlnet-large-cased-config.json'
xlnet_config = XLNetConfig.from_json_file(xlnet_model_config)
 xlnet_config.output_hidden_states = True
xlnet_config.hidden_dropout_prob = 0
model_path = os.path.join('.. /input/xlnet-pretrained-models-pytorch/' + model_type + '-pytorch_model.bin')
 self.xlnet_model = XLNetModel.from_pretrained(model_path, config=xlnet_config)
 elif model_type == "roberta-base":
roberta_model_config = '.. /input/roberta-transformers-pytorch/roberta-base/config.json'
roberta_config = RobertaConfig.from_json_file(roberta_model_config)
 roberta_config.output_hidden_states = True
roberta_config.hidden_dropout_prob = 0
model_path = os.path.join('.. /input/roberta-transformers-pytorch/roberta-base/pytorch_model.bin')
 self.roberta_model = RobertaModel.from_pretrained(model_path, config=roberta_config)
 self.roberta_model.resize_token_embeddings(len(tokenizer))
if model_type == "bert-base-uncased":
self.hidden_size = 768
elif model_type == "bert-large-uncased":
self.hidden_size = 1024
elif model_type == "bert-base-cased":
self.hidden_size = 768
elif model_type == "xlnet-base-cased":
self.hidden_size = 768
elif model_type == "xlnet-large-cased":
self.hidden_size = 1024
elif model_type == "roberta-base":
self.hidden_size = 768
else:
raise NotImplementedError
self.fc_1 = nn.Linear(self.hidden_size * len(hidden_layers), self.hidden_size)
 self.fc = nn.Linear(self.hidden_size, n_classes)
 self.selu = nn.SELU()
self.relu = nn.ReLU()
self.tanh = nn.Tanh()
self.dropouts = nn.ModuleList([
nn.Dropout(0.5)for _ in range(5)
 ])
 def forward(self, ids, seg_ids):
attention_mask =(ids > 0)
 if(( self.model_type == "bert-base-uncased")\
or(self.model_type == "bert-base-cased")\
or(self.model_type == "bert-large-uncased")\
or(self.model_type == "bert-large-cased")) :
outputs = self.bert_model(input_ids=ids, token_type_ids=seg_ids, attention_mask=attention_mask)
 hidden_states = outputs[2]
for i in range(len(self.hidden_layers)) :
if i == 0:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
fuse_hidden = torch.unsqueeze(hidden_state, dim=-1)
 else:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
h = torch.unsqueeze(hidden_state, dim=-1)
 fuse_hidden = torch.cat([fuse_hidden, h], dim=-1)
 fuse_hidden = fuse_hidden.reshape(fuse_hidden.shape[0], -1)
 h = self.relu(self.fc_1(fuse_hidden))
elif(( self.model_type == "xlnet-base-cased")\
or(self.model_type == "xlnet-large-cased")) :
attention_mask = attention_mask.float()
outputs = self.xlnet_model(input_ids=ids, token_type_ids=seg_ids, attention_mask=attention_mask)
 hidden_states = outputs[1]
for i in range(len(self.hidden_layers)) :
if i == 0:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
fuse_hidden = torch.unsqueeze(hidden_state, dim=-1)
 else:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
h = torch.unsqueeze(hidden_state, dim=-1)
 fuse_hidden = torch.cat([fuse_hidden, h], dim=-1)
 fuse_hidden = fuse_hidden.reshape(fuse_hidden.shape[0], -1)
 h = self.relu(self.fc_1(fuse_hidden))
elif(self.model_type == "roberta-base"):
attention_mask = attention_mask.float()
outputs = self.roberta_model(input_ids=ids, token_type_ids=seg_ids, attention_mask=attention_mask)
 hidden_states = outputs[2]
for i in range(len(self.hidden_layers)) :
if i == 0:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
fuse_hidden = torch.unsqueeze(hidden_state, dim=-1)
 else:
hidden_layer = self.hidden_layers[i]
hidden_state = hidden_states[hidden_layer][:, 0]
h = torch.unsqueeze(hidden_state, dim=-1)
 fuse_hidden = torch.cat([fuse_hidden, h], dim=-1)
 fuse_hidden = fuse_hidden.reshape(fuse_hidden.shape[0], -1)
 h = self.relu(self.fc_1(fuse_hidden))
for j, dropout in enumerate(self.dropouts):
if j == 0:
logit = self.fc(dropout(h))
else:
logit += self.fc(dropout(h))
return logit / len(self.dropouts)
 def test_model(model_type="bert-base-cased", hidden_layers=[-1, -3, -5, -7, -9]):
x = torch.tensor([[1,2,3,4,5, 0, 0], [1,2,3,4,5, 0, 0]])
 seg_ids = torch.tensor([[0,0,0,0,0, 0, 0], [0,0,0,0,0, 0, 0]])
 model = QuestModel(model_type=model_type, hidden_layers=hidden_layers)
 y = model(x, seg_ids)
 print(y )
per = np.random.permutation(train_features.index)
 target = train_targets_scored.loc[per]
train1, test1 = process1()
train1 = train1.loc[per]
train1<feature_engineering>
test_model(model_type="bert-base-cased", hidden_layers=[-3, -4, -5, -6, -7] )
train2, test2 = process2()
train2 = train2.loc[per]
train2<feature_engineering>
def create_bert_base_uncased_models() :
models = []
for i in range(10):
model = QuestModel(model_type="bert-base-uncased", hidden_layers=[-1, -3, -5, -7, -9])
 model.load_state_dict(torch.load(f'.. /input/qabertuncasedaugdiffv2swa/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_cased_models() :
models = []
for i in range(10):
model = QuestModel(model_type="bert-base-cased", hidden_layers=[-1, -3, -5, -7, -9])
 model.load_state_dict(torch.load(f'.. /input/qabertbasecasedaugdiffv2swa/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_xlnet_base_cased_models() :
models = []
for i in range(5):
model = QuestModel(model_type="xlnet-base-cased", hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qaxlnetbasecasedaugquestionanswerswa/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_xlnet_base_cased_question_models() :
models = []
for i in range(5):
model = QuestModel(model_type="xlnet-base-cased", n_classes=21, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qaxlnetbasecasedaugdiffswaquestion/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_xlnet_base_cased_answer_models() :
models = []
for i in range(5):
model = QuestModel(model_type="xlnet-base-cased", n_classes=9, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qaxlnetbasecasedaugdiffswaanswer/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_uncased_question_models() :
models = []
for i in range(5):
model = QuestModel(model_type="bert-base-uncased", n_classes=21, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qabertbaseuncasedaugdiffswaquestion/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_uncased_answer_models() :
models = []
for i in range(5):
model = QuestModel(model_type="bert-base-uncased", n_classes=9, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qabertbaseuncasedaugdiffswaanswer/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_cased_question_models() :
models = []
for i in range(5):
model = QuestModel(model_type="bert-base-cased", n_classes=21, hidden_layers=[-2, -4, -6, -8, -10])
 model.load_state_dict(torch.load(f'.. /input/qabertbasecasedaugdiffswaquestion/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_bert_base_cased_answer_models() :
models = []
for i in range(5):
model = QuestModel(model_type="bert-base-cased", n_classes=9, hidden_layers=[-2, -4, -6, -8, -10])
 model.load_state_dict(torch.load(f'.. /input/qabertbasecasedaugdiffswaanswer/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_roberta_base_models(tokenizer):
models = []
for i in range(5):
model = QuestModel(model_type="roberta-base", tokenizer=tokenizer, n_classes=30, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qarobertabaseaugdiffswa/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_roberta_base_question_models(tokenizer):
models = []
for i in range(5):
model = QuestModel(model_type="roberta-base", tokenizer=tokenizer, n_classes=21, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qarobertabasecasedaugdiffswaquestion/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
def create_roberta_base_answer_models(tokenizer):
models = []
for i in range(5):
model = QuestModel(model_type="roberta-base", tokenizer=tokenizer, n_classes=9, hidden_layers=[-3, -4, -5, -6, -7])
 model.load_state_dict(torch.load(f'.. /input/qarobertabasecasedaugdiffswaanswer/fold_{i}_checkpoint_swa.pth'))
model.eval()
models.append(model)
 return models
train3, test3 = process3()
train3 = train3.loc[per]
train3<import_modules>
def predict(models, test_loader):
all_scores = []
with torch.no_grad() :
for ids, seg_ids in tqdm(test_loader, total=test_loader.num // test_loader.batch_size):
ids, seg_ids = ids.cuda() , seg_ids.cuda()
scores = []
for model in models:
model = model.cuda()
outputs = torch.sigmoid(model(ids, seg_ids)).cpu()
scores.append(outputs)
 all_scores.append(torch.mean(torch.stack(scores), 0))
all_scores = torch.cat(all_scores, 0 ).numpy()
return all_scores
import tensorflow as tf
from tensorflow.keras.models import Model
from tensorflow.keras.layers import *
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping
from tensorflow.keras.metrics import binary_crossentropy<compute_test_metric>
test_loader, tokenizer = get_test_loader(model_type="roberta-base", content="Question_Answer", batch_size=32 )
p_min = 0.001
p_max = 0.999
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return binary_crossentropy(y_true, y_pred)
 max_epoch = 50
lr_decay = ReduceLROnPlateau(monitor='val_logloss', patience=1, verbose=1, min_lr=1e-8)
 early_stop = EarlyStopping(monitor='val_logloss', patience=5, verbose=1, restore_best_weights=True )<define_search_model>
roberta_base_models = create_roberta_base_models(tokenizer)
 roberta_base_preds = predict(roberta_base_models, test_loader )
def build_model1(inp_shape1, inp_shape2, tar_shape):
inp1 = Input(( inp_shape1,), name='inp1')
 inp2 = Input(( inp_shape2,), name='inp2')
 X1 = BatchNormalization()(inp1)
 X1 = Dropout (.2 )(X1)
 X1 = Dense(1024 )(X1)
 X1 = BatchNormalization()(X1)
 X1 = LeakyReLU()(X1)
 X1 = Dense(512 )(X1)
 X1 = BatchNormalization()(X1)
 X1 = Activation('relu' )(X1)
 X2 = Dropout (.3 )(inp2)
 X2 = Dense(1024 )(X2)
 X2 = BatchNormalization()(X2)
 X2 = LeakyReLU()(X2)
 X2 = Dense(512 )(X2)
 X2 = BatchNormalization()(X2)
 X2 = LeakyReLU()(X2)
 inp3 = Concatenate(name='concat' )([X1, X2])
 X3 = Dropout (.3 )(inp3)
 X3 = Dense(2048 )(X3)
 X3 = BatchNormalization()(X3)
 X3 = Activation('relu' )(X3)
 X3 = Dense(512 )(X3)
 X3 = BatchNormalization()(X3)
 X3 = Activation('relu' )(X3)
 X3 = Dropout (.4 )(X3)
 X3 = Dense(inp_shape1 )(X3)
 X3 = BatchNormalization()(X3)
 X3 = Activation('relu' )(X3)
 inp4 = Add(name='add' )([X3, inp1])
 out = Dropout (.3 )(inp4)
 out = Dense(1024 )(out)
 out = BatchNormalization()(out)
 out = LeakyReLU()(out)
 out = Dropout (.4 )(out)
 out = Dense(512 )(out)
 out = BatchNormalization()(out)
 out = LeakyReLU()(out)
 out = Dropout (.5 )(out)
 out = Dense(tar_shape )(out)
 out = Activation('sigmoid' )(out)
 model = Model(inputs=[inp1, inp2], outputs=out, name='model1')
 return model
def build_model2(inp_shape, tar_shape):
inp = Input(( inp_shape,), name='inp')
 out = BatchNormalization()(inp)
 out = Dropout (.3 )(out)
 out = Dense(1024 )(out)
 out = BatchNormalization()(out)
 out = Activation('elu' )(out)
 out = Dropout (.3 )(out)
 out = Dense(512 )(out)
 out = BatchNormalization()(out)
 out = Activation('elu' )(out)
 out = Dropout (.3 )(out)
 out = Dense(256 )(out)
 out = BatchNormalization()(out)
 out = Activation('elu' )(out)
 out = Dropout (.3 )(out)
 out = Dense(tar_shape )(out)
 out = BatchNormalization()(out)
 out = Activation('sigmoid' )(out)
 model = Model(inputs=inp, outputs=out, name='model2')
 return model
def build_model3(inp_shape, tar_shape):
inp = Input(( inp_shape,), name='inp')
 out = BatchNormalization()(inp)
 out = Dropout (.3 )(out)
 out = Dense(2048 )(out)
 out = BatchNormalization()(out)
 out = Activation('elu' )(out)
 out = Dropout (.5 )(out)
 out = Dense(256 )(out)
 out = BatchNormalization()(out)
 out = Activation('elu' )(out)
 out = Dropout (.3 )(out)
 out = Dense(1024 )(out)
 out = BatchNormalization()(out)
 out = Activation('elu' )(out)
 out = Dropout (.5 )(out)
 out = Dense(tar_shape )(out)
 out = BatchNormalization()(out)
 out = Activation('sigmoid' )(out)
 model = Model(inputs=inp, outputs=out, name='model3')
 return model<train_model>
del roberta_base_models, test_loader
torch.cuda.empty_cache()
gc.collect()
his1 = model1.fit([train1, train2], target, batch_size=64, epochs=max_epoch, validation_split=.2,
shuffle=False, callbacks=[lr_decay, early_stop], verbose=2 )<choose_model_class>
test_loader, _ = get_test_loader(model_type="xlnet-base-cased", batch_size=32 )
model2 = build_model2(train3.shape[1], target.shape[1])
 model2.compile(optimizer=Adam(1e-1), loss='binary_crossentropy', metrics=[logloss])
 model2.summary()<train_model>
xlnet_base_cased_models = create_xlnet_base_cased_models()
xlnet_base_cased_preds = predict(xlnet_base_cased_models, test_loader )
his2 = model2.fit(train3, target, batch_size=64, epochs=max_epoch, validation_split=.2,
shuffle=False, callbacks=[lr_decay, early_stop], verbose=2 )<choose_model_class>
del xlnet_base_cased_models, test_loader
torch.cuda.empty_cache()
gc.collect()
model3 = build_model3(train3.shape[1], target.shape[1])
 model3.compile(optimizer=Adam (.1), loss='binary_crossentropy', metrics=[logloss])
 model3.summary()<train_model>
test_loader, _ = get_test_loader(model_type="xlnet-base-cased", content="Question", batch_size=32 )
his3 = model3.fit(train3, target, batch_size=128, epochs=max_epoch, validation_split=.2,
shuffle=False, callbacks=[lr_decay, early_stop], verbose=2 )<predict_on_test>
xlnet_base_cased_question_models = create_xlnet_base_cased_question_models()
xlnet_base_cased_question_preds = predict(xlnet_base_cased_question_models, test_loader )
pred1 = model1.predict([test1, test2])
 pred2 = model2.predict(test3)
 pred3 = model3.predict(test3)
 pred =(pred1 + pred2 + pred3)/ 3.0
pred.shape<load_from_csv>
del xlnet_base_cased_question_models, test_loader
torch.cuda.empty_cache()
gc.collect()
out = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 out.iloc[:,1:] = np.clip(pred, p_min, p_max)
 out.iloc[test_features['cp_type'] == 'ctl_vehicle', 1:] = 0
out<save_to_csv>
test_loader, _ = get_test_loader(model_type="xlnet-base-cased", content="Answer", batch_size=32 )
out.to_csv('submission.csv', index=False )<set_options>
xlnet_base_cased_answer_models = create_xlnet_base_cased_answer_models()
xlnet_base_cased_answer_preds = predict(xlnet_base_cased_answer_models, test_loader )
warnings.filterwarnings("ignore" )<install_modules>
del xlnet_base_cased_answer_models, test_loader
torch.cuda.empty_cache()
gc.collect()
!pip install -q /kaggle/input/iterative-stratification/iterative-stratification-master/<import_modules>
xlnet_base_cased_question_answer_preds = np.concatenate([xlnet_base_cased_question_preds, xlnet_base_cased_answer_preds], axis=1 )
from fastai.basics import *<define_variables>
test_loader, tokenizer = get_test_loader(model_type="roberta-base", content="Question", batch_size=32 )
datapath = Path("/kaggle/input/lish-moa" )<load_from_csv>
roberta_base_question_models = create_roberta_base_question_models(tokenizer)
 roberta_base_question_preds = predict(roberta_base_question_models, test_loader )
train_features_df = pd.read_csv(datapath/'train_features.csv')
 test_features_df = pd.read_csv(datapath/'test_features.csv')
 train_targets_scored_df = pd.read_csv(datapath/'train_targets_scored.csv')
 train_targets_nonscored_df = pd.read_csv(datapath/'train_targets_nonscored.csv' )<define_variables>
del roberta_base_question_models, test_loader, tokenizer
torch.cuda.empty_cache()
gc.collect()
[(k,train_target_distrib[k])for k in train_target_distrib if train_target_distrib[k] == 1]<define_variables>
test_loader, tokenizer = get_test_loader(model_type="roberta-base", content="Answer", batch_size=32 )
n_train, n_test = train_features_df.shape[0], test_features_df.shape[0]
train_test_features_df = pd.concat([train_features_df, test_features_df] ).reset_index(drop=True)
 g_cols = list(o for o in train_test_features_df.columns if o.startswith("g-"))
c_cols = list(o for o in train_test_features_df.columns if o.startswith("c-"))
gc_cols = g_cols + c_cols
len(g_cols), len(c_cols), len(gc_cols )<import_modules>
roberta_base_answer_models = create_roberta_base_answer_models(tokenizer)
 roberta_base_answer_preds = predict(roberta_base_answer_models, test_loader )
from seaborn import distplot<init_hyperparams>
del roberta_base_answer_models, test_loader, tokenizer
torch.cuda.empty_cache()
gc.collect()
ParamConfig = SimpleNamespace(
 do_rank_gauss = True,
do_quantile_tfms = False,
n_quantiles = 100,
do_pca = False,
pca_reduction_factor = 3,
do_umap = False,
umap_reduction_factor = 3,
umap_n_neighbors = 150,
do_knn_encoding = True,
knn_k = 100,
do_transfer_learning = False,
tl_n_epochs = 30,
tl_smoothing = 0.001,
tl_lr = 0.001,
ft_n_epochs = 30,
ft_smoothing = 0.001,
ft_lr = 0.001,
model_n_layers = 2,
model_layer_width = 512,
model_ps = 0.25
)<import_modules>
roberta_base_question_answer_preds = np.concatenate([roberta_base_question_preds, roberta_base_answer_preds], axis=1 )
import cupy as cp
from cupyx.scipy.special import erfinv
import cudf as gd
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from scipy.special import erfinv as sp_erfinv<statistical_test>
test_loader, _ = get_test_loader(model_type="bert-base-cased", content="Question", batch_size=32 )
def to_rankgauss(x):
"https://medium.com/rapids-ai/gauss-rank-transformation-is-100x-faster-with-rapids-and-cupy-7c947e3397da"
x_cpu = x
r_cpu = x_cpu.argsort().argsort()
epsilon = 1e-6
r_cpu =(r_cpu/r_cpu.max() -0.5)*2
r_cpu = cp.clip(r_cpu,-1+epsilon,1-epsilon)
 r_cpu = sp_erfinv(r_cpu)
 return r_cpu<categorify>
bert_base_cased_question_models = create_bert_base_cased_question_models()
bert_base_cased_question_preds = predict(bert_base_cased_question_models, test_loader )
if ParamConfig.do_rank_gauss:
for col in gc_cols:
train_test_features_df[col] = to_rankgauss(train_test_features_df[col] )<import_modules>
del bert_base_cased_question_models, test_loader
torch.cuda.empty_cache()
gc.collect()
from sklearn.preprocessing import QuantileTransformer<feature_engineering>
test_loader, _ = get_test_loader(model_type="bert-base-cased", content="Answer", batch_size=32 )
if ParamConfig.do_quantile_tfms:
transformer = QuantileTransformer(n_quantiles=ParamConfig.n_quantiles, random_state=0, output_distribution="normal")
 train_test_features_df[gc_cols] = transformer.fit_transform(train_test_features_df[gc_cols] )<groupby>
bert_base_cased_answer_models = create_bert_base_cased_answer_models()
bert_base_cased_answer_preds = predict(bert_base_cased_answer_models, test_loader )
train_test_features_df.groupby(['cp_type', 'cp_time', 'cp_dose'])[['sig_id']].count()<count_unique_values>
del bert_base_cased_answer_models, test_loader
torch.cuda.empty_cache()
gc.collect()
len(train_test_features_df['sig_id'].unique() )<merge>
bert_base_cased_question_answer_preds = np.concatenate([bert_base_cased_question_preds, bert_base_cased_answer_preds], axis=1 )
all(train_test_features_df.merge(train_targets_scored_df, on='sig_id' ).query("cp_type == 'ctl_vehicle'")[scored_cols].sum() ==0 )<merge>
test_loader, _ = get_test_loader(model_type="bert-base-uncased", content="Question", batch_size=32 )
all(train_test_features_df.merge(train_targets_nonscored_df, on='sig_id' ).query("cp_type == 'ctl_vehicle'")[nonscored_cols].sum() ==0 )<import_modules>
bert_base_uncased_question_models = create_bert_base_uncased_question_models()
bert_base_uncased_question_preds = predict(bert_base_uncased_question_models, test_loader )
from cuml.decomposition import PCA as cumlPCA<concatenate>
del bert_base_uncased_question_models, test_loader
torch.cuda.empty_cache()
gc.collect()
%%time
if ParamConfig.do_pca:
pca_feats = []
colnames = []
for name, cols in [("gene", g_cols),("cell", c_cols)]:
n_comp = int(len(cols)/ParamConfig.pca_reduction_factor)
 pca = cumlPCA(n_components=n_comp, iterated_power=500)
 pca_feat = pca.fit_transform(train_test_features_df[cols])
 pca_feats += [pca_feat]
colnames += [f"pca_{name}_{i}" for i in range(n_comp)]
pca_feats = np.hstack(pca_feats)
 train_test_features_df[colnames] = pca_feats<import_modules>
test_loader, _ = get_test_loader(model_type="bert-base-uncased", content="Answer", batch_size=32 )
from cuml.manifold import UMAP as cumlUMAP<categorify>
bert_base_uncased_answer_models = create_bert_base_uncased_answer_models()
bert_base_uncased_answer_preds = predict(bert_base_uncased_answer_models, test_loader )
%%time
if ParamConfig.do_umap:
umap_feats = []
colnames = []
for name, cols in [("gene", g_cols),("cell", c_cols)]:
n_comp = int(len(cols)/ParamConfig.umap_reduction_factor)
 umap = cumlUMAP(n_components=n_comp, n_neighbors=ParamConfig.umap_n_neighbors)
 umap_feat = umap.fit_transform(train_test_features_df[cols])
 umap_feats += [umap_feat]
colnames += [f"umap_{name}_{i}" for i in range(n_comp)]
umap_feats = np.hstack(umap_feats)
 train_test_features_df[colnames] = umap_feats<categorify>
del bert_base_uncased_answer_models, test_loader
torch.cuda.empty_cache()
gc.collect()
for ohe_col in ['cp_type', 'cp_time', 'cp_dose']:
vals = train_test_features_df[ohe_col].unique()
if len(vals)== 2:
col = ohe_col+"s"
cat2code = {v:k for k,v in enumerate(vals)}
train_test_features_df[col] = train_test_features_df[ohe_col].map(cat2code)
 else:
cat2code = {v:k for k,v in enumerate(vals)}
encoded = train_test_features_df[ohe_col].map(cat2code)
 ohe_arr = np.zeros(( len(vals), len(vals)))
 ohe_arr[np.diag_indices_from(ohe_arr)] = 1
col = [f"{ohe_col}s_{o}" for o in vals]
train_test_features_df[col] = ohe_arr[encoded]<drop_column>
bert_base_uncased_question_answer_preds = np.concatenate([bert_base_uncased_question_preds, bert_base_uncased_answer_preds], axis=1 )
train_test_features_df = train_test_features_df.drop("cp_types", 1 )<import_modules>
test_loader, _ = get_test_loader(model_type="bert-base-cased", batch_size=32 )
from cuml.neighbors import KNeighborsRegressor as cumlKNN<merge>
bert_base_cased_models = create_bert_base_cased_models()
bert_base_cased_preds = predict(bert_base_cased_models, test_loader )
if ParamConfig.do_knn_encoding:
knn = cumlKNN(n_neighbors=ParamConfig.knn_k)
 merged_nonscored_train_test_df = train_test_features_df.merge(train_targets_nonscored_df, on='sig_id', how='left')
 unique_cp_time, unique_cp_dose = merged_nonscored_train_test_df['cp_time'].unique() , merged_nonscored_train_test_df['cp_dose'].unique()<categorify>
del bert_base_cased_models, test_loader
torch.cuda.empty_cache()
gc.collect()
if ParamConfig.do_knn_encoding:
knn_groups = []
for i in unique_cp_time:
for j in unique_cp_dose: knn_groups.append(( i,j))
knn_cols = [f"knn{ParamConfig.knn_k}_{i}" for i in range(len(nonscored_cols)) ]
for c in knn_cols: train_test_features_df[c] = 0<categorify>
test_loader, _ = get_test_loader(model_type="bert-base-uncased", batch_size=32 )
if ParamConfig.do_knn_encoding:
for time, dose in knn_groups:
X = merged_nonscored_train_test_df.query(f"cp_time == '{time}' & cp_dose == '{dose}'")[gc_cols]
y = merged_nonscored_train_test_df.query(f"cp_time == '{time}' & cp_dose == '{dose}'")[nonscored_cols]
idxs = array(list(X.index))
train_idxs = idxs[np.where(( y.isna().sum(1)== 0)) [0]]
test_idxs = idxs[np.where(( y.isna().sum(1)!= 0)) [0]]
X_train, y_train, X_test, y_test = X.loc[train_idxs], y.loc[train_idxs], X.loc[test_idxs], y.loc[test_idxs]
knn.fit(X_train, y_train)
 train_preds, test_preds = knn.predict(X_train), knn.predict(X_test)
 train_test_features_df.loc[train_idxs, knn_cols] = train_preds
train_test_features_df.loc[test_idxs, knn_cols] = test_preds<count_missing_values>
bert_base_uncased_models = create_bert_base_uncased_models()
bert_base_uncased_preds = predict(bert_base_uncased_models, test_loader )
train_features_df = train_test_features_df[:n_train].reset_index(drop=True)
 test_features_df = train_test_features_df[n_train:].reset_index(drop=True)
 assert np.sum(train_features_df.isna() ).sum() == 0
assert np.sum(test_features_df.isna() ).sum() == 0<import_modules>
del bert_base_uncased_models, test_loader
torch.cuda.empty_cache()
gc.collect()
from fastai.tabular.all import *<compute_train_metric>
preds = 0.25 *(0.5 * bert_base_uncased_preds + 0.5 * bert_base_uncased_question_answer_preds)\
+ 0.25 *(0.5 * bert_base_cased_preds + 0.5 * bert_base_cased_question_answer_preds)\
+ 0.25 *(0.5 * xlnet_base_cased_preds + 0.5 * xlnet_base_cased_question_answer_preds)\
+ 0.25 *(0.5 * roberta_base_preds + 0.5 * roberta_base_question_answer_preds)
@log_args
class LabelSmoothingBCEWithLogits(Module):
y_int = True
def __init__(self, eps:float=0.1): store_attr('eps')
 def forward(self, output, target):
target = target + self.eps*(1-2*target)
 return F.binary_cross_entropy_with_logits(output, target)
 def decodes(self, x): return x>self.thresh
def activation(self, x): return torch.sigmoid(x )<compute_train_metric>
sub[TARGET_COLUMNS] = bert_base_uncased_preds
sub.to_csv('submission_bert_base_uncased.csv', index=False)
 sub[TARGET_COLUMNS] = bert_base_cased_preds
sub.to_csv('submission_bert_base_cased.csv', index=False)
 sub[TARGET_COLUMNS] = xlnet_base_cased_preds
sub.to_csv('submission_xlnet_base_cased.csv', index=False)
def clipped_bce(inp, targ): return F.binary_cross_entropy(torch.clamp(inp.sigmoid() ,1e-15, 1-1e-15), targ.float())
 metric = AvgMetric(clipped_bce )<prepare_x_and_y>
sub[TARGET_COLUMNS] = preds
cat_names = []
cont_names = L(list(train_features_df.columns[4:]))
y_names = nonscored_cols<merge>
test = pd.read_csv(f'{DATA_DIR}/test.csv' )
merged_nonscored_train_df = train_features_df.merge(train_targets_nonscored_df, on='sig_id', how='left' )<filter>
test = test.set_index('qa_id' ).join(sub.set_index('qa_id'))
ctl_sig_ids = merged_nonscored_train_df.query("cp_type=='ctl_vehicle'")['sig_id'].values
trn_df = merged_nonscored_train_df[~merged_nonscored_train_df['sig_id'].isin(ctl_sig_ids)]<create_dataframe>
def postprocessing(oof_df):
oof_values = oof_df[TARGET_COLUMNS].values
DEGREE = 90
oof_values = np.around(oof_values * DEGREE)/ DEGREE
oof_df[TARGET_COLUMNS] = oof_values
return oof_df
procs = []
dls = TabularDataLoaders.from_df(merged_nonscored_train_df,
procs=procs,
cat_names=cat_names,
cont_names=cont_names,
y_names=y_names,
valid_idx=[],
bs=128 )<choose_model_class>
test = postprocessing(test )
tl_learner = tabular_learner(dls,
layers=ParamConfig.model_n_layers*[ParamConfig.model_layer_width],
config = {"ps": ParamConfig.model_n_layers*[ParamConfig.model_ps]},
n_out=len(nonscored_cols),
loss_func=LabelSmoothingBCEWithLogits(ParamConfig.tl_smoothing),
metrics=[metric] )<find_best_params>
for column in TARGET_COLUMNS:
print(test[column].value_counts() )
tl_learner.model<train_model>
sub = test[TARGET_COLUMNS].reset_index()
if ParamConfig.do_transfer_learning: tl_learner.fit_flat_cos(ParamConfig.tl_n_epochs, lr=ParamConfig.tl_lr )<save_model>
sub[ sub[TARGET_COLUMNS] > 1.0] = 1.0
sub[ sub[TARGET_COLUMNS] < 0] = 0
if ParamConfig.do_transfer_learning: tl_learner.save("pretrained_tabular", with_opt=False);<import_modules>
sub['question_type_spelling'].value_counts()
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold<merge>
sub.to_csv('submission.csv', index=False )
merged_scored_train_df = train_features_df.merge(train_targets_scored_df, on='sig_id', how='left')
 ctl_sig_ids = merged_scored_train_df.query("cp_type=='ctl_vehicle'")['sig_id'].values
trn_df = merged_scored_train_df[~merged_scored_train_df['sig_id'].isin(ctl_sig_ids)]<prepare_x_and_y>
sub.to_csv('submission.csv', index=False )