Kaggle competition: Google QUEST Q&A Labeling
submit.to_csv('submit.csv', index = False )<set_options>
ROOT = '.. /input/google-quest-challenge/'
test_df = pd.read_csv(ROOT+'test.csv')
 train_df = pd.read_csv(ROOT+'train.csv' )
%matplotlib inline
plt.style.use('seaborn-whitegrid')
 np.random.seed(1234)
 print(os.listdir(".. /input"))<train_model>
target_cols = ['question_asker_intent_understanding', 'question_body_critical',
'question_conversational', 'question_expect_short_answer',
'question_fact_seeking', 'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible',
'answer_relevance', 'answer_satisfaction',
'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']
class PipelineWithCoef(Pipeline):

 def __init__(self, steps, memory=None):
super(PipelineWithCoef, self ).__init__(steps, memory)
 def fit(self, X, y=None, **fit_params):

 super(PipelineWithCoef, self ).fit(X, y, **fit_params)
 self.coef_ = self.steps[-1][1].coef_
return self<load_from_csv>
import torch
from torch.utils.data import TensorDataset
from torch.utils.data import DataLoader
from torch.utils.data import RandomSampler, SequentialSampler
from pytorch_transformers import BertTokenizer
from sklearn.preprocessing import MinMaxScaler
target_column = 'target'
converters = {target_column: lambda v: int(float(v)) }
train_df = pd.read_csv('.. /input/train.csv', index_col=0, converters=converters)
 feature_columns = train_df.columns.drop(target_column)
 print(train_df.shape)
 train_df.head()
train_df.target.value_counts()<count_missing_values>
def read_data(raw_data_path):
test = pd.read_csv(raw_data_path, encoding='utf-8')
 targets = [-1] * len(test)
 sentence_a = test['question_title'] + test['question_body']
sentence_b = test['answer']
return targets, sentence_a, sentence_b
def save_pickle(data, file_path):

 if isinstance(file_path, Path):
file_path = str(file_path)
 with open(file_path, 'wb')as f:
pickle.dump(data, f)
 y, X_a, X_b = read_data(ROOT+'test.csv' )
( train_df.isnull().sum() > 0 ).any()<sort_values>
data = []
for step,(data_x_a, data_x_b, data_y)in enumerate(zip(X_a, X_b, y)) :
data.append(( [data_x_a, data_x_b], data_y))
target_correlations = corr[target_column].drop(target_column, axis=0)
 top_target_correlations = target_correlations.abs().sort_values(ascending=False)[:50]
top_corr_feature_columns = list(top_target_correlations.index)
 top_target_correlations[:20]<normalization>
tokenizer = BertTokenizer("/kaggle/input/bertpretrained/uncased_L-24_H-1024_A-16/uncased_L-24_H-1024_A-16/vocab.txt", True )
std = StandardScaler()
X_scaled = std.fit_transform(train_df[feature_columns])
 X_scaled = pd.DataFrame(X_scaled, columns=feature_columns )<train_on_grid>
class InputExample(object):
def __init__(self, guid, text_a, text_b=None, label=None):

 self.guid = guid
self.text_a = text_a
self.text_b = text_b
self.label = label
class InputFeature(object):

 def __init__(self, input_ids, input_mask, segment_ids, label_id, input_len):
self.input_ids = input_ids
self.input_mask = input_mask
self.segment_ids = segment_ids
self.label_id = label_id
self.input_len = input_len
def create_examples(lines, example_type):

 examples = []
for i, line in enumerate(lines):
guid = '%s-%d' %(example_type, i)
 text_a = line[0][0]
text_b = line[0][1]
label = line[1]
example = InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)
 examples.append(example)
 return examples
def truncate_seq_pair(tokens_a, tokens_b, max_length=512):
while True:
total_length = len(tokens_a)+ len(tokens_b)
 if total_length <= max_length:
break
if len(tokens_a)> len(tokens_b):
tokens_a.pop()
else:
tokens_b.pop()
def create_features(examples, max_seq_len=512):

 features = []
for ex_id, example in enumerate(examples):
tokenizer = BertTokenizer("/kaggle/input/bertpretrained/uncased_L-24_H-1024_A-16/uncased_L-24_H-1024_A-16/vocab.txt", True)
 tokens_a = tokenizer.tokenize(example.text_a)
 tokens_b = None
label_id = example.label
if example.text_b:
tokens_b = tokenizer.tokenize(example.text_b)
 truncate_seq_pair(tokens_a, tokens_b, max_length=max_seq_len - 3)
 else:
if len(tokens_a)> max_seq_len - 2:
tokens_a = tokens_a[:max_seq_len - 2]
tokens = ['[CLS]'] + tokens_a + ['[SEP]']
segment_ids = [0] * len(tokens)
 if tokens_b:
tokens += tokens_b + ['[SEP]']
segment_ids += [1] *(len(tokens_b)+ 1)
 input_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_mask = [1] * len(input_ids)
 padding = [0] *(max_seq_len - len(input_ids))
input_len = len(input_ids)
 input_ids += padding
input_mask += padding
segment_ids += padding
assert len(input_ids)== max_seq_len
assert len(input_mask)== max_seq_len
assert len(segment_ids)== max_seq_len
feature = InputFeature(input_ids=input_ids,
input_mask=input_mask,
segment_ids=segment_ids,
label_id=label_id,
input_len=input_len)
 features.append(feature)
 return features
lr = LogisticRegression(solver='liblinear', random_state=42)
 param_grid = {'class_weight' : ['balanced', None],
'penalty' : ['l2','l1'],
'C' : [0.001, 0.01, 0.1, 1, 10, 100, 1000]}
grid = GridSearchCV(estimator=lr, param_grid=param_grid, iid=False, cv=3, scoring='roc_auc', n_jobs=-1)
 grid.fit(X_scaled[feature_columns], train_df[target_column])
 print("Grid best score was: {}".format(grid.best_score_))
lr = LogisticRegression(solver='liblinear', **grid.best_params_)
 lr.fit(X_scaled[feature_columns], train_df[target_column])
 rfe_lr = RFE(lr, 25, step=1)
 rfe_lr.fit(X_scaled, train_df[target_column] )<save_to_csv>
test_examples = create_examples(data, 'test')
 test_features = create_features(test_examples )
test_df = pd.read_csv('.. /input/test.csv', index_col=0)
 X_test_scaled = std.transform(test_df[feature_columns])
 X_test_scaled = pd.DataFrame(X_test_scaled, columns=feature_columns)
 submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['target'] = rfe_lr.predict_proba(X_test_scaled)
 submission.to_csv('rfe_lr_submission.csv', index=False )<train_model>
def create_dataset(features, is_sorted=False):
if is_sorted:
logger.info("sorted data by th length of input")
 features = sorted(features, key=lambda x: x.input_len, reverse=True)
 all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
 all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
 all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
 all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.float)
 dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
 return dataset
def test_model(model, parameter_grid, train_feature_values, train_target_values, retrain_with_top_features=True, importance_threshold=0.0005):

 metric = metrics.make_scorer(metrics.roc_auc_score, greater_is_better=True, needs_proba=True)
 def cv_train_model(model, parameter_grid, train_feature_values, train_target_values):

 folds = RepeatedStratifiedKFold(n_splits=5, n_repeats=3, random_state=1234)
 folds_iterable = folds.split(train_feature_values, train_target_values)
 grid_search = GridSearchCV(model, param_grid=parameter_grid, scoring=metric, iid=False, refit=True, cv=folds_iterable, return_train_score=True, n_jobs=-1)
 grid_search_model = grid_search.fit(train_feature_values, train_target_values)
 cv_results = pd.DataFrame.from_dict(grid_search_model.cv_results_)
 cv_results.set_index('rank_test_score', drop=True, inplace=True)
 cv_results.sort_index(inplace=True)
 cv_result_columns = [column for column in cv_results.columns if column.startswith('param_')] + ['mean_test_score', 'std_test_score', 'mean_train_score', 'std_train_score']
display(cv_results[cv_result_columns].head(5))
return grid_search_model
print("Training first model...")
 grid_search_model = cv_train_model(model, parameter_grid, train_feature_values, train_target_values)
 print("Checking feature importances...")
 perm = eli5.sklearn.PermutationImportance(grid_search_model, scoring=metric, n_iter=10, random_state=1234 ).fit(train_feature_values, train_target_values)
 display(eli5.show_weights(perm, top=25))
features_importance_df = eli5.formatters.as_dataframe.explain_weights_df(perm)
 top_feature_columns = [train_feature_values.columns[int(feature_column[1:])]
for feature_column
in features_importance_df[features_importance_df.weight > importance_threshold].feature]
if len(top_feature_columns)== 0:
print("No top feature columns found.Exiting function.")
 return grid_search_model, list(train_feature_values.columns)
 top_feature_columns = top_feature_columns[:50]
if retrain_with_top_features and len(top_feature_columns)< len(train_feature_values.columns):
print("Retraining with top {} features.".format(len(top_feature_columns)))
 grid_search_model = cv_train_model(model, parameter_grid, train_feature_values[top_feature_columns], train_target_values)
 elif retrain_with_top_features:
print("All features are important.Therefore not retraining.")
 top_feature_columns = list(train_feature_values.columns)
 else:
print("Not retraining with top features.")
 top_feature_columns = list(train_feature_values.columns)
 return grid_search_model, top_feature_columns<define_variables>
test_dataset = create_dataset(test_features)
 test_sampler = SequentialSampler(test_dataset)
 test_dataloader = DataLoader(test_dataset,sampler=test_sampler,batch_size=32 )
all_scores = dict()<define_search_space>
class BertForMultiClass(BertPreTrainedModel):
def __init__(self, config):
super(BertForMultiClass, self ).__init__(config)
 self.bert = BertModel(config)
 self.dropout = nn.Dropout(config.hidden_dropout_prob)
 self.classifier = nn.Linear(config.hidden_size, config.num_labels)
 self.apply(self.init_weights)
 def forward(self, input_ids, token_type_ids=None, attention_mask=None, head_mask=None):
outputs = self.bert(input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask,
head_mask=head_mask)
 pooled_output = outputs[1]
pooled_output = self.dropout(pooled_output)
 logits = self.classifier(pooled_output)
 return logits
sc_lr = Pipeline(steps=[('sc', StandardScaler(with_mean=True, with_std=True)) , 
('logistic_regression', LogisticRegression(solver='liblinear', max_iter=100, random_state=42)) ])
 param_grid = {'logistic_regression__class_weight': ['balanced', None],
'logistic_regression__penalty': ['l1', 'l2'],
'logistic_regression__C': [0.01, 0.1, 0.5, 1., 1.5, 10.]}
sc_lr_model, sc_lr_feature_columns = test_model(sc_lr, param_grid, train_df[feature_columns], train_df[target_column])
 all_scores['sc_lr'] = sc_lr_model.best_score_<choose_model_class>
def prepare_device(use_gpu=0):

 n_gpu_use = [int(x)for x in use_gpu.split(",")]
if not use_gpu:
device_type = 'cpu'
else:
device_type = f"cuda:{n_gpu_use[0]}"
n_gpu = torch.cuda.device_count()
if len(n_gpu_use)> 0 and n_gpu == 0:
device_type = 'cpu'
if len(n_gpu_use)> n_gpu:
msg = f"Warning: The number of GPU's configured to use is {n_gpu}, but only {n_gpu} are available on this machine."
n_gpu_use = range(n_gpu)
 device = torch.device(device_type)
 list_ids = n_gpu_use
return device, list_ids
def model_device(n_gpu, model):

 device, device_ids = prepare_device(n_gpu)
 if len(device_ids)> 1:
model = torch.nn.DataParallel(model, device_ids=device_ids)
 if len(device_ids)== 1:
os.environ['CUDA_VISIBLE_DEVICES'] = str(device_ids[0])
 model = model.to(device)
 return model, device
class Predictor(object):
def __init__(self, model, n_gpu):
self.model = model
self.model, self.device = model_device(n_gpu=n_gpu, model=self.model)
 def predict(self, data):
all_logits = None
self.model.eval()
with torch.no_grad() :
for step, batch in enumerate(data):
batch = tuple(t.to(self.device)for t in batch)
 input_ids, input_mask, segment_ids, label_ids = batch
logits = self.model(input_ids, segment_ids, input_mask)
 if all_logits is None:
all_logits = torch.sigmoid(logits ).detach().cpu().numpy()
else:
all_logits = np.concatenate([all_logits, torch.sigmoid(logits ).detach().cpu().numpy() ], axis=0)
 if 'cuda' in str(self.device):
torch.cuda.empty_cache()
return all_logits
sc_pca_lr = Pipeline(steps=[('sc', StandardScaler(with_mean=True, with_std=True)) , 
('pca', PCA()), 
('logistic_regression', LogisticRegression(solver='liblinear', max_iter=1000)) ])
 param_grid = {'pca__n_components': [5, 10, 15],
'logistic_regression__class_weight': ['balanced', None],
'logistic_regression__penalty': ['l1', 'l2'],
'logistic_regression__C': [0.1, 1., 10.]}
sc_pca_lr_model, sc_pca_lr_feature_columns = test_model(sc_pca_lr, param_grid, train_df[feature_columns], train_df[target_column], importance_threshold=0.001)
 all_scores['sc_pca_lr'] = sc_pca_lr_model.best_score_<choose_model_class>
model1 = BertForMultiClass.from_pretrained("/kaggle/input/bert-fold1/", num_labels=30)
 model2 = BertForMultiClass.from_pretrained("/kaggle/input/bert-fold2/", num_labels=30)
 model3 = BertForMultiClass.from_pretrained("/kaggle/input/bert-fold3/", num_labels=30)
 model4 = BertForMultiClass.from_pretrained("/kaggle/input/bert-fold4/", num_labels=30)
 model5 = BertForMultiClass.from_pretrained("/kaggle/input/bert-fold5/", num_labels=30)
 result = []
predictor1 = Predictor(model=model1, n_gpu='0')
 result1 = predictor1.predict(data=test_dataloader)
 predictor2 = Predictor(model=model2, n_gpu='0')
 result2 = predictor2.predict(data=test_dataloader)
 predictor3 = Predictor(model=model3, n_gpu='0')
 result3 = predictor3.predict(data=test_dataloader)
 predictor4 = Predictor(model=model4, n_gpu='0')
 result4 = predictor4.predict(data=test_dataloader)
 predictor5 = Predictor(model=model5, n_gpu='0')
 result5 = predictor5.predict(data=test_dataloader)
 result.append([result1, result2, result3, result4, result5] )
sc_lda_lr = Pipeline(steps=[('sc', StandardScaler(with_mean=True, with_std=True)) , 
('lda', LinearDiscriminantAnalysis()), 
('logistic_regression', LogisticRegression(solver='liblinear', max_iter=1000)) ])
 param_grid = {'lda__n_components': [5, 10, 15],
'logistic_regression__class_weight': ['balanced', None],
'logistic_regression__penalty': ['l1', 'l2'],
'logistic_regression__C': [0.1, 1., 10.]}
sc_lda_lr_model, sc_lda_lr_feature_columns = test_model(sc_lda_lr, param_grid, train_df[feature_columns], train_df[target_column])
 all_scores['sc_lda_lr'] = sc_lda_lr_model.best_score_<find_best_params>
test_predictions = np.average(result[0], axis=0 )
svc = Pipeline(steps=[('svc', SVC(kernel='linear', gamma='auto', probability=True)) ])
 param_grid = {'svc__C': [0.01, 0.1, 0.5, 1., 1.5, 10., 100.],
'svc__class_weight': ['balanced', None]}
svc_model, svc_feature_columns = test_model(svc, param_grid, train_df[top_corr_feature_columns], train_df[target_column])
 all_scores['svc'] = svc_model.best_score_<train_model>
n=test_df['url'].apply(lambda x:('english.stackexchange.com' in x)).tolist()
spelling=[]
for x in n:
if x:
spelling.append(0.5)
 else:
spelling.append(0.)
def test_model_with_rfe(best_model, train_feature_values, train_target_values, min_nr_features=10, max_nr_features=40):

 deepcopied_best_model = deepcopy(best_model.best_estimator_)
 nr_feaures_to_select_range = list(range(min_nr_features, max_nr_features + 1))
all_scores = []
all_estimators = []
for n_features_to_select in tqdm(nr_feaures_to_select_range):
best_pipeline = PipelineWithCoef(steps=deepcopied_best_model.steps)
 best_pipeline.fit(train_feature_values, train_target_values)
 estimator = RFE(best_pipeline, n_features_to_select=n_features_to_select)
 estimator.fit(train_feature_values, train_target_values)
 folds = RepeatedStratifiedKFold(n_splits=5, n_repeats=1, random_state=1234)
 folds_iterable = folds.split(train_feature_values, train_target_values)
 scores = cross_val_score(estimator, train_feature_values, train_target_values, scoring='roc_auc', cv=folds_iterable)
 all_scores.append(scores.mean())
 all_estimators.append(estimator)
 def plot_roc_auc_scores(nr_features_list, scores_list):

 fig, ax = plt.subplots(figsize=(20, 4))
ax.plot(nr_features_list, scores_list)
 for s in ax.spines:
ax.spines[s].set_visible(False)
 plt.xticks(fontsize=12)
 plt.yticks(fontsize=12)
 ax.set_title('ROC-AUC Score', fontsize=16)
 ax.set_xlabel('Nr Features Selected', fontsize=14);
plot_roc_auc_scores(nr_feaures_to_select_range, all_scores)
 top_index = np.argmax(all_scores)
 print("Number of features selected for best score: {}".format(nr_feaures_to_select_range[top_index]))
best_estimator = all_estimators[top_index]
return best_estimator<train_model>
test_preds = test_predictions
y_train = train_df[target_cols].values
for column_ind in range(30):
curr_column = y_train[:, column_ind]
values = np.unique(curr_column)
 map_quantiles = []
for val in values:
occurrence = np.mean(curr_column == val)
 cummulative = sum(el['occurrence'] for el in map_quantiles)
 map_quantiles.append({'value': val, 'occurrence': occurrence, 'cummulative': cummulative})
 for quant in map_quantiles:
pred_col = test_preds[:, column_ind]
q1, q2 = np.quantile(pred_col, quant['cummulative']), np.quantile(pred_col, min(quant['cummulative'] + quant['occurrence'], 1))
pred_col[(pred_col >= q1)&(pred_col <= q2)] = quant['value']
test_preds[:, column_ind] = pred_col
rfe_sc_lr_model = test_model_with_rfe(sc_lr_model, train_df[feature_columns], train_df[target_column], max_nr_features=30 )<load_from_csv>
submission_df = pd.read_csv(ROOT+'sample_submission.csv')
 submission_df[target_cols] = test_preds
submission_df['question_type_spelling']=spelling
submission_df
<save_to_csv><EOS> 
sub_file_name = 'submission.csv'
submission_df.to_csv(sub_file_name, index=False)