Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
%matplotlib inline
warnings.filterwarnings("ignore")
 sns.set(style="white", font_scale=1.2)
test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv')
 test.head(3 )<define_variables>
df_train = pd.read_csv('.. /input/titanic/train.csv')
 df_test = pd.read_csv('.. /input/titanic/test.csv' )
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
] 
<feature_engineering>
lengths = train['question_title'].apply(len)
 train['lengths'] = lengths
lengths = train.loc[train['lengths']<4000]['lengths']
sns.distplot(lengths, color='b')
 plt.show()<feature_engineering>
df_train.isnull().sum()
stopwords=stopwords.words('english')
 train['que_stopwords']=train['question_body'].apply(lambda x : [x for x in x.split() if x in stopwords])
 train['ans_stopwords']=train['answer'].apply(lambda x: [x for x in x.split() if x in stopwords] )<count_unique_values>
df_test.isnull().sum()
def common_ngrams(col,common=10):
corpus=[]
for question in train[col].values:
words=[str(x[0]+' '+x[1])for x in ngrams(question.split() ,2)]
corpus.append(words)
 flatten=[x for one in corpus for x in one]
counter=Counter(flatten)
 most_common=counter.most_common(common)
 string,value=zip(*(most_common))
return string,value 
<set_options>
def check_missing_values(df, df_name=None):
print(f'{df_name} - Missing values:')
 print('-'*30)
 columns = df.columns
for column in columns:
count_missing_values = df[column].isnull().sum()
missing_values =(count_missing_values / len(df[column])) * 100
if missing_values !=0:
print(f'{column} --> {count_missing_values} values | {missing_values:.2f}%' )
np.set_printoptions(suppress=True )<load_from_csv>
check_missing_values(df_train, 'TRAIN' )
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = FullTokenizer(BERT_PATH+'/assets/vocab.txt', True)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )<string_transform>
check_missing_values(df_test, 'TEST' )
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<compute_test_metric>
df_train.drop(['PassengerId', 'Cabin', 'Ticket'], axis=1, inplace=True)
 submission = pd.DataFrame()
submission['PassengerId'] = df_test['PassengerId']
df_test.drop(['PassengerId', 'Cabin', 'Ticket'], axis=1, inplace=True )
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback<load_pretrained>
check_missing_values(df_train, 'DF TRAIN' )
models = []
for i in range(5):
model_path = f'.. /input/bertuned-f{i}/bertuned_f{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model)
 model_path = f'.. /input/bertf1e15/Full-0.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )<load_pretrained>
check_missing_values(df_test, 'DF TEST' )
for i in range(2):
model_path = f".. /input/bertmodelpretrained/bert-{i}.h5"
model = bert_model()
model.load_weights(model_path )<concatenate>
df_train['Familysize'] = df_train['SibSp'] + df_train['Parch']
df_test['Familysize'] = df_test['SibSp'] + df_test['Parch']
models.append(model )<define_variables>
df_train['Alone'] = df_train['Familysize'].apply(lambda x: 1 if x == 0 else 0)
 df_test['Alone'] = df_test['Familysize'].apply(lambda x: 1 if x == 0 else 0 )
test_predictions = []<predict_on_test>
df_train[df_train['Embarked'].isnull() ]
for model in models:
test_predictions.append(model.predict(test_inputs, batch_size=8))<prepare_output>
df_train['Embarked'] = df_train['Embarked'].fillna('C' )
final_predictions = np.mean(test_predictions, axis=0 )<save_to_csv>
df_test[df_test['Fare'].isnull() ]
df_sub.iloc[:, 1:] = final_predictions
df_sub.to_csv('submission.csv', index=False )<load_from_csv>
median_fare = df_test[(df_test['Pclass'] == 3)&(df_test['Embarked'] == 'S')&(df_test['Alone'] == 1)]['Fare'].median()
median_fare
train = pd.read_csv(".. /input/google-quest-challenge/train.csv", index_col='qa_id')
 train.shape<load_from_csv>
df_test['Fare'] = df_test['Fare'].fillna(median_fare )
test = pd.read_csv(".. /input/google-quest-challenge/test.csv", index_col='qa_id')
 test.shape<define_variables>
def get_age(cols):
age = cols[0]
pclass = cols[1]
sex = cols[2]
if pd.isnull(age):
if pclass == 1:
if sex == 'male':
return 40
else:
return 35
elif pclass == 2:
if sex == 'male':
return 30
else:
return 28
else:
if sex == 'male':
return 25
else:
return 21.5
else:
return age
target_columns = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]<prepare_x_and_y>
df_train['Age'] = df_train[['Age','Pclass', 'Sex']].apply(get_age, axis=1)
 df_test['Age'] = df_test[['Age','Pclass', 'Sex']].apply(get_age, axis=1 )
y_train = train[target_columns].copy()
x_train = train.drop(target_columns, axis=1)
 del train
x_test = test.copy()
del test<import_modules>
df_train['Title'] = df_train['Name'].apply(lambda x: get_title(x))
df_test['Title'] = df_test['Name'].apply(lambda x: get_title(x))
import tensorflow_hub as hub
import tensorflow as tf<define_variables>
df_train['Title'].value_counts()
copyfile(src = ".. /input/tf-bert-tokenization/tokenization.py", dst = ".. /working/tokenization.py")
<define_variables>
df_train.drop('Name', axis=1, inplace=True)
 df_test.drop('Name', axis=1, inplace=True )
BERT = '.. /input/bert-model'
tokenizer = FullTokenizer(BERT + '/assets/vocab.txt', True )<string_transform>
for dataframe in [df_train, df_test]:
dataframe['Title'] = dataframe['Title'].replace(['Lady', 'Capt', 'Col','Don', 'Dr',
'Major', 'Rev', 'Sir', 'Dona', 'Countess', 'Jonkheer'], 'Other')
 dataframe['Title'] = dataframe['Title'].replace('Mlle', 'Miss')
 dataframe['Title'] = dataframe['Title'].replace('Ms', 'Miss')
 dataframe['Title'] = dataframe['Title'].replace('Mme', 'Mrs' )
tokenizer.tokenize('Hello world from BERT FullTokenizer!' )<categorify>
sex = pd.get_dummies(df_train['Sex'], prefix='Sex', drop_first=True)
 embarked = pd.get_dummies(df_train['Embarked'], prefix='Embarked', drop_first=True)
 pclass = pd.get_dummies(df_train['Pclass'], prefix='Pclass', drop_first=True)
 title = pd.get_dummies(df_train['Title'], prefix='Title', drop_first=True)
 df_train.drop(['Sex', 'Embarked', 'Pclass', 'Title'], axis=1, inplace=True)
 df_train = pd.concat([df_train, sex, embarked, pclass, title], axis=1 )
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids<feature_engineering>
sex = pd.get_dummies(df_test['Sex'], prefix='Sex', drop_first=True)
 embarked = pd.get_dummies(df_test['Embarked'], prefix='Embarked',drop_first=True)
 pclass = pd.get_dummies(df_test['Pclass'], prefix='Pclass',drop_first=True)
 title = pd.get_dummies(df_test['Title'], prefix='Title', drop_first=True)
 df_test.drop(['Sex', 'Embarked', 'Pclass', 'Title'], axis=1, inplace=True)
 df_test = pd.concat([df_test, sex, embarked, pclass, title], axis=1 )
def trim_tokens(t, q, a, max_t, max_q, max_a):
if(len(t)+ len(q)+ len(a)) >(max_t + max_q + max_a):
_max_t = max_t
_max_q = max_q
_max_a = max_a
if len(t)> _max_t:
t = t[:_max_t]
else:
x =(_max_t - len(t)) / 2.
 _max_q += math.ceil(x)
 _max_a += math.floor(x)
 if len(q)> _max_q:
q = q[:_max_q]
else:
_max_a +=(_max_q - len(q))
if len(a)> _max_a:
a = a[:_max_a]
return t,q,a
def make_bert_input(x, max_question_title_length=24, max_question_body_length=242, max_answer_length=242):
max_sequence_length = max_question_title_length + max_question_body_length + max_answer_length + 4
print('Calculated max sequence length:', max_sequence_length)
 input_ids = []
input_masks = []
input_segments = []
for idx, row in x[['question_title', 'question_body', 'answer']].iterrows() :
t,q,a = trim_tokens(tokenizer.tokenize(row.question_title),
tokenizer.tokenize(row.question_body),
tokenizer.tokenize(row.answer),
max_question_title_length,
max_question_body_length,
max_answer_length)
 tokens = ['[CLS]'] + t + ['[SEP]'] + q + ['[SEP]'] + a + ['[SEP]']
input_ids.append(_get_ids(tokens, tokenizer, max_sequence_length))
input_masks.append(_get_masks(tokens, max_sequence_length))
input_segments.append(_get_segments(tokens, max_sequence_length))
return [
np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)
 ]<define_variables>
scaler = StandardScaler()
df_train[['Age', 'Fare']] = scaler.fit_transform(df_train[['Age', 'Fare']])
 df_test[['Age', 'Fare']] = scaler.transform(df_test[['Age', 'Fare']] )
max_sequence_length = 512 
<choose_model_class>
from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, make_scorer
def make_model() :
input_word_ids = tf.keras.layers.Input(shape=(max_sequence_length,), dtype=tf.int32,
name="input_word_ids")
 input_mask = tf.keras.layers.Input(shape=(max_sequence_length,), dtype=tf.int32,
name="input_mask")
 segment_ids = tf.keras.layers.Input(shape=(max_sequence_length,), dtype=tf.int32,
name="segment_ids")
 bert_layer = hub.KerasLayer(BERT, trainable=True)
 pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 tmp = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 tmp = tf.keras.layers.Dropout(0.2 )(tmp)
 out = tf.keras.layers.Dense(len(target_columns), activation='sigmoid', name='dense_output' )(tmp)
 model = tf.keras.models.Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(loss='binary_crossentropy', optimizer=tf.keras.optimizers.Adam(learning_rate=1e-5))
return model<compute_test_metric>
X = df_train.drop('Survived', axis=1)
 y = df_train['Survived']
def mean_spearmanr_correlation_score(y_true, y_pred):
return np.mean([spearmanr(y_pred[:, idx] + np.random.normal(0, 1e-7, y_pred.shape[0]),
y_true[:, idx] ).correlation for idx in range(len(target_columns)) ] )<define_variables>
X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, test_size=0.2, random_state=1 )
trained_estimators = []<train_model>
predictions = {}
n_splits = 5
scores = []
cv = KFold(n_splits=n_splits, random_state=42)
 idx = 1
for train_idx, valid_idx in cv.split(x_train, y_train, groups=x_train.question_body):
x_train_train = x_train.iloc[train_idx]
y_train_train = y_train.iloc[train_idx]
x_train_valid = x_train.iloc[valid_idx]
y_train_valid = y_train.iloc[valid_idx]
K.clear_session()
estimator = make_model()
estimator.fit(make_bert_input(x_train_train), y_train_train, batch_size=8, epochs=5)
 trained_estimators.append(estimator)
 oof_part = estimator.predict(make_bert_input(x_train_valid))
score = mean_spearmanr_correlation_score(y_train_valid.values, oof_part)
 print('Score:', score)
 if not math.isnan(score):
scores.append(score)
 idx += 1
if idx > 3:
break
print('Mean score:', np.mean(scores))<predict_on_test>
from sklearn.linear_model import LogisticRegression
y_pred = []
for estimator in trained_estimators:
y_pred.append(estimator.predict(make_bert_input(x_test)) )<concatenate>
logreg = LogisticRegression(random_state=121)
 logreg.fit(X_train, y_train)
 y_pred = logreg.predict(X_test)
 accuracy = accuracy_score(y_test, y_pred)
 print('Accuracy:', accuracy )
def blend_by_ranking(data, weights):
out = np.zeros(data.shape[0])
 for idx,column in enumerate(data.columns):
out += weights[idx] * rankdata(data[column].values)
 out /= np.max(out)
 return out<load_from_csv>
logreg = LogisticRegression(random_state=121)
 param_grid = {
'penalty': ['l1', 'l2', 'elasticnet'],
'C': [0.01, 0.05, 0.1, 0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1,2,3,4,5,6,7,8,9,10,12,13,14,15,16,16.5,17,18],
'solver': ['liblinear','saga']}
submission = pd.read_csv(".. /input/google-quest-challenge/sample_submission.csv", index_col='qa_id')
 out = pd.DataFrame(index=submission.index)
 for column_idx,column in enumerate(target_columns):
column_data = pd.DataFrame(index=submission.index)
 for prediction_idx,prediction in enumerate(y_pred):
column_data[str(prediction_idx)] = prediction[:, column_idx]
out[column] = blend_by_ranking(column_data, np.ones(column_data.shape[1]))<save_to_csv>
model = GridSearchCV(logreg, param_grid=param_grid, scoring='accuracy', cv=10, n_jobs=-1)
 model.fit(X_train, y_train)
 print('Best Params:', model.best_params_ )
out.to_csv("submission.csv" )<set_options>
best_lr = LogisticRegression(C=0.9, penalty='l1', solver='liblinear')
 best_lr.fit(X_train, y_train)
 y_pred = best_lr.predict(X_test )
SEED = 0
warnings.filterwarnings("ignore")
 sns.set(font_scale=1.5)
 plt.rcParams.update({'font.size': 16})
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
print(f'Accuracy: {accuracy_score(y_test, y_pred)*100:.2f}%')
 print('-'*55)
 print(classification_report(y_test, y_pred))
print('-'*55)
 print(confusion_matrix(y_test, y_pred))
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv')
 test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv')
 train['set'] = 'train'
test['set'] = 'test'
complete_set = train.append(test)
 print('Train samples: %s' % len(train))
print('Test samples: %s' % len(test))
display(train.head() )<train_model>
from sklearn.neighbors import KNeighborsClassifier
samp_id = 9
print('Question Title: %s
' % train['question_title'].values[samp_id])
 print('Question Body: %s
' % train['question_body'].values[samp_id])
 print('Answer: %s' % train['answer'].values[samp_id] )<define_variables>
from sklearn.neighbors import KNeighborsClassifier
question_target_cols = ['question_asker_intent_understanding','question_body_critical', 'question_conversational',
'question_expect_short_answer', 'question_fact_seeking', 'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self', 'question_multi_intent',
'question_not_really_a_question', 'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence', 'question_type_definition',
'question_type_entity', 'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling', 'question_well_written']
answer_target_cols = ['answer_helpful', 'answer_level_of_information', 'answer_plausible', 'answer_relevance',
'answer_satisfaction', 'answer_type_instructions', 'answer_type_procedure',
'answer_type_reason_explanation', 'answer_well_written']
target_cols = question_target_cols + answer_target_cols
print('Question labels')
 display(train.iloc[[samp_id]][question_target_cols])
 print('Answer labels')
 display(train.iloc[[samp_id]][answer_target_cols] )<count_unique_values>
from sklearn.neighbors import KNeighborsClassifier
train_users = set(train['question_user_page'].unique())
 test_users = set(test['question_user_page'].unique())
 print('Unique users in train set: %s' % len(train_users))
print('Unique users in test set: %s' % len(test_users))
print('Users in both sets: %s' % len(train_users & test_users))
print('What users are in both sets? %s' % list(train_users & test_users))<count_unique_values>
error_rate = []
for i in range(1,40):
knn = KNeighborsClassifier(n_neighbors=i)
 knn.fit(X_train, y_train)
 pred_i = knn.predict(X_test)
 error_rate.append(np.mean(pred_i != y_test))
train_users = set(train['answer_user_page'].unique())
 test_users = set(test['answer_user_page'].unique())
 print('Unique users in train set: %s' % len(train_users))
print('Unique users in test set: %s' % len(test_users))
print('Users in both sets: %s' % len(train_users & test_users))<feature_engineering>
knn = KNeighborsClassifier(n_neighbors=25)
 knn.fit(X_train, y_train)
 y_pred = knn.predict(X_test )
complete_set['question_title_len'] = complete_set['question_title'].apply(lambda x : len(x))
complete_set['question_body_len'] = complete_set['question_body'].apply(lambda x : len(x))
complete_set['answer_len'] = complete_set['answer'].apply(lambda x : len(x))
complete_set['question_title_wordCnt'] = complete_set['question_title'].apply(lambda x : len(x.split(' ')))
 complete_set['question_body_wordCnt'] = complete_set['question_body'].apply(lambda x : len(x.split(' ')))
 complete_set['answer_wordCnt'] = complete_set['answer'].apply(lambda x : len(x.split(' ')))
 f,(ax1, ax2)= plt.subplots(2, 1, figsize=(24, 7), sharex=True)
 sns.distplot(complete_set[complete_set['set'] == 'train']['question_title_len'], ax=ax1 ).set_title("Train")
 sns.distplot(complete_set[complete_set['set'] == 'test']['question_title_len'], ax=ax2 ).set_title("Test")
 plt.show()
f,(ax1, ax2)= plt.subplots(2, 1, figsize=(24, 7), sharex=True)
 sns.distplot(complete_set[complete_set['set'] == 'train']['question_title_wordCnt'], ax=ax1 ).set_title("Train")
 sns.distplot(complete_set[complete_set['set'] == 'test']['question_title_wordCnt'], ax=ax2 ).set_title("Test")
 plt.show()<feature_engineering>
print(f'Accuracy: {accuracy_score(y_test, y_pred)*100:.2f}%')
 print('-'*55)
 print(classification_report(y_test, y_pred))
print('-'*55)
 print(confusion_matrix(y_test, y_pred))
eng_stopwords = stopwords.words('english')
 complete_set['question_title'] = complete_set['question_title'].str.replace('[^a-z ]','')
 complete_set['question_body'] = complete_set['question_body'].str.replace('[^a-z ]','')
 complete_set['answer'] = complete_set['answer'].str.replace('[^a-z ]','')
 complete_set['question_title'] = complete_set['question_title'].apply(lambda x: x.lower())
 complete_set['question_body'] = complete_set['question_body'].apply(lambda x: x.lower())
 complete_set['answer'] = complete_set['answer'].apply(lambda x: x.lower())
 freq_dist = FreqDist([word for comment in complete_set['question_title'] for word in comment.split() if word not in eng_stopwords])
 plt.figure(figsize=(20, 6))
plt.title('Word frequency on question title' ).set_fontsize(20)
 freq_dist.plot(60, marker='.', markersize=10)
 plt.show()
freq_dist = FreqDist([word for comment in complete_set['question_body'] for word in comment.split() if word not in eng_stopwords])
 plt.figure(figsize=(20, 6))
plt.title('Word frequency on question body' ).set_fontsize(20)
 freq_dist.plot(60, marker='.', markersize=10)
 plt.show()
freq_dist = FreqDist([word for comment in complete_set['answer'] for word in comment.split() if word not in eng_stopwords])
 plt.figure(figsize=(20, 6))
plt.title('Word frequency on answer' ).set_fontsize(20)
 freq_dist.plot(60, marker='.', markersize=10)
 plt.show() 
<set_options>
from sklearn.ensemble import RandomForestClassifier
np.set_printoptions(suppress=True )<load_from_csv>
rf = RandomForestClassifier(random_state=121)
 rf.fit(X_train, y_train)
 y_pred = rf.predict(X_test)
 accuracy = accuracy_score(y_test, y_pred)
 print('Accuracy:', accuracy )
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = tokenization.FullTokenizer(BERT_PATH+'/assets/vocab.txt', True)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )<string_transform>
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<compute_test_metric>
best_rf = RandomForestClassifier(random_state=121, criterion='entropy', max_depth=15, min_samples_leaf=5, min_samples_split=2, n_estimators=50)
 best_rf.fit(X_train, y_train)
 y_pred = best_rf.predict(X_test)
 accuracy = accuracy_score(y_test, y_pred )
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.mean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback 
<split>
print(f'Accuracy: {accuracy_score(y_test, y_pred)*100:.2f}%')
 print('-'*55)
 print(classification_report(y_test, y_pred))
print('-'*55)
 print(confusion_matrix(y_test, y_pred))
gkf = GroupKFold(n_splits=5 ).split(X=df_train.question_body, groups=df_train.question_body)
 outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )<choose_model_class>
from xgboost import XGBClassifier
histories = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold < 3:
K.clear_session()
model = bert_model()
train_inputs = [inputs[i][train_idx] for i in range(3)]
train_outputs = outputs[train_idx]
valid_inputs = [inputs[i][valid_idx] for i in range(3)]
valid_outputs = outputs[valid_idx]
history = train_and_predict(model,
train_data=(train_inputs, train_outputs),
valid_data=(valid_inputs, valid_outputs),
test_data=test_inputs,
learning_rate=3e-5, epochs=4, batch_size=8,
loss_function='binary_crossentropy', fold=fold)
 histories.append(history)
<save_to_csv>
xgb = XGBClassifier(random_state=121)
 xgb.fit(X_train, y_train)
 y_pred = xgb.predict(X_test)
 accuracy = accuracy_score(y_test, y_pred)
 print('Accuracy:', accuracy )
test_predictions = [histories[i].test_predictions for i in range(len(histories)) ]
test_predictions = [np.average(test_predictions[i], axis=0)for i in range(len(test_predictions)) ]
test_predictions = np.mean(test_predictions, axis=0)
 df_sub.iloc[:, 1:] = test_predictions
df_sub.to_csv('submission.csv', index=False )<install_modules>
classifiers = [('Logistic Regression', best_lr), 
('KNN', knn), 
('Random Forest', best_rf), 
('Xgboost', xgb)]
for name_clf, clf in classifiers:
y_pred = clf.predict(X_test)
 acc = accuracy_score(y_test, y_pred)
 print(f'{name_clf} accuracy: {round(acc, 3)}%' )
!pip install.. /input/sacremoses/sacremoses-master/ > /dev/null<install_modules>
from sklearn.ensemble import VotingClassifier
!pip install ".. /input/kerasswa/keras-swa-0.1.2" > /dev/null<feature_engineering>
vc = VotingClassifier(estimators=classifiers)
 vc.fit(X_train, y_train)
 y_pred = vc.predict(X_test)
 acc_vc = accuracy_score(y_test, y_pred)
 print(f'Ensembler Accuracy: {round(acc_vc, 3)}%' )
<define_variables><EOS> 
vc.fit(X, y)
 prediction = vc.predict(df_test)
 submission['Survived'] = prediction
submission.to_csv('Submission.csv', index=False )