Kaggle competition: DonorsChoose.org Application Screening
<SOS> metric: AUC Kaggle data source: donorschoose.org-application-screening<compute_train_metric>
nltk.download("punkt" )
%%capture --no-display
%%capture --no-display
predictions, gbm_fi = model_gbm(train_set, train_labels, test_set, test_ids, return_preds=True )<save_to_csv>
def tokenize_sentences(sentences, words_dict):
tokenized_sentences = []
for sentence in tqdm.tqdm(sentences):
if hasattr(sentence, "decode"):
sentence = sentence.decode("utf-8")
 tokens = nltk.tokenize.word_tokenize(sentence)
 result = []
for word in tokens:
word = word.lower()
if word not in words_dict:
words_dict[word] = len(words_dict)
 word_index = words_dict[word]
result.append(word_index)
 tokenized_sentences.append(result)
 return tokenized_sentences, words_dict
predictions.to_csv('lgb1_submission.csv', index = False )<save_to_csv>
def read_embedding_list(file_path):
embedding_word_dict = {}
embedding_list = []
f = open(file_path)
 for index, line in enumerate(f):
if index == 0:
continue
values = line.split()
word = values[0]
try:
coefs = np.asarray(values[1:], dtype='float32')
 except:
continue
embedding_list.append(coefs)
 embedding_word_dict[word] = len(embedding_word_dict)
 f.close()
embedding_list = np.array(embedding_list)
 return embedding_list, embedding_word_dict
%%capture
submission, gbm_fi, valid_scores = model_gbm(train_set, train_labels,
test_set, test_ids, return_preds=False)
 submission.to_csv('gbm_baseline.csv' )<split>
def clear_embedding_list(embedding_list, embedding_word_dict, words_dict):
cleared_embedding_list = []
cleared_embedding_word_dict = {}
for word in words_dict:
if word not in embedding_word_dict:
continue
word_id = embedding_word_dict[word]
row = embedding_list[word_id]
cleared_embedding_list.append(row)
 cleared_embedding_word_dict[word] = len(cleared_embedding_word_dict)
 return cleared_embedding_list, cleared_embedding_word_dict
%%capture --no-display
submission, gbm_fi_selected, valid_scores_selected = model_gbm(train_selected, train_labels,
test_selected, test_ids)
<create_dataframe>
def convert_tokens_to_ids(tokenized_sentences, words_list, embedding_word_dict, sentences_length):
words_train = []
for sentence in tokenized_sentences:
current_words = []
for word_index in sentence:
word = words_list[word_index]
word_id = embedding_word_dict.get(word, len(embedding_word_dict)- 2)
 current_words.append(word_id)
 if len(current_words)>= sentences_length:
current_words = current_words[:sentences_length]
else:
current_words += [len(embedding_word_dict)- 1] *(sentences_length - len(current_words))
words_train.append(current_words)
 return words_train
model_results = model_results.append(pd.DataFrame({'model': ["GBM_10Fold", "GBM_10Fold_SEL"],
'cv_mean': [valid_scores.mean() , valid_scores_selected.mean() ],
'cv_std': [valid_scores.std() , valid_scores_selected.std() ]}),
sort = True )<compute_train_metric>
from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint
from keras.engine import Layer
from keras.layers import Activation, Add, Bidirectional, Conv1D, Dense, Dropout, Embedding, Flatten
from keras.layers import concatenate, GRU, Input, K, LSTM, MaxPooling1D
from keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D, SpatialDropout1D
from keras.models import Model
from keras.optimizers import Adam
from keras.preprocessing import text, sequence
from sklearn.metrics import accuracy_score, roc_auc_score, log_loss
from sklearn.model_selection import train_test_split
from keras import initializers, regularizers, constraints, optimizers, layers, callbacks
%%capture
submission, gbm_fi, valid_scores = model_gbm(train_set, train_labels, test_set, test_ids,
nfolds=10, return_preds=False )<save_to_csv>
gru_len = 128
Routings = 5
Num_capsule = 10
Dim_capsule = 16
dropout_p = 0.3
rate_drop_dense = 0.3
submission.to_csv('gbm_10fold.csv', index = False)
<compute_train_metric>
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
%%capture
submission, gbm_fi_selected, valid_scores_selected = model_gbm(train_selected, train_labels, test_selected, test_ids,
nfolds=10 )<save_to_csv>
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer='glorot_uniform',
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(K.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = K.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule )
submission.to_csv('gmb_10fold_selected.csv', index = False)
<create_dataframe>
def get_model(embedding_matrix, sequence_length, dropout_rate, recurrent_units, dense_size):
input1 = Input(shape=(sequence_length,))
embed_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],
weights=[embedding_matrix], trainable=False )(input1)
 embed_layer = SpatialDropout1D(rate_drop_dense )(embed_layer)
 x = Bidirectional(
 GRU(gru_len, activation='relu', dropout=dropout_p, recurrent_dropout=dropout_p, return_sequences=True))(
 embed_layer)
 capsule = Capsule(num_capsule=Num_capsule, dim_capsule=Dim_capsule, routings=Routings,
share_weights=True )(x)
 capsule = Flatten()(capsule)
 capsule = Dropout(dropout_p )(capsule)
 output = Dense(1, activation='sigmoid' )(capsule)
 model = Model(inputs=input1, outputs=output)
 model.compile(
 loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
model_results = model_results.append(pd.DataFrame({'model': ["GBM_10Fold", "GBM_10Fold_SEL"],
'cv_mean': [valid_scores.mean() , valid_scores_selected.mean() ],
'cv_std': [valid_scores.std() , valid_scores_selected.std() ]}),
sort = True )<set_options>
def _train_model(model, batch_size, train_x, train_y, val_x, val_y):
num_labels = train_y.shape[1]
patience = 5
best_loss = -1
best_weights = None
best_epoch = 0
current_epoch = 0
while True:
model.fit(train_x, train_y, batch_size=batch_size, epochs=1)
 y_pred = model.predict(val_x, batch_size=batch_size)
 total_loss = 0
for j in range(num_labels):
loss = log_loss(val_y[:, j], y_pred[:, j])
 total_loss += loss
total_loss /= num_labels
print("Epoch {0} loss {1} best_loss {2}".format(current_epoch, total_loss, best_loss))
current_epoch += 1
if total_loss < best_loss or best_loss == -1:
best_loss = total_loss
best_weights = model.get_weights()
best_epoch = current_epoch
else:
if current_epoch - best_epoch == patience:
break
model.set_weights(best_weights)
 return model
%matplotlib inline
warnings.filterwarnings("ignore" )<categorify>
def train_folds(X, y, X_test, fold_count, batch_size, get_model_func):
print("="*75)
 fold_size = len(X)// fold_count
models = []
result_path = "predictions"
if not os.path.exists(result_path):
os.mkdir(result_path)
 for fold_id in range(0, fold_count):
fold_start = fold_size * fold_id
fold_end = fold_start + fold_size
if fold_id == fold_size - 1:
fold_end = len(X)
 train_x = np.concatenate([X[:fold_start], X[fold_end:]])
 train_y = np.concatenate([y[:fold_start], y[fold_end:]])
 val_x = np.array(X[fold_start:fold_end])
 val_y = np.array(y[fold_start:fold_end])
 model = _train_model(get_model_func() , batch_size, train_x, train_y, val_x, val_y)
 train_predicts_path = os.path.join(result_path, "train_predicts{0}.npy".format(fold_id))
test_predicts_path = os.path.join(result_path, "test_predicts{0}.npy".format(fold_id))
train_predicts = model.predict(X, batch_size=512, verbose=1)
 test_predicts = model.predict(X_test, batch_size=512, verbose=1)
 np.save(train_predicts_path, train_predicts)
 np.save(test_predicts_path, test_predicts)
 return models
def encode_data(df):
df['idhogar'] = LabelEncoder().fit_transform(df['idhogar'])
 def feature_importance(forest, X_train, display_results=True):
ranked_list = []
zero_features = []
importances = forest.feature_importances_
indices = np.argsort(importances)[::-1]
if display_results:
print("Feature ranking:")
 for f in range(X_train.shape[1]):
if display_results:
print("%d.feature %d(%f)" %(f + 1, indices[f], importances[indices[f]])+ " - " + X_train.columns[indices[f]])
 ranked_list.append(X_train.columns[indices[f]])
 if importances[indices[f]] == 0.0:
zero_features.append(X_train.columns[indices[f]])
 return ranked_list, zero_features<feature_engineering>
train_file_path = ".. /input/donorschooseorg-preprocessed-data/train_small.csv"
test_file_path = ".. /input/donorschooseorg-preprocessed-data/test_small.csv"
embedding_path = ".. /input/donorschooseorg-preprocessed-data/embeddings_small.vec"
batch_size = 128
recurrent_units = 16
dropout_rate = 0.3
dense_size = 8
sentences_length = 10
fold_count = 2
def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
df.drop(['Id'], axis=1, inplace=True)
 return df<categorify>
UNKNOWN_WORD = "_UNK_"
END_WORD = "_END_"
NAN_WORD = "_NAN_"
CLASSES = ["project_is_approved"]
def convert_OHE2LE(df):
tmp_df = df.copy(deep=True)
 for s_ in ['pared', 'piso', 'techo', 'abastagua', 'sanitario', 'energcocinar', 'elimbasu',
'epared', 'etecho', 'eviv', 'estadocivil', 'parentesco',
'instlevel', 'lugar', 'tipovivi',
'manual_elec']:
if 'manual_' not in s_:
cols_s_ = [f_ for f_ in df.columns if f_.startswith(s_)]
elif 'elec' in s_:
cols_s_ = ['public', 'planpri', 'noelec', 'coopele']
sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print('The OHE in {} is incomplete.A new column will be added before label encoding' 
.format(s_))
col_dummy = s_+'_dummy'
tmp_df[col_dummy] =(tmp_df[cols_s_].sum(axis=1)== 0 ).astype(np.int8)
 cols_s_.append(col_dummy)
 sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print("The category completion did not work")
 tmp_cat = tmp_df[cols_s_].idxmax(axis=1)
 tmp_df[s_ + '_LE'] = LabelEncoder().fit_transform(tmp_cat ).astype(np.int16)
 if 'parentesco1' in cols_s_:
cols_s_.remove('parentesco1')
 tmp_df.drop(cols_s_, axis=1, inplace=True)
 return tmp_df<load_from_csv>
print("Loading data...")
 train_data = pd.read_csv(train_file_path)
 test_data = pd.read_csv(test_file_path)
 list_sentences_train = train_data["application_text"].fillna(NAN_WORD ).values
list_sentences_test = test_data["application_text"].fillna(NAN_WORD ).values
y_train = train_data[CLASSES].values
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 test_ids = test.Id<categorify>
print("Tokenizing sentences in train set...")
 tokenized_sentences_train, words_dict = tokenize_sentences(list_sentences_train, {})
 print("Tokenizing sentences in test set...")
 tokenized_sentences_test, words_dict = tokenize_sentences(list_sentences_test, words_dict )
def process_df(df_):
encode_data(df_)
 return do_features(df_)
 train = process_df(train)
 test = process_df(test )<feature_engineering>
words_dict[UNKNOWN_WORD] = len(words_dict)
 print("Loading embeddings...")
 embedding_list, embedding_word_dict = read_embedding_list(embedding_path)
 embedding_size = len(embedding_list[0] )
train['dependency'] = np.sqrt(train['SQBdependency'])
 test['dependency'] = np.sqrt(test['SQBdependency'])
 train.loc[train['edjefa'] == "no", "edjefa"] = 0
train.loc[train['edjefe'] == "no", "edjefe"] = 0
test.loc[test['edjefa'] == "no", "edjefa"] = 0
test.loc[test['edjefe'] == "no", "edjefe"] = 0
train.loc[(train['edjefa'] == "yes")&(train['parentesco1'] == 1), "edjefa"] = train.loc[(train['edjefa'] == "yes")&(train['parentesco1'] == 1), "escolari"]
train.loc[(train['edjefe'] == "yes")&(train['parentesco1'] == 1), "edjefe"] = train.loc[(train['edjefe'] == "yes")&(train['parentesco1'] == 1), "escolari"]
test.loc[(test['edjefa'] == "yes")&(test['parentesco1'] == 1), "edjefa"] = test.loc[(test['edjefa'] == "yes")&(test['parentesco1'] == 1), "escolari"]
test.loc[(test['edjefe'] == "yes")&(test['parentesco1'] == 1), "edjefe"] = test.loc[(test['edjefe'] == "yes")&(test['parentesco1'] == 1), "escolari"]
train.loc[train['edjefa'] == "yes", "edjefa"] = 4
train.loc[train['edjefe'] == "yes", "edjefe"] = 4
test.loc[test['edjefa'] == "yes", "edjefa"] = 4
test.loc[test['edjefe'] == "yes", "edjefe"] = 4
train['edjefe'] = train['edjefe'].astype("int")
 train['edjefa'] = train['edjefa'].astype("int")
 test['edjefe'] = test['edjefe'].astype("int")
 test['edjefa'] = test['edjefa'].astype("int")
 train['edjef'] = np.max(train[['edjefa','edjefe']], axis=1)
 test['edjef'] = np.max(test[['edjefa','edjefe']], axis=1)
 train['v2a1']=train['v2a1'].fillna(0)
 test['v2a1']=test['v2a1'].fillna(0)
 test['v18q1']=test['v18q1'].fillna(0)
 train['v18q1']=train['v18q1'].fillna(0)
 train['rez_esc']=train['rez_esc'].fillna(0)
 test['rez_esc']=test['rez_esc'].fillna(0)
 train.loc[train.meaneduc.isnull() , "meaneduc"] = 0
train.loc[train.SQBmeaned.isnull() , "SQBmeaned"] = 0
test.loc[test.meaneduc.isnull() , "meaneduc"] = 0
test.loc[test.SQBmeaned.isnull() , "SQBmeaned"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "v14a"] = 0
train.loc[(train.v14a == 1)&(train.sanitario1 == 1)&(train.abastaguano == 0), "sanitario1"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "v14a"] = 0
test.loc[(test.v14a == 1)&(test.sanitario1 == 1)&(test.abastaguano == 0), "sanitario1"] = 0<concatenate>
print("Preparing data...")
 embedding_list, embedding_word_dict = clear_embedding_list(embedding_list, embedding_word_dict, words_dict)
 embedding_word_dict[UNKNOWN_WORD] = len(embedding_word_dict)
 embedding_list.append([0.] * embedding_size)
 embedding_word_dict[END_WORD] = len(embedding_word_dict)
 embedding_list.append([-1.] * embedding_size)
 embedding_matrix = np.array(embedding_list)
 id_to_word = dict(( id, word)for word, id in words_dict.items())
 train_list_of_token_ids = convert_tokens_to_ids(
 tokenized_sentences_train,
id_to_word,
embedding_word_dict,
sentences_length)
 test_list_of_token_ids = convert_tokens_to_ids(
 tokenized_sentences_test,
id_to_word,
embedding_word_dict,
sentences_length)
 X_train = np.array(train_list_of_token_ids)
 X_test = np.array(test_list_of_token_ids )
def train_test_apply_func(train_, test_, func_):
test_['Target'] = 0
xx = pd.concat([train_, test_])
 xx_func = func_(xx)
 train_ = xx_func.iloc[:train_.shape[0], :]
test_ = xx_func.iloc[train_.shape[0]:, :].drop('Target', axis=1)
 del xx, xx_func
return train_, test_<categorify>
get_model_func = lambda: get_model(
 embedding_matrix,
sentences_length,
dropout_rate,
recurrent_units,
dense_size )
train, test = train_test_apply_func(train, test, convert_OHE2LE )<categorify>
del train_data, test_data, list_sentences_train, list_sentences_test
del tokenized_sentences_train, tokenized_sentences_test, words_dict
del embedding_list, embedding_word_dict
del train_list_of_token_ids, test_list_of_token_ids
gc.collect() ;
cols_2_ohe = ['eviv_LE', 'etecho_LE', 'epared_LE', 'elimbasu_LE',
'energcocinar_LE', 'sanitario_LE', 'manual_elec_LE',
'pared_LE']
cols_nums = ['age', 'meaneduc', 'dependency',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total',
'bedrooms', 'overcrowding']
def convert_geo2aggs(df_):
tmp_df = pd.concat([df_[(['lugar_LE', 'idhogar']+cols_nums)],
pd.get_dummies(df_[cols_2_ohe],
columns=cols_2_ohe)],axis=1)
 geo_agg = tmp_df.groupby(['lugar_LE','idhogar'] ).mean().groupby('lugar_LE' ).mean().astype(np.float32)
 geo_agg.columns = pd.Index(['geo_' + e for e in geo_agg.columns.tolist() ])
 del tmp_df
return df_.join(geo_agg, how='left', on='lugar_LE')
 train, test = train_test_apply_func(train, test, convert_geo2aggs )<categorify>
print("Starting to train models...")
 models = train_folds(X_train, y_train, X_test, fold_count, batch_size, get_model_func )
train['num_over_18'] = 0
train['num_over_18'] = train[train.age >= 18].groupby('idhogar' ).transform("count")
 train['num_over_18'] = train.groupby("idhogar")["num_over_18"].transform("max")
 train['num_over_18'] = train['num_over_18'].fillna(0)
 test['num_over_18'] = 0
test['num_over_18'] = test[test.age >= 18].groupby('idhogar' ).transform("count")
 test['num_over_18'] = test.groupby("idhogar")["num_over_18"].transform("max")
 test['num_over_18'] = test['num_over_18'].fillna(0)
 def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
df['r4t3_to_tamhog'] = df['r4t3']/df['tamhog']
df['r4t3_to_rooms'] = df['r4t3']/df['rooms']
df['v2a1_to_r4t3'] = df['v2a1']/df['r4t3']
df['v2a1_to_r4t3'] = df['v2a1']/(df['r4t3'] - df['r4t1'])
 df['hhsize_to_rooms'] = df['hhsize']/df['rooms']
df['rent_to_hhsize'] = df['v2a1']/df['hhsize']
df['rent_to_over_18'] = df['v2a1']/df['num_over_18']
df.loc[df.num_over_18 == 0, "rent_to_over_18"] = df[df.num_over_18 == 0].v2a1
extract_features(train)
 extract_features(test )<drop_column>
LABELS = ["project_is_approved"]
base = ".. /input/donorschooseorg-application-screening-predictions/predictions/predictions/"
predict_list = []
for j in range(10):
predict_list.append(np.load(base + "predictions_001/test_predicts%d.npy"%j))
print("Rank averaging on ", len(predict_list), " files")
 predcitions = np.zeros_like(predict_list[0])
 for predict in predict_list:
predcitions = np.add(predcitions.flatten() , rankdata(predict)/predcitions.shape[0])
 predcitions /= len(predict_list)
 submission = pd.read_csv('.. /input/donorschoose-application-screening/sample_submission.csv')
 submission[LABELS] = predcitions
submission.to_csv('submission.csv', index=False )