Kaggle competition: DonorsChoose.org Application Screening
<SOS> metric: AUC Kaggle data source: donorschoose.org-application-screening<choose_model_class>
quick_run = False
max_features = 80000
embed_size = 300
pj_repeat = 3
rs_repeat = 1
dpcnn_folds = 5
batch_size = 64
epochs = 5
project_maxlen = 210
resouse_max_len = 30
maxlen = project_maxlen + resouse_max_len
if quick_run == True:
max_features = 1000
epochs = 2
EMBEDDING_FILE = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
def model() :
model = Sequential()
model.add(Conv2D(32, kernel_size = 3, activation='relu', input_shape =(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Conv2D(128, kernel_size = 4, activation='relu'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
adam = tf.keras.optimizers.Adam()
model.compile(optimizer=adam, loss="categorical_crossentropy", metrics=["accuracy"])
 early = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=50)
 checkpoint_path = 'training_1/cp.ckpt'
checkpoint_dir = os.path.dirname(checkpoint_path)
 cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path,
save_weights_only=True,
verbose=1)
 return model<choose_model_class>
warnings.filterwarnings('ignore' )
def model() :
model = Sequential()
model.add(Conv2D(32, kernel_size = 3, activation='relu', input_shape =(28, 28, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Conv2D(128, kernel_size = 4, activation='relu'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
adam = tf.keras.optimizers.Adam(
 learning_rate=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-07,
amsgrad=False)
 model.compile(optimizer=adam, loss="categorical_crossentropy", metrics=["accuracy"])
 early = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=50)
 checkpoint_path = 'training_1/cp.ckpt'
checkpoint_dir = os.path.dirname(checkpoint_path)
 cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path,
save_weights_only=True,
verbose=1)
 print(model.summary())
 history = model.fit(xtrain, ytrain, epochs=100, callbacks=[cp_callback, early], validation_data=(xval, yval))
prediction = model.predict(test)
 prediction = np.argmax(prediction, axis=1)
 return history, prediction<find_best_params>
train_df = pd.read_csv('.. /input/donorschoose-application-screening/train.csv')
 test_df = pd.read_csv('.. /input/donorschoose-application-screening/test.csv')
 resouse_df = pd.read_csv('.. /input/donorschoose-application-screening/resources.csv' )
history, prediction = model()<save_to_csv>
%%time
resouse_df['description'].fillna('', inplace=True)
 res_nums = pd.DataFrame(resouse_df[['id', 'price']].groupby('id' ).price.agg(['count',
'sum',
'min',
'max',
'mean',
'std',
lambda x: len(np.unique(x)) ,])).reset_index()
res_nums = res_nums.rename(columns={'count': 'res_count',
'sum': 'res_sum',
'min': 'res_min',
'max': 'res_max',
'mean': 'res_mean',
'std': 'res_std',
'<lambda>': 'res_unique' })
 res_descp = resouse_df[['id', 'description']].groupby('id' ).description.agg([ lambda x: ' '.join(x)] ).reset_index().rename(columns={'<lambda>':'res_description'})
 resouse_df = res_nums.merge(res_descp, on='id', how='left')
 train_df = train_df.merge(resouse_df, on='id', how='left')
 test_df = test_df.merge(resouse_df, on='id', how='left')
 del res_nums
del res_descp
del resouse_df
gc.collect()
data = {"ImageId": image_id, "Label":prediction}
results = pd.DataFrame(data)
 results.to_csv("result.csv",index=False )<import_modules>
if quick_run == True:
train_df = train_df[:10000]
test_df = test_df[:100]
from keras.preprocessing.image import ImageDataGenerator<train_model>
train_target = train_df['project_is_approved'].values
gc.collect()
datagen_train = datagen_valid = ImageDataGenerator(
 featurewise_center = False,
samplewise_center = False,
featurewise_std_normalization = False,
samplewise_std_normalization = False,
zca_whitening = False,
horizontal_flip = False,
vertical_flip = False,
fill_mode = 'nearest',
rotation_range = 10,
zoom_range = 0.1,
width_shift_range = 0.1,
height_shift_range = 0.1)
 datagen_train.fit(xtrain)
 train_gen = datagen_train.flow(xtrain, ytrain, batch_size=64)
 datagen_valid.fit(xval)
 valid_gen = datagen_valid.flow(xval, yval, batch_size=64 )<choose_model_class>
essay_cols = ['project_essay_1', 'project_essay_2','project_essay_3', 'project_essay_4']
essay_length_cols = [item+'_len' for item in essay_cols]
def count_essay_length(df):
for col in essay_cols:
df[col] = df[col].fillna('unknown')
 df[col+'_len'] = df[col].apply(len)
 return df
train_df = count_essay_length(train_df)
 test_df = count_essay_length(test_df)
 train_df['project_essay'] = ''
test_df['project_essay'] = ''
for col in essay_cols:
train_df['project_essay'] += train_df[col] + ' '
test_df['project_essay'] += test_df[col] + ' '
train_df[['project_essay']].head()
model = Sequential([
Conv2D(32, kernel_size=(3, 3), activation='relu', padding='same', input_shape =(28,28,1)) ,
Conv2D(32, kernel_size=(3, 3), activation='relu'),
BatchNormalization() ,
MaxPool2D(pool_size=(2, 2)) ,
Dropout(0.2),
Conv2D(64, kernel_size=(3, 3), activation='relu', padding='same'),
Conv2D(64, kernel_size=(3, 3), activation='relu'),
BatchNormalization() ,
MaxPool2D(pool_size=(2, 2)) ,
Dropout(0.2),
Conv2D(128, kernel_size=(3, 3), activation='relu', padding='same'),
Conv2D(128, kernel_size=(3, 3), activation='relu'),
BatchNormalization() ,
MaxPool2D(pool_size=(2, 2)) ,
Dropout(0.2),
Flatten() ,
Dense(512, activation='relu'),
Dropout(0.5),
Dense(10, activation = "softmax")
 ])
 adam = tf.keras.optimizers.Adam(
 learning_rate=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-07,
amsgrad=False)
 model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])
 early = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=50)
 checkpoint_path = 'training_1/cp.ckpt'
checkpoint_dir = os.path.dirname(checkpoint_path)
 cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path,
save_weights_only=True,
verbose=1)
 print(model.summary())
 history = model.fit(( train_gen), epochs=100, callbacks=[cp_callback, early], validation_data=(valid_gen))
prediction = model.predict(test)
 prediction = np.argmax(prediction, axis=1 )<save_to_csv>
time_cols = ['sub_year', 'sub_month', 'sub_day', 'sub_hour', 'sub_dayofweek', 'sub_dayofyear']
def time_stamp_features(df):
time_df = pd.to_datetime(df['project_submitted_datetime'])
 df['sub_year'] = time_df.apply(lambda x: x.year)
 df['sub_month'] = time_df.apply(lambda x: x.month)
 df['sub_day'] = time_df.apply(lambda x: x.day)
 df['sub_hour'] = time_df.apply(lambda x: x.hour)
 df['sub_dayofweek'] = time_df.apply(lambda x: x.dayofweek)
 df['sub_dayofyear'] = time_df.apply(lambda x: x.dayofyear)
 return df
train_df = time_stamp_features(train_df)
 test_df = time_stamp_features(test_df )
data = {"ImageId": image_id, "Label":prediction}
results = pd.DataFrame(data)
 results.to_csv("result_data_generator.csv",index=False )<load_from_csv>
str_cols = ['teacher_id', 'teacher_prefix', 'school_state',
'project_submitted_datetime', 'project_grade_category',
'project_subject_categories', 'project_subject_subcategories',
'project_title', 'project_resource_summary','res_description', 'project_essay']
num_cols = ['teacher_number_of_previously_posted_projects',
'res_count', 'res_sum', 'res_min', 'res_max', 'res_mean', 'res_std', 'res_unique'] + essay_length_cols
train_df[str_cols] =train_df[str_cols].fillna('unknown')
 train_df[num_cols] = train_df[num_cols].fillna(0)
 test_df[str_cols] =test_df[str_cols].fillna('unknown')
 test_df[num_cols] = test_df[num_cols].fillna(0)
 for col in str_cols:
train_df[col] = train_df[col].str.lower()
test_df[col] = test_df[col].str.lower()
mnist_test = pd.read_csv(".. /input/mnist-in-csv/mnist_test.csv")
 mnist_train = pd.read_csv(".. /input/mnist-in-csv/mnist_train.csv" )<load_from_csv>
std_scaler = MinMaxScaler()
train_none_text_features = std_scaler.fit_transform(train_df[num_cols].values)
 test_none_text_features = std_scaler.transform(test_df[num_cols].values)
 train_df = train_df.drop(num_cols, axis=1)
 test_df = test_df.drop(num_cols, axis=1)
 del std_scaler
gc.collect()
sample_submission = pd.read_csv(".. /input/digit-recognizer/sample_submission.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv")
 train = pd.read_csv(".. /input/digit-recognizer/train.csv" )<feature_engineering>
train_df['project_descp'] = train_df['project_subject_categories'] + ' ' + train_df['project_subject_subcategories'] + ' ' + train_df['project_title'] + ' ' + train_df['project_resource_summary'] + ' ' + train_df['project_essay']
test_df['project_descp'] = test_df['project_subject_categories'] + ' ' + test_df['project_subject_subcategories'] + ' ' + test_df['project_title'] + ' ' + test_df['project_resource_summary'] + ' ' + test_df['project_essay']
train_df = train_df.drop([ 'project_essay'], axis=1)
 test_df = test_df.drop(['project_essay'], axis=1)
 gc.collect()
test['dataset'] = 'test'<feature_engineering>
label_cols = [
'teacher_prefix',
'school_state',
'project_grade_category',
'project_subject_categories',
'project_subject_subcategories'
] + time_cols
def mean_encoding(train_df, test_df, col):
gp = train_df.groupby(col)['project_is_approved'].mean().reset_index().rename(columns={'project_is_approved': col+'_mean'})
 train_df = pd.merge(train_df, gp, how='left', on=[col])
 test_df = pd.merge(test_df, gp, how='left', on=[col])
 return train_df, test_df
for col in label_cols:
train_df, test_df = mean_encoding(train_df, test_df, col)
 label_mean_cols = [item+'_mean' for item in label_cols]
train_label_features = train_df[label_mean_cols].values
test_df[label_mean_cols] = test_df[label_mean_cols].fillna(test_df[label_mean_cols].mean())
 test_label_features = test_df[label_mean_cols].values

 train_df = train_df.drop(label_cols, axis=1)
 test_df = test_df.drop(label_cols, axis=1)
 train_df = train_df.drop(label_mean_cols, axis=1)
 test_df = test_df.drop(label_mean_cols, axis=1)
 gc.collect()
train['dataset'] = 'train'<concatenate>
%%time
def clean_descp(descp):
low_case = re.compile('([a-z]*)')
 words = low_case.findall(descp)
 return ' '.join(words)
 for col in essay_cols:
train_df[col] = train_df[col].apply(clean_descp)
 test_df[col] = test_df[col].apply(clean_descp)
 train_df['project_descp'] = train_df['project_descp'].apply(clean_descp)
 test_df['project_descp'] = test_df['project_descp'].apply(clean_descp )
dataset = pd.concat([train.drop('label', axis=1), test] ).reset_index()<concatenate>
def getTextFeatures(Col, max_features=10000):
print('processing: ', Col)
 vectorizer = CountVectorizer(stop_words=None,
max_features=max_features,
binary=True,
ngram_range=(1,2))
train_features = vectorizer.fit_transform(train_df[Col])
 test_features = vectorizer.transform(test_df[Col])
 return train_features, test_features
train_text_es1, test_text_es1 = getTextFeatures('project_essay_1', max_features=5000)
 train_text_es2, test_text_es2 = getTextFeatures('project_essay_2', max_features=7000)
 train_text_prs, test_text_prs = getTextFeatures('project_resource_summary', max_features=1000)
 train_text_rsd, test_text_rsd = getTextFeatures('res_description', max_features=1000)
 train_text_pjt, test_text_pjt = getTextFeatures('project_title', max_features=500 )
mnist = pd.concat([mnist_train, mnist_test] ).reset_index(drop=True)
 labels = mnist['label'].values
mnist.drop('label', axis=1, inplace=True)
 mnist.columns = cols<sort_values>
train_text_features = hstack([train_text_es1, train_text_es2, train_text_prs, train_text_rsd, train_text_pjt] ).tocsr()
test_text_features = hstack([test_text_es1, test_text_es2, test_text_prs, test_text_rsd, test_text_pjt] ).tocsr()
train_text_features.shape, test_text_features.shape
idx_mnist = mnist.sort_values(by=list(mnist.columns)).index
dataset_from = dataset.sort_values(by=list(mnist.columns)) ['dataset'].values
original_idx = dataset.sort_values(by=list(mnist.columns)) ['index'].values<feature_engineering>
train_features = hstack([train_none_text_features, train_label_features, train_text_features] ).tocsr()
test_features = hstack([test_none_text_features, test_label_features, test_text_features] ).tocsr()
del train_text_es1
del train_text_es2
del train_text_prs
del train_text_rsd
del train_text_pjt
del test_text_es1
del test_text_es2
del test_text_prs
del test_text_rsd
del test_text_pjt
del train_text_features
del test_text_features
gc.collect()
for i in range(len(idx_mnist)) :
if dataset_from[i] == 'test':
sample_submission.loc[original_idx[i], 'Label'] = labels[idx_mnist[i]]<save_to_csv>
class qiaofeng_kfold_stack:
def __init__(self, train, train_target, test, split_target, model, preprocess_func=None, score_func=None, kfolds=5, random_seed=9527, logger=None):
self.train = train
self.train_target = train_target
self.test = test
self.split_target = split_target
self.model = model
self.preprocess_func = preprocess_func
self.score_func = score_func
self.kfolds = kfolds
self.random_seed = random_seed
self.logger= logger
self.skf = KFold(n_splits=self.kfolds, random_state= self.random_seed)
 self.predict_test_kfolds = []
self.predict_valid_kfolds = np.zeros(( self.train.shape[0]))
def print_params(self):
print('kfolds : %s' % self.kfolds)
 print('random seed : %s' % self.random_seed)
 def preprocess(self):
if self.preprocess_func != None:
self.train, self.test = self.preprocess_func(self.train, self.test)
 def score(self, target, predict):
return self.score_func(target, predict)
 def model_fit(self, train, train_target):
self.model.fit(train, train_target)
 def model_predict(self, dataset):
return self.model.predict(dataset)
 def model_fit_predict(self, train, train_target, dataset):
self.model_fit(train, train_target)
 predict_train = None
predict_valid = self.model_predict(dataset)
 predict_test = self.model_predict(self.test)
 return predict_train, predict_valid, predict_test
def clear_predicts(self):
self.predict_test_kfolds = []
self.predict_valid_kfolds = np.zeros(( self.train.shape[0]))
def model_train_with_kfold(self):
self.clear_predicts()
for(folder_index,(train_index, valid_index)) in enumerate(self.skf.split(self.train)) :
x_train, x_valid = self.train[train_index], self.train[valid_index]
y_train, y_valid = self.train_target[train_index], self.train_target[valid_index]
predict_train, predict_valid, predict_test = self.model_fit_predict(x_train, y_train, x_valid)
 self.predict_test_kfolds.append(predict_test)
 self.predict_valid_kfolds[valid_index] = predict_valid
if self.logger != None:
valid_score = self.score(y_valid, predict_valid)
 self.logger('Fold: %s, valid score: %s' %(folder_index, valid_score))
def predict_test_mean(self):
return reduce(lambda x,y:x+y, self.predict_test_kfolds)/ self.kfolds
sample_submission.to_csv('submission.csv', index=False )<import_modules>
class qiaofeng_predict_prob(qiaofeng_kfold_stack):
def model_predict(self, dataset):
return self.model.predict_proba(dataset)[:,1]
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D
from keras.utils import np_utils
from keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import seaborn as sns<load_from_csv>
%%time
model = RandomForestClassifier(n_jobs=4,
criterion="entropy",
max_depth=20,
n_estimators=100,
max_features='sqrt',
random_state=233, 
)
 qiaofeng_rf = qiaofeng_predict_prob(train=train_features, train_target=train_target, test=test_features, kfolds=5,split_target=train_target,
score_func=roc_auc_score, logger=print, model=model)
 qiaofeng_rf.model_train_with_kfold()
pred_valid_rf = qiaofeng_rf.predict_valid_kfolds
pred_test_avg_rf = qiaofeng_rf.predict_test_mean()
del qiaofeng_rf
gc.collect()
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test = pd.read_csv('.. /input/digit-recognizer/test.csv' )<prepare_x_and_y>
%%time
class qiaofeng_lgb_clf(qiaofeng_kfold_stack):
def model_fit(self, train, train_target):
X_tra, X_val, y_tra, y_val = train_test_split(train, train_target, train_size=0.95, random_state=233)
 self.model = lgb.LGBMClassifier(n_jobs=4,
max_depth=4,
metric="auc",
n_estimators=1000,
num_leaves=15,
boosting_type="gbdt",
learning_rate=0.05,
feature_fraction=0.45,
colsample_bytree=0.45,
bagging_fraction=0.8,
bagging_freq=5,
reg_lambda=0.2)
 self.model.fit(X=X_tra, y=y_tra,
eval_set=[(X_val, y_val)],
early_stopping_rounds=200,
verbose=False)
 def model_predict(self, dataset):
return self.model.predict_proba(dataset)[:,1]
qiaofeng_lgb = qiaofeng_lgb_clf(train=train_features, train_target=train_target, test=test_features, kfolds=5,split_target=train_target,
score_func=roc_auc_score, logger=print, model=None)
 qiaofeng_lgb.model_train_with_kfold()
pred_valid_lgb = qiaofeng_lgb.predict_valid_kfolds
pred_test_avg_lgb = qiaofeng_lgb.predict_test_mean()
del qiaofeng_lgb
gc.collect()
Y_train = train["label"]
X_train = train.drop(labels = ["label"],axis = 1 )<feature_engineering>
%%time
class qiaofeng_lgb_clf(qiaofeng_kfold_stack):
def model_fit(self, train, train_target):
X_tra, X_val, y_tra, y_val = train_test_split(train, train_target, train_size=0.95, random_state=233)
 self.model = lgb.LGBMClassifier(n_jobs=4,
max_depth=4,
metric="auc",
n_estimators=1000,
num_leaves=15,
boosting_type="gbdt",
learning_rate=0.05,
feature_fraction=0.45,
colsample_bytree=0.45,
bagging_fraction=0.8,
bagging_freq=5,
reg_lambda=0.2)
 self.model.fit(X=X_tra, y=y_tra,
eval_set=[(X_val, y_val)],
early_stopping_rounds=200,
verbose=False)
 def model_predict(self, dataset):
return self.model.predict_proba(dataset)[:,1]
qiaofeng_lgb = qiaofeng_lgb_clf(train=train_features.astype(np.bool ).astype(np.float32),
train_target=train_target,
test=test_features.astype(np.bool ).astype(np.float32),
kfolds=5,split_target=train_target,
score_func=roc_auc_score,
logger=print, model=None)
 qiaofeng_lgb.model_train_with_kfold()
pred_valid_lgb_bool = qiaofeng_lgb.predict_valid_kfolds
pred_test_avg_lgb_bool = qiaofeng_lgb.predict_test_mean()
del qiaofeng_lgb
gc.collect()
X_train = X_train / 255.0
test = test / 255.0<categorify>
pd.Series(pred_valid_lgb ).corr(pd.Series(pred_valid_lgb_bool)) , pd.Series(pred_test_avg_lgb ).corr(pd.Series(pred_test_avg_lgb_bool))
Y_train = np_utils.to_categorical(Y_train, num_classes = 10 )<split>
clf = FTRL(alpha=0.01, beta=0.1, L1=0.1, L2=1000.0, D=train_features.shape[1], iters=1,
inv_link="identity", threads=4)
 X_tra, X_val, y_tra, y_val = train_test_split(train_features, train_target, train_size=0.95, random_state=7)
 train_scores = []
valid_scores = []
for i in range(10):
clf.fit(X_tra, y_tra)
 train_predict = clf.predict(X_tra)
 tp = train_predict - train_predict.min()
tp = tp / tp.max()
valid_predict = clf.predict(X_val)
 vp = valid_predict - valid_predict.min()
vp = vp / vp.max()
train_auc = roc_auc_score(y_tra, tp)
 valid_auc = roc_auc_score(y_val, vp)
 train_scores.append(train_auc)
 valid_scores.append(valid_auc)
 print(i+1, train_auc, valid_auc)
 plt.clf()
plt.figure(figsize=(10,10))
plt.plot(np.arange(len(train_scores)) , train_scores, label='train')
 plt.plot(np.arange(len(valid_scores)) , valid_scores, label='valid')
 valid_max = max(valid_scores)
 plt.title('%s - %s'%(valid_scores.index(valid_max)+1, valid_max))
plt.legend()
plt.show()
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1 )<choose_model_class>
%%time
def sigmoid(x):
return 1 /(1 + np.exp(-x))
class qiaofeng_ftrl(qiaofeng_kfold_stack):
def model_fit(self, train, train_target):
self.model = FTRL(alpha=0.01, beta=0.1, L1=0.1, L2=1000.0, D=train_features.shape[1], iters=2,
inv_link="identity", threads=4)
 self.model.fit(train, train_target)
 def model_predict(self, dataset):
predict = self.model.predict(dataset)
 pred_nan = np.isnan(predict)
 if pred_nan.shape[0] == predict.shape[0]:
predict[pred_nan] = 0
else:
predict[pred_nan] = np.nanmean(predict)
 return sigmoid(predict)
 qiaofeng_ftrl = qiaofeng_ftrl(train=train_features, train_target=train_target, test=test_features, kfolds=20, split_target=train_target,
score_func=roc_auc_score, logger=print, model=None)
 qiaofeng_ftrl.model_train_with_kfold()
pred_valid_ftrl = qiaofeng_ftrl.predict_valid_kfolds
pred_test_avg_ftrl = qiaofeng_ftrl.predict_test_mean()
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))<choose_model_class>
del train_features
del test_features
gc.collect()
model.compile(optimizer = 'Adam' , loss = "categorical_crossentropy", metrics=["accuracy"] )<choose_model_class>
tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_df['project_descp'])+ list(test_df['project_descp'])+ list(train_df['res_description'])+ list(test_df['res_description']))
train_pj = sequence.pad_sequences(tokenizer.texts_to_sequences(train_df['project_descp']), maxlen=project_maxlen)
 test_pj = sequence.pad_sequences(tokenizer.texts_to_sequences(test_df['project_descp']), maxlen=project_maxlen)
 train_res = sequence.pad_sequences(tokenizer.texts_to_sequences(train_df['res_description']), maxlen=resouse_max_len)
 test_res = sequence.pad_sequences(tokenizer.texts_to_sequences(test_df['res_description']), maxlen=resouse_max_len)
 train_seq = np.hstack([train_pj, train_res])
 test_seq = np.hstack([test_pj, test_res] )
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<define_variables>
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE))
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 logs['roc_auc_val'] = score
print("
ROC-AUC - epoch: %d - score: %.6f
" %(epoch+1, score))
epochs = 20
batch_size = 86<init_hyperparams>
train_num_features = np.hstack([train_none_text_features, train_label_features])
 test_num_features = np.hstack([test_none_text_features, test_label_features])
 del train_none_text_features
del train_label_features
del test_none_text_features
del test_label_features
gc.collect()
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
<train_model>
train_seq = np.hstack([train_seq, train_num_features])
 test_seq = np.hstack([test_seq, test_num_features] )
datagen.fit(X_train )<import_modules>
gc.collect()
gc.disable()
from keras.callbacks import ReduceLROnPlateau 
<choose_model_class>
if 1:
def get_model() :
session_conf = tf.ConfigProto(intra_op_parallelism_threads=4, inter_op_parallelism_threads=4)
 K.set_session(tf.Session(graph=tf.get_default_graph() , config=session_conf))
filter_nr = 32
filter_size = 3
max_pool_size = 3
max_pool_strides = 2
dense_nr = 64
spatial_dropout = 0.2
dense_dropout = 0.05
train_embed = False
project = Input(shape=(project_maxlen,), name='project')
 emb_project = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=train_embed )(project)
 emb_project = SpatialDropout1D(spatial_dropout )(emb_project)
 pj_block1 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear' )(emb_project)
 pj_block1 = BatchNormalization()(pj_block1)
 pj_block1 = PReLU()(pj_block1)
 pj_block1 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear' )(pj_block1)
 pj_block1 = BatchNormalization()(pj_block1)
 pj_block1 = PReLU()(pj_block1)
 pj_resize_emb = Conv1D(filter_nr, kernel_size=1, padding='same', activation='linear' )(emb_project)
 pj_resize_emb = PReLU()(pj_resize_emb)
 pj_block1_output = add([pj_block1, pj_resize_emb])
 for _ in range(pj_repeat):
pj_block1_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides )(pj_block1_output)
 pj_block2 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear' )(pj_block1_output)
 pj_block2 = BatchNormalization()(pj_block2)
 pj_block2 = PReLU()(pj_block2)
 pj_block2 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear' )(pj_block2)
 pj_block2 = BatchNormalization()(pj_block2)
 pj_block2 = PReLU()(pj_block2)
 pj_block1_output = add([pj_block2, pj_block1_output])
 resouse = Input(shape=(resouse_max_len,), name='resouse')
 emb_resouse = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=train_embed )(resouse)
 emb_resouse = SpatialDropout1D(spatial_dropout )(emb_resouse)
 rs_block1 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear' )(emb_resouse)
 rs_block1 = BatchNormalization()(rs_block1)
 rs_block1 = PReLU()(rs_block1)
 rs_block1 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear' )(rs_block1)
 rs_block1 = BatchNormalization()(rs_block1)
 rs_block1 = PReLU()(rs_block1)
 rs_resize_emb = Conv1D(filter_nr, kernel_size=1, padding='same', activation='linear' )(emb_resouse)
 rs_resize_emb = PReLU()(rs_resize_emb)
 rs_block1_output = add([rs_block1, rs_resize_emb])
 for _ in range(rs_repeat):
rs_block1_output = MaxPooling1D(pool_size=max_pool_size, strides=max_pool_strides )(rs_block1_output)
 rs_block2 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear' )(rs_block1_output)
 rs_block2 = BatchNormalization()(rs_block2)
 rs_block2 = PReLU()(rs_block2)
 rs_block2 = Conv1D(filter_nr, kernel_size=filter_size, padding='same', activation='linear' )(rs_block2)
 rs_block2 = BatchNormalization()(rs_block2)
 rs_block2 = PReLU()(rs_block2)
 rs_block1_output = add([rs_block2, rs_block1_output])
 pj_output = GlobalMaxPooling1D()(pj_block1_output)
 pj_output = BatchNormalization()(pj_output)
 rs_output = GlobalMaxPooling1D()(rs_block1_output)
 rs_output = BatchNormalization()(rs_output)
 inp_num = Input(shape=(train_seq.shape[1]-maxlen,), name='num_input')
 bn_inp_num = BatchNormalization()(inp_num)
 conc = concatenate([pj_output, rs_output, bn_inp_num])
 output = Dense(dense_nr, activation='linear' )(conc)
 output = BatchNormalization()(output)
 output = PReLU()(output)
 output = Dropout(dense_dropout )(output)
 output = Dense(1, activation='sigmoid' )(output)
 model = Model(inputs=[project, resouse, inp_num], outputs=output)
 model.compile(loss='binary_crossentropy',
optimizer='nadam',
metrics=['accuracy'])
 return model
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<train_model>
class qiaofeng_dpcnn(qiaofeng_kfold_stack):
def model_fit_predict(self, train, train_target, valid):
self.model = get_model()
early_stopping = EarlyStopping(monitor='roc_auc_val', patience=1, mode='max',min_delta=0.0005)
 X_tra, X_val, y_tra, y_val = train_test_split(train, train_target, train_size=0.98, random_state=233)
 X_tra = { 'project' : X_tra[:,:project_maxlen], 'resouse' : X_tra[:,project_maxlen:project_maxlen+resouse_max_len], 'num_input' : X_tra[:,maxlen:] }
X_val = { 'project' : X_val[:,:project_maxlen], 'resouse' : X_val[:,project_maxlen:project_maxlen+resouse_max_len], 'num_input' : X_val[:,maxlen:] }
x_test = { 'project' : self.test[:,:project_maxlen], 'resouse' : self.test[:,project_maxlen:project_maxlen+resouse_max_len], 'num_input' : self.test[:,maxlen:] }
valid = { 'project' : valid[:,:project_maxlen], 'resouse' : valid[:,project_maxlen:project_maxlen+resouse_max_len], 'num_input' : valid[:,maxlen:] }
RocAuc = RocAucEvaluation(validation_data=(X_val, y_val), interval=1)
 hist = self.model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),
callbacks=[RocAuc, early_stopping], verbose=2)
 predict_train = None
predict_valid = self.model.predict(valid, batch_size=1024)[:, 0]
predict_test = self.model.predict(x_test, batch_size=1024)[:, 0]
return predict_train, predict_valid, predict_test
dpcnn_kfold_model = qiaofeng_dpcnn(train=train_seq, train_target=train_target, test=test_seq, kfolds=dpcnn_folds, split_target=train_target,
score_func=roc_auc_score, logger=print, model=None)
 dpcnn_kfold_model.model_train_with_kfold()
pred_valid_dpcnn = dpcnn_kfold_model.predict_valid_kfolds
pred_test_avg_dpcnn = dpcnn_kfold_model.predict_test_mean()
del dpcnn_kfold_model
gc.enable()
gc.collect()
gc.disable()
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )<predict_on_test>
scaler = MinMaxScaler()
predict_valid_list = [pred_valid_rf, pred_valid_lgb, pred_valid_dpcnn, pred_valid_ftrl]
predict_test_list = [pred_test_avg_rf, pred_test_avg_lgb, pred_test_avg_dpcnn, pred_test_avg_ftrl]
valid_results = np.hstack([scaler.fit_transform(item.reshape(( item.shape[0], 1)))for item in predict_valid_list])
 test_results = np.hstack([scaler.fit_transform(item.reshape(( item.shape[0], 1)))for item in predict_test_list])
 train_features = np.hstack([valid_results ])
 test_features = np.hstack([test_results ])
 lgb_model = lgb.LGBMClassifier(n_jobs=4,
max_depth=4,
metric="auc",
n_estimators=400,
num_leaves=10,
boosting_type="gbdt",
learning_rate=0.01,
feature_fraction=0.45,
colsample_bytree=0.45,
bagging_fraction=0.4,
bagging_freq=5,
reg_lambda=0.2)
 X_tra, X_val, y_tra, y_val = train_test_split(train_features, train_target, train_size=0.8, random_state=233)
 lgb_model.fit(X=X_tra, y=y_tra,
eval_set=[(X_val, y_val)],
verbose=False)
 print('Valid Score is %.4f' % roc_auc_score(y_val, lgb_model.predict_proba(X_val)[:,1]))
final_predict = lgb_model.predict_proba(test_features)[:,1]
if quick_run == False:
sample_df = pd.read_csv('.. /input/donorschoose-application-screening/sample_submission.csv')
 sample_df['project_is_approved'] = final_predict
sample_df.to_csv('submission.csv', index=False )
<save_to_csv><EOS> 
scaler = MinMaxScaler()
predict_valid_list = [pred_valid_rf, pred_valid_lgb, pred_valid_dpcnn, pred_valid_ftrl, pred_valid_lgb_bool]
predict_test_list = [pred_test_avg_rf, pred_test_avg_lgb, pred_test_avg_dpcnn, pred_test_avg_ftrl, pred_test_avg_lgb_bool]
valid_results = np.hstack([scaler.fit_transform(item.reshape(( item.shape[0], 1)))for item in predict_valid_list])
 test_results = np.hstack([scaler.fit_transform(item.reshape(( item.shape[0], 1)))for item in predict_test_list])
 train_features = np.hstack([valid_results ])
 test_features = np.hstack([test_results ])
 lgb_model = lgb.LGBMClassifier(n_jobs=4,
max_depth=4,
metric="auc",
n_estimators=400,
num_leaves=10,
boosting_type="gbdt",
learning_rate=0.01,
feature_fraction=0.45,
colsample_bytree=0.45,
bagging_fraction=0.4,
bagging_freq=5,
reg_lambda=0.2)
 X_tra, X_val, y_tra, y_val = train_test_split(train_features, train_target, train_size=0.8, random_state=233)
 lgb_model.fit(X=X_tra, y=y_tra,
eval_set=[(X_val, y_val)],
verbose=False)
 print('Valid Score is %.4f' % roc_auc_score(y_val, lgb_model.predict_proba(X_val)[:,1]))
final_predict = lgb_model.predict_proba(test_features)[:,1]
if quick_run == False:
sample_df = pd.read_csv('.. /input/donorschoose-application-screening/sample_submission.csv')
 sample_df['project_is_approved'] = final_predict
sample_df.to_csv('submission_with_lgb_bool.csv', index=False )