Kaggle competition: Instant Gratification
<SOS> metric: AUC Kaggle data source: instant-gratification<prepare_x_and_y>
sns.set_style('darkgrid')
 pd.options.display.float_format = '{:,.3f}'.format
print(os.listdir(".. /input"))
W_fc2 = tf.Variable(tf.truncated_normal([1024, 10], stddev=0.1))
b_fc2 = tf.Variable(tf.constant(0.1, shape=[10]))
y_conv=tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2)+ b_fc2 )<compute_train_metric>
%%time
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print(train.shape, test.shape )
cross_entropy = -tf.reduce_sum(y_*tf.log(y_conv))
train_step = tf.train.AdamOptimizer(1e-4 ).minimize(cross_entropy)
 correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
predict = tf.argmax(y_conv,1 )<features_selection>
null_cnt = train.isnull().sum().sort_values()
print('null count:', null_cnt[null_cnt > 0] )
sess.run(tf.global_variables_initializer() )<load_from_csv>
print(train['wheezy-copper-turtle-magic'].describe())
 print()
print('unique value count:', train['wheezy-copper-turtle-magic'].nunique() )
input_data=ReadCsvData()
x_train, y_label, x_test = input_data.get_data()<define_variables>
numcols = train.drop(['id','target','wheezy-copper-turtle-magic'],axis=1 ).select_dtypes(include='number' ).columns.values
Batch_size=100
Train_Number=5300
accracies = []<train_model>
X_subset = train[train['wheezy-copper-turtle-magic'] == 0][numcols]
Y_subset = train[train['wheezy-copper-turtle-magic'] == 0]['target']
for k in range(2, 10):
knc = KNeighborsClassifier(n_neighbors=k)
 knc.fit(X_subset, Y_subset)
 score = knc.score(X_subset, Y_subset)
 print("[{}] score: {:.2f}".format(k, score))
print('Start Learning', datetime.now() ,)
 for j in range(3):
train_dataset = DataSet(x_train,y_label)
 for i in range(Train_Number):
batch_x, batch_y = train_dataset.next_batch(Batch_size)
 if i%100 == 0:
train_accuracy = accuracy.eval(feed_dict={x:batch_x, y_: batch_y, rate: 0.0})
 accracies.append(train_accuracy)
 print("step %d, training accuracy %g"%(i, train_accuracy))
train_step.run(feed_dict={x: batch_x, y_: batch_y, rate: 0.5})
 print("step %d, training accuracy %g"%(i, train_accuracy))
print('Finish Learning', datetime.now() , )<save_to_csv>
all_data = train.append(test, sort=False ).reset_index(drop=True)
 del train, test
gc.collect()
all_data.head()
submission_file=pd.DataFrame({'ImageId':np.arange(1,(x_test.shape[0] + 1)) , 'Label':predict.eval(feed_dict={x: x_test, rate: 0.0})})
 print(submission_file)
 submission_file.to_csv("submission_v1.csv", index=False)
 print('Save submission', datetime.now() , )<load_from_csv>
constant_column = [col for col in all_data.columns if all_data[col].nunique() == 1]
print('drop columns:', constant_column)
 all_data.drop(constant_column, axis=1, inplace=True )
data_folder = Path(".. /input")
 train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/sample_submission.csv")
 test_img = ImageList.from_df(test_df, path=data_folder/'test', folder='test')
 trfm = get_transforms(do_flip=True, flip_vert=True, max_rotate=10.0, max_zoom=1.1, max_lighting=0.2, max_warp=0.2, p_affine=0.75, p_lighting=0.75)
 train_img =(ImageList.from_df(train_df, path=data_folder/'train', folder='train')
.split_by_rand_pct(0.01)
.label_from_df() 
.add_test(test_img)
.transform(trfm, size=128)
.databunch(path='.', bs=64, device= torch.device('cuda:0')) 
.normalize(imagenet_stats)
)
 learn = cnn_learner(train_img, models.densenet161, metrics=[error_rate, accuracy] )<train_model>
corr_matrix = all_data.corr().abs()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [c for c in upper.columns if any(upper[c] > 0.95)]
del upper
drop_column = all_data.columns[to_drop]
print('drop columns:', drop_column)
lr = 3e-02
learn.fit_one_cycle(5, slice(lr))<save_to_csv>
X_train = all_data[all_data['target'].notnull() ].reset_index(drop=True)
 X_test = all_data[all_data['target'].isnull() ].drop(['target'], axis=1 ).reset_index(drop=True)
 del all_data
gc.collect()
X_train.drop(['id'], axis=1, inplace=True)
 X_test_ID = X_test.pop('id')
 Y_train = X_train.pop('target')
 print(X_train.shape, X_test.shape )
preds,_ = learn.get_preds(ds_type=DatasetType.Test)
 test_df.has_cactus = preds.numpy() [:, 0]
test_df.to_csv('submission.csv', index=False )<set_options>
oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 splits = 11
for i in range(512):
train2 = X_train[X_train['wheezy-copper-turtle-magic'] == i][numcols]
train2_y = Y_train[X_train['wheezy-copper-turtle-magic'] == i]
test2 = X_test[X_test['wheezy-copper-turtle-magic'] == i][numcols]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5)
 train2 = sel.fit_transform(train2)
 test2 = sel.transform(test2)
 skf = StratifiedKFold(n_splits=splits, random_state=42)
 for train_index, test_index in skf.split(train2, train2_y):
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train2[train_index], train2_y.iloc[train_index])
 oof_preds[idx1[test_index]] = clf.predict_proba(train2[test_index])[:,1]
sub_preds[idx2] += clf.predict_proba(test2)[:,1] / skf.n_splits
%matplotlib inline
sns.set(style='white', context='notebook', palette='deep' )<load_from_csv>
len(X_train[(oof_preds > 0.3)&(oof_preds < 0.7)] )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 IDtest = test["PassengerId"]<drop_column>
X_train = X_train[(oof_preds <= 0.3)|(oof_preds >= 0.7)]
Y_train = Y_train[(oof_preds <= 0.3)|(oof_preds >= 0.7)]
train = train.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True )<concatenate>
X_test_p1 = X_test[(sub_preds <= 0.01)].copy()
X_test_p2 = X_test[(sub_preds >= 0.99)].copy()
X_test_p1['target'] = 0
X_test_p2['target'] = 1
print(X_test_p1.shape, X_test_p2.shape)
 Y_train = pd.concat([Y_train, X_test_p1.pop('target'), X_test_p2.pop('target')], axis=0)
 X_train = pd.concat([X_train, X_test_p1, X_test_p2], axis=0)
 Y_train.reset_index(drop=True, inplace=True)
 X_train.reset_index(drop=True, inplace=True )
train_len = len(train)
 dataset = pd.concat(objs=[train, test], axis=0 ).reset_index(drop=True )<count_missing_values>
_=
for i in range(512):
train_f =(X_train['wheezy-copper-turtle-magic'] == i)
 test_f =(X_test['wheezy-copper-turtle-magic'] == i)
 X_train_sub = X_train[train_f][numcols]
Y_train_sub = Y_train[train_f]
X_test_sub = X_test[test_f][numcols]
lda = LDA(n_components=1)
 lda.fit(X_train_sub, Y_train_sub)
 X_train.loc[train_f, 'lda'] = lda.transform(X_train_sub ).reshape(-1)
 X_test.loc[test_f, 'lda'] = lda.transform(X_test_sub ).reshape(-1)
 knc = KNeighborsClassifier(n_neighbors=3)
 knc.fit(X_train_sub, Y_train_sub)
 X_train.loc[train_f, 'knc'] = knc.predict_proba(X_train_sub)[:,1]
X_test.loc[test_f, 'knc'] = knc.predict_proba(X_test_sub)[:,1]
dataset = dataset.fillna(np.nan)
 dataset.isnull().sum()<feature_engineering>
oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 splits = 11
for i in range(512):
train2 = X_train[X_train['wheezy-copper-turtle-magic'] == i][numcols]
train2_y = Y_train[X_train['wheezy-copper-turtle-magic'] == i]
test2 = X_test[X_test['wheezy-copper-turtle-magic'] == i][numcols]
idx1 = train2.index; idx2 = test2.index
train2.reset_index(drop=True,inplace=True)
 sel = VarianceThreshold(threshold=1.5)
 train2 = sel.fit_transform(train2)
 test2 = sel.transform(test2)
 skf = StratifiedKFold(n_splits=splits, random_state=42)
 for train_index, test_index in skf.split(train2, train2_y):
clf = QuadraticDiscriminantAnalysis(reg_param=0.5)
 clf.fit(train2[train_index], train2_y.iloc[train_index])
 oof_preds[idx1[test_index]] = clf.predict_proba(train2[test_index])[:,1]
sub_preds[idx2] += clf.predict_proba(test2)[:,1] / skf.n_splits
<feature_engineering><EOS> 
submission = pd.DataFrame({
'id': X_test_ID,
'target': sub_preds
})
 submission.to_csv("submission.csv", index=False )