Kaggle competition: Santander Customer Satisfaction
<SOS> metric: AUC Kaggle data source: santander-customer-satisfaction<init_hyperparams>
data=pd.read_csv('.. /input/train.csv')
 out_data=pd.read_csv('.. /input/test.csv')
COLUMNS = col_train_num
FEATURES = col_train_num_bis
LABEL = "SalePrice"
FEATURES_CAT = col_train_cat
engineered_features = []
for continuous_feature in FEATURES:
engineered_features.append(
 tf.contrib.layers.real_valued_column(continuous_feature))
for categorical_feature in FEATURES_CAT:
sparse_column = tf.contrib.layers.sparse_column_with_hash_bucket(
 categorical_feature, hash_bucket_size=1000)
 engineered_features.append(tf.contrib.layers.embedding_column(sparse_id_column=sparse_column, dimension=16,combiner="sum"))
training_set = train[FEATURES + FEATURES_CAT]
prediction_set = train.SalePrice
x_train, x_test, y_train, y_test = train_test_split(training_set[FEATURES + FEATURES_CAT] ,
prediction_set, test_size=0.20, random_state=42)
 y_train = pd.DataFrame(y_train, columns = [LABEL])
 training_set = pd.DataFrame(x_train, columns = FEATURES + FEATURES_CAT ).merge(y_train, left_index = True, right_index = True)
 training_sub = training_set[FEATURES + FEATURES_CAT]
testing_sub = test[FEATURES + FEATURES_CAT]<create_dataframe>
x=data.loc[:,'var3':'var38']
y=data['TARGET']
y_test = pd.DataFrame(y_test, columns = [LABEL])
 testing_set = pd.DataFrame(x_test, columns = FEATURES + FEATURES_CAT ).merge(y_test, left_index = True, right_index = True )<feature_engineering>
x=np.asarray(x)
 y=np.asarray(y )
training_set[FEATURES_CAT] = training_set[FEATURES_CAT].applymap(str)
 testing_set[FEATURES_CAT] = testing_set[FEATURES_CAT].applymap(str)
 def input_fn_new(data_set, training = True):
continuous_cols = {k: tf.constant(data_set[k].values)for k in FEATURES}
categorical_cols = {k: tf.SparseTensor(
 indices=[[i, 0] for i in range(data_set[k].size)], values = data_set[k].values, dense_shape = [data_set[k].size, 1])for k in FEATURES_CAT}
feature_cols = dict(list(continuous_cols.items())+ list(categorical_cols.items()))
if training == True:
label = tf.constant(data_set[LABEL].values)
 return feature_cols, label
return feature_cols
regressor = tf.contrib.learn.DNNRegressor(feature_columns = engineered_features,
activation_fn =leaky_relu, hidden_units=[256, 128, 64, 32, 16])
<define_variables>
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.metrics import metrics
from sklearn.cross_validation import train_test_split
categorical_cols = {k: tf.SparseTensor(indices=[[i, 0] for i in range(training_set[k].size)], values = training_set[k].values, dense_shape = [training_set[k].size, 1])for k in FEATURES_CAT}<train_model>
matplotlib.use("Agg")
 training = pd.read_csv(".. /input/train.csv", index_col=0)
 test = pd.read_csv(".. /input/test.csv", index_col=0)
 print(training.shape)
 print(test.shape)
 training = training.replace(-999999,2)
 X = training.iloc[:,:-1]
y = training.TARGET
X['n0'] =(X == 0 ).sum(axis=1)
 p = 86
p = 80
p = 77
p = 75
X_bin = Binarizer().fit_transform(scale(X))
selectChi2 = SelectPercentile(chi2, percentile=p ).fit(X_bin, y)
 selectF_classif = SelectPercentile(f_classif, percentile=p ).fit(X, y)
 chi2_selected = selectChi2.get_support()
chi2_selected_features = [ f for i,f in enumerate(X.columns)if chi2_selected[i]]
print('Chi2 selected {} features {}.'.format(chi2_selected.sum() ,
chi2_selected_features))
f_classif_selected = selectF_classif.get_support()
f_classif_selected_features = [ f for i,f in enumerate(X.columns)if f_classif_selected[i]]
print('F_classif selected {} features {}.'.format(f_classif_selected.sum() ,
f_classif_selected_features))
selected = chi2_selected & f_classif_selected
print('Chi2 & F_classif selected {} features'.format(selected.sum()))
features = [ f for f,s in zip(X.columns, selected)if s]
print(features )
regressor.fit(input_fn = lambda: input_fn_new(training_set), steps=2000 )<compute_test_metric>
X_sel = X[features]
X_train, X_test, y_train, y_test = \
cross_validation.train_test_split(X_sel, y, random_state=1301, stratify=y, test_size=0.3)
 clf = xgb.XGBClassifier(missing=9999999999,
max_depth = 8,
n_estimators=1000,
learning_rate=0.02,
nthread=4,
subsample=0.8,
colsample_bytree=0.5,
min_child_weight = 7,
seed=4242)
 clf.fit(X_train, y_train, early_stopping_rounds=50, eval_metric="auc",
eval_set=[(X_train, y_train),(X_test, y_test)])
 print('Overall AUC:', roc_auc_score(y, clf.predict_proba(X_sel, ntree_limit=clf.best_iteration)[:,1]))
test['n0'] =(test == 0 ).sum(axis=1)
 sel_test = test[features]
y_pred = clf.predict_proba(sel_test, ntree_limit=clf.best_iteration)
 submission = pd.DataFrame({"ID":test.index, "TARGET":y_pred[:,1]})
 submission.to_csv("submission.csv", index=False)
 mapFeat = dict(zip(["f"+str(i)for i in range(len(features)) ],features))
ts = pd.Series(clf.booster().get_fscore())
 ts.sort_values() [-15:].plot(kind="barh", title=("features importance"))
featp = ts.sort_values() [-15:].plot(kind='barh', x='feature', y='fscore', legend=False, figsize=(6, 10))
plt.title('XGBoost Feature Importance')
 fig_featp = featp.get_figure()
fig_featp.savefig('feature_importance_xgb.png', bbox_inches='tight', pad_inches=1 )
ev = regressor.evaluate(input_fn=lambda: input_fn_new(testing_set, training = True), steps=1)
 loss_score4 = ev["loss"]
print("Final Loss on the testing set: {0:f}".format(loss_score4))<choose_model_class>
a=clf.booster().get_fscore()
regressor = tf.contrib.learn.DNNRegressor(feature_columns = engineered_features,
activation_fn = tf.nn.relu, hidden_units=[2500] )<train_model>
sorted_x = sorted(a.items() , key=operator.itemgetter(1))
regressor.fit(input_fn = lambda: input_fn_new(training_set), steps=2000 )<compute_test_metric>
fe=['num_var22_hace3','num_med_var45_ult3','saldo_var5','saldo_var42','num_var45_hace3','num_var22_ult3','saldo_medio_var5_ult1','num_var45_ult3','saldo_medio_var5_hace2',
'saldo_var30',
'saldo_medio_var5_ult3',
'saldo_medio_var5_hace3',
'var15',
'var38']
x=data[fe]
ev = regressor.evaluate(input_fn=lambda: input_fn_new(testing_set, training = True), steps=1)
 loss_score5 = ev["loss"]<compute_test_metric>
gb=GradientBoostingClassifier()
fe=np.asarray(fe)
 gb.fit(fe,y )