Kaggle competition: Costa Rican Household Poverty Level Prediction
def learning_rate_010_decay_power_099(current_iter):
base_learning_rate = 0.1
lr = base_learning_rate * np.power (.99, current_iter)
 return lr if lr > 1e-3 else 1e-3
def learning_rate_010_decay_power_0995(current_iter):
base_learning_rate = 0.1
lr = base_learning_rate * np.power (.995, current_iter)
 return lr if lr > 1e-3 else 1e-3
def learning_rate_005_decay_power_099(current_iter):
base_learning_rate = 0.05
lr = base_learning_rate * np.power (.99, current_iter)
 return lr if lr > 1e-3 else 1e-3
fit_params={"early_stopping_rounds":30,
"eval_metric" : 'auc',
'eval_names': ['valid'],
'verbose': 100,
'categorical_feature': 'auto'}
param_test ={'num_leaves': sp_randint(6, 50),
'min_child_samples': sp_randint(100, 500),
'min_child_weight': [1e-5, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4],
'subsample': sp_uniform(loc=0.2, scale=0.8),
'colsample_bytree': sp_uniform(loc=0.4, scale=0.6),
'reg_alpha': [0, 1e-1, 1, 2, 5, 7, 10, 50, 100],
'reg_lambda': [0, 1e-1, 1, 5, 10, 20, 50, 100]}
n_HP_points_to_test = 100
clf = lgb.LGBMClassifier(max_depth=-1, random_state=314, silent=True, metric='None', n_jobs=4, n_estimators=5000)
 gs = RandomizedSearchCV(
 estimator=clf, param_distributions=param_test,
n_iter=n_HP_points_to_test,
scoring='roc_auc',
cv=3,
refit=True,
random_state=314,
verbose=True)
 opt_parameters = {'colsample_bytree': 0.48263575356020577, 'min_child_samples': 311,
'min_child_weight': 1, 'num_leaves': 7, 'reg_alpha': 7, 'reg_lambda': 0.1,
'subsample': 0.3542761367404292} 
<split>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 test_Id = test_df['Id']
def kfold_lightgbm(train_df,test_df,Y_train,num_folds,stratified = False, debug= False):
if stratified:
folds = StratifiedKFold(n_splits= num_folds, shuffle=True, random_state=1001)
 else:
folds = KFold(n_splits= num_folds, shuffle=True, random_state=1001)
 oof_preds = np.zeros(train_df.shape[0])
 sub_preds = np.zeros(test_df.shape[0])
 feature_importance_df = pd.DataFrame()
feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df[feats], train_df['TARGET'])) :
train_x, train_y = train_df[feats].iloc[train_idx], train_df['TARGET'].iloc[train_idx]
valid_x, valid_y = train_df[feats].iloc[valid_idx], train_df['TARGET'].iloc[valid_idx]
clf = LGBMClassifier(boosting_type='gbdt', class_weight=None,
colsample_bytree=0.48263575356020577, importance_type='split',
learning_rate=0.1, max_depth=-1, metric='None',
min_child_samples=311, min_child_weight=1, min_split_gain=0.0,
n_estimators=10000, n_jobs=4, num_leaves=7, objective=None,
random_state=None, reg_alpha=7, reg_lambda=0.1, silent=True,
subsample=0.3542761367404292, subsample_for_bin=200000,
subsample_freq=0,scale_pos_weight = 2)
 clf.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)],
eval_metric= 'auc', verbose= 100, early_stopping_rounds= 30,callbacks=[lgb.reset_parameter(learning_rate=learning_rate_010_decay_power_0995)])
 oof_preds[valid_idx] = clf.predict_proba(valid_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test_df[feats], num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feats
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))
 del clf, train_x, train_y, valid_x, valid_y
gc.collect()
print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'], oof_preds))
folds_idx = [(trn_idx, val_idx)for trn_idx, val_idx in folds.split(train_df[feats], train_df['TARGET'])]
display_roc_curve(y_=Y_train,oof_preds_=oof_preds,sub_preds_ = sub_preds, folds_idx_=folds_idx)
 display_precision_recall(y_=Y_train, oof_preds_=oof_preds, folds_idx_=folds_idx)
 return feature_importance_df,sub_preds
def display_roc_curve(y_, oof_preds_,sub_preds_,folds_idx_):
plt.figure(figsize=(6,6))
scores = []
for n_fold,(_, val_idx)in enumerate(folds_idx_):
fpr, tpr, thresholds = roc_curve(y_.iloc[val_idx], oof_preds_[val_idx])
 score = roc_auc_score(y_.iloc[val_idx], oof_preds_[val_idx])
 scores.append(score)
 plt.plot(fpr, tpr, lw=1, alpha=0.3, label='ROC fold %d(AUC = %0.4f)' %(n_fold + 1, score))
plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='r', label='Chance', alpha=.8)
 fpr, tpr, thresholds = roc_curve(y_, oof_preds_)
 score = roc_auc_score(y_, oof_preds_)
 plt.plot(fpr, tpr, color='b',
label='Avg ROC(AUC = %0.4f $\pm$ %0.4f)' %(score, np.std(scores)) ,
lw=2, alpha=.8)
 plt.xlim([-0.05, 1.05])
 plt.ylim([-0.05, 1.05])
 plt.xlabel('False Positive Rate')
 plt.ylabel('True Positive Rate')
 plt.title('LightGBM ROC Curve')
 plt.legend(loc="lower right")
 plt.tight_layout()
plt.savefig('roc_curve.png')
 def display_precision_recall(y_, oof_preds_, folds_idx_):
plt.figure(figsize=(6,6))
scores = []
for n_fold,(_, val_idx)in enumerate(folds_idx_):
fpr, tpr, thresholds = roc_curve(y_.iloc[val_idx], oof_preds_[val_idx])
 score = average_precision_score(y_.iloc[val_idx], oof_preds_[val_idx])
 scores.append(score)
 plt.plot(fpr, tpr, lw=1, alpha=0.3, label='AP fold %d(AUC = %0.4f)' %(n_fold + 1, score))
precision, recall, thresholds = precision_recall_curve(y_, oof_preds_)
 score = average_precision_score(y_, oof_preds_)
 plt.plot(precision, recall, color='b',
label='Avg ROC(AUC = %0.4f $\pm$ %0.4f)' %(score, np.std(scores)) ,
lw=2, alpha=.8)
 plt.xlim([-0.05, 1.05])
 plt.ylim([-0.05, 1.05])
 plt.xlabel('Recall')
 plt.ylabel('Precision')
 plt.title('LightGBM Recall / Precision')
 plt.legend(loc="best")
 plt.tight_layout()
def compiler2(train_df,test_df,Y_train,debug = False):
num_rows = 10000 if debug else None
feat_importance,sub_preds = kfold_lightgbm(train_df,test_df,
Y_train, num_folds= 5,
stratified= True, debug= debug)
 return feat_importance,sub_preds 
<create_dataframe>
def preprocess(train_df, test_df, dropna = False, drop_obtype = True, replace_nan_with = 0, return_with_top100 = False, lgbfactor = 0):
train_df.drop(columns = ['Id'],inplace = True, errors="ignore")
 test_df.drop(columns = ['Id'], inplace = True, errors = "ignore")
 nanvalues = train_df.isnull().sum(axis = 0 ).values
nullvalues = pd.DataFrame({"Column":train_df.columns, "Count":nanvalues})
 nullvalues.sort_values(by=["Count"], ascending = False, inplace = True)
 nancols = nullvalues[nullvalues['Count']>0]["Column"].values.tolist()
objcols = train_df[train_df.columns[(train_df.dtypes == 'object' ).values]].columns.values.tolist()
if(drop_obtype == True):
train_df.drop(columns = objcols,inplace = True)
 test_df.drop(columns = objcols,inplace = True)
 if(dropna == True):
train_df.drop(columns = nancols, inplace = True)
 test_df.drop(columns = nancols, inplace = True)
 if(dropna == False):
train_df.replace(np.nan, replace_nan_with, inplace = True)
 test_df.replace(np.nan, replace_nan_with, inplace = True)
 labels = train_df['Target']
train_df.drop(columns = ['Target'],inplace = True, errors="ignore")
 rfc = RandomForestClassifier()
rfc.fit(train_df, labels)
 scores = rfc.feature_importances_.tolist()
report = pd.DataFrame({"feature":train_df.columns, "score":scores})
 report.sort_values(by = ["score"],ascending = False, inplace = True)
 top100cols = report.head(100)['feature'].values.tolist()
if(return_with_top100 == True):
train_df = train_df[top100cols]
test_df = test_df[top100cols]
class_weights = class_weight.compute_class_weight('balanced',
np.unique(labels),
labels)
 class_weights = dict(zip(np.unique(labels.values-lgbfactor), class_weights))
return train_df, test_df, labels, nancols, objcols, report, top100cols, class_weights
feat_importance,sub_preds = compiler2(train_dff,test_dff,Y_train,debug = False)
 result = pd.DataFrame()
result['SK_ID_CURR'] = test_df['SK_ID_CURR'].values.tolist()
result['TARGET'] = sub_preds 
<save_to_csv>
train, test, labels, nancols,objcols, report, top100cols, class_weightss = preprocess(train_df.copy() , test_df.copy() ,dropna = False, lgbfactor = 1 )
result.to_csv('submission_kernel_lightGBM.csv', index= False)
<split>
import lightgbm as lgb
import sklearn.model_selection as model_selection
from sklearn.metrics import f1_score, make_scorer
if lgb.compat.MATPLOTLIB_INSTALLED:
else:
raise ImportError('You need to install matplotlib for plot_example.py.')
 X_train_, X_test_, Y_train_, Y_test_ = train_test_split(X_train, Y_train, test_size=0.1,random_state=42)
 lgb_train = lgb.Dataset(X_train_, Y_train_)
 lgb_test = lgb.Dataset(X_test_, Y_test_, reference=lgb_train)
 params = {'colsample_bytree': 0.48263575356020577, 'min_child_samples': 311,
'min_child_weight': 1, 'num_leaves': 7, 'reg_alpha': 7, 'reg_lambda': 0.1,
'subsample': 0.3542761367404292, 'metric': 'roc_auc'}
print('Starting training...')
 gbm = lgb.train(params,
lgb_train,
num_boost_round=100,
valid_sets=[lgb_train, lgb_test],
feature_name=[X_train.columns.tolist() [i] for i in range(len(X_train.columns.tolist())) ],
verbose_eval=10)
<categorify>
lgmodel = lgb.LGBMClassifier(class_weight=class_weightss, metric = "multi_logloss",num_class = 4)
def eval_wrapper(yhat, y): 
y = np.array(y)
y = y.astype(int)
yhat = np.array(yhat)
yhat = np.clip(np.round(yhat), np.min(y), np.max(y)).astype(int)
return quadratic_weighted_kappa(yhat, y)

def get_params() :

params = {}
params["objective"] = "reg:linear" 
params["eta"] = 0.05
params["min_child_weight"] = 240
params["subsample"] = 0.9
params["colsample_bytree"] = 0.67
params["silent"] = 1
params["max_depth"] = 6
plst = list(params.items())

return plst

def apply_offset(data, bin_offset, sv, scorer=eval_wrapper):
data[1, data[0].astype(int)==sv] = data[0, data[0].astype(int)==sv] + bin_offset
score = scorer(data[1], data[2])
return score

columns_to_drop = ['Id', 'Response', 'Medical_History_10','Medical_History_24']
xgb_num_rounds = 700
num_classes = 8
eta_list = [0.05] * 200 
eta_list = eta_list + [0.02] * 500

print("Load the data using pandas")
train = pd.read_csv(".. /input/train.csv")
test = pd.read_csv(".. /input/test.csv")

all_data = train.append(test)

all_data['Product_Info_2_char'] = all_data.Product_Info_2.str[0]
all_data['Product_Info_2_num'] = all_data.Product_Info_2.str[1]

all_data['Product_Info_2'] = pd.factorize(all_data['Product_Info_2'])[0]
all_data['Product_Info_2_char'] = pd.factorize(all_data['Product_Info_2_char'])[0]
all_data['Product_Info_2_num'] = pd.factorize(all_data['Product_Info_2_num'])[0]

all_data['BMI_Age'] = all_data['BMI'] * all_data['Ins_Age']

med_keyword_columns = all_data.columns[all_data.columns.str.startswith('Medical_Keyword_')]
all_data['Med_Keywords_Count'] = all_data[med_keyword_columns].sum(axis=1)

print('Eliminate missing values')
all_data.fillna(-1, inplace=True)

all_data['Response'] = all_data['Response'].astype(int)

train = all_data[all_data['Response']>0].copy() 
test = all_data[all_data['Response']<1].copy() 
test['Response'] = test['Response']-1

xgtrain = xgb.DMatrix(train.drop(columns_to_drop, axis=1), train['Response'].values)
xgtest = xgb.DMatrix(test.drop(columns_to_drop, axis=1), label=test['Response'].values)

plst = get_params() 
print(plst)

model = xgb.train(plst, xgtrain, xgb_num_rounds, learning_rates=eta_list)

train_preds = model.predict(xgtrain, ntree_limit=model.best_iteration)
print('Train score is:', eval_wrapper(train_preds, train['Response'])) 
test_preds = model.predict(xgtest, ntree_limit=model.best_iteration)
train_preds = np.clip(train_preds, 1, 8)
test_preds = np.clip(test_preds, 1, 8)

offsets = np.array([0.1, -1, -2, -1, -0.8, 0.02, 0.8, 1])
data = np.vstack(( train_preds, train_preds, train['Response'].values)) 
for j in range(num_classes):
data[1, data[0].astype(int)==j] = data[0, data[0].astype(int)==j] + offsets[j] 
for j in range(num_classes):
train_offset = lambda x: -apply_offset(data, x, j)
offsets[j] = fmin_powell(train_offset, offsets[j])

data = np.vstack(( test_preds, test_preds, test['Response'].values)) 
for j in range(num_classes):
data[1, data[0].astype(int)==j] = data[0, data[0].astype(int)==j] + offsets[j] 

final_test_preds = np.round(np.clip(data[1], 1, 8)).astype(int)
xgtest = xgb.DMatrix(test.drop(columns_to_drop, axis=1), label=final_test_preds)
final_test_preds = model.predict(xgtest, ntree_limit=model.best_iteration)
preds_out = pd.DataFrame({"Id": test['Id'].values, "Response": final_test_preds})
preds_out = preds_out.set_index('Id')
preds_out.to_csv('xgb_offset_submission.csv' )<normalization>
def get_score(model, train, label, fold):
score = model_selection.cross_val_score(model, train , label, cv = fold, scoring = make_scorer(f1_score, average = "macro"))
return score.mean()
random.seed(23)
 def preprocess_data(X, scaler=None):
if not scaler:
scaler = StandardScaler()
scaler.fit(X)
 X = scaler.transform(X)
 return X, scaler
def eval_wrapper(y, yhat):
y = np.array(y)
 y = y.astype(int)
 yhat = np.array(yhat)
 yhat = np.clip(np.round(yhat), np.min(y), np.max(y)).astype(int)
 return quadratic_weighted_kappa(yhat, y)
 def preprocess_labels(labels, encoder=None, categorical=True):
if not encoder:
encoder = LabelEncoder()
encoder.fit(labels)
 y = encoder.transform(labels ).astype(np.int32)
 if categorical:
y = np_utils.to_categorical(y)
 return y, encoder
def apply_offset(data, bin_offset, sv, scorer=eval_wrapper):
data[1, data[0].astype(int)==sv] = data[0, data[0].astype(int)==sv] + bin_offset
score = scorer(data[1], data[2])
 return score
columns_to_drop = ['Id', 'Response', 'Medical_History_10', 'Medical_History_24', 'Medical_History_32']
num_classes = 8
print("Load the data using pandas")
 train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 all_data = train.append(test)
 all_data['Product_Info_2_char'] = all_data.Product_Info_2.str[1]
all_data['Product_Info_2_num'] = all_data.Product_Info_2.str[2]
all_data['Product_Info_2'] = pd.factorize(all_data['Product_Info_2'])[0]
all_data['Product_Info_2_char'] = pd.factorize(all_data['Product_Info_2_char'])[0]
all_data['Product_Info_2_num'] = pd.factorize(all_data['Product_Info_2_num'])[0]
all_data['BMI_Age'] = all_data['BMI'] * all_data['Ins_Age']
med_keyword_columns = all_data.columns[all_data.columns.str.startswith('Medical_Keyword_')]
all_data['Med_Keywords_Count'] = all_data[med_keyword_columns].sum(axis=1)
 print('Eliminate missing values')
 all_data.fillna(-1, inplace=True)
 all_data['Response'] = all_data['Response'].astype(int)
 all_data['Split'] = np.random.randint(5, size=all_data.shape[0])
 train = all_data[all_data['Response']>0].copy()
test = all_data[all_data['Response']<1].copy()
testId = test["Id"]
labels = train["Response"]
labels_test = test["Response"]
y, encoder = preprocess_labels(labels)
 train = train.drop(columns_to_drop, axis = 1)
 test = test.drop(columns_to_drop, axis = 1)
 train, scaler = preprocess_data(train)
 test, _ = preprocess_data(test, scaler)
 dims = train.shape[1]
print(dims, 'dims')
 print('Building model...')
 model = Sequential()
model.add(Dense(output_dim=32, init='glorot_uniform', input_dim=dims, activation='tanh'))
model.add(Dropout(0.1))
model.add(Dense(32, init='glorot_uniform'))
model.add(PReLU())
 model.add(Dense(num_classes, init='glorot_uniform'))
model.add(Activation('softmax'))
SGDopt = optimizers.SGD(lr=0.01, momentum=0.001, decay=0.0, nesterov=False)
 model.compile(loss='categorical_crossentropy', optimizer=SGDopt)
 train = np.array(train, dtype=np.float32)
 y = np.array(y, dtype=np.int32)
 print('Training model...')
 model.fit(train, y, nb_epoch = 10, batch_size = 16, validation_split = 0.4, verbose = 0)
 print('Generating submission...')
 test = np.array(test, dtype=np.float32)
 test_preds = model.predict_classes(test, batch_size = 50, verbose = 0)
 test_preds += 1
train_preds = model.predict_classes(train, batch_size = 50, verbose = 0)
 train_preds += 1
train_probs = model.predict_proba(train, batch_size = 50, verbose = 0)
 test_probs = model.predict_proba(test, batch_size = 50, verbose = 0)
 classes = np.array(range(num_classes)) +1
expected_value_train = np.dot(train_probs, classes)
 expected_value_test = np.dot(test_probs, classes)
 train_preds = expected_value_train
test_preds = expected_value_test
expected_value_train = np.clip(expected_value_train, 1, 8)
 expected_value_test = np.clip(expected_value_test, 1, 8)
 expected_value_train = np.round(expected_value_train ).astype(int)
 expected_value_test = np.round(expected_value_test ).astype(int)
 print('Train score: ',eval_wrapper(expected_value_train, labels))
train_preds = np.clip(train_preds, -0.99, 8.99)
 test_preds = np.clip(test_preds, -0.99, 8.99)
 offsets = np.array([-1, -1, -1, -1, -1, -1, -1, -1])
 offset_train_preds = np.vstack(( train_preds, train_preds, labels))
for j in range(num_classes):
train_offset = lambda x: -apply_offset(offset_train_preds, x, j)
 offsets[j] = fmin_powell(train_offset, offsets[j])
 data = np.vstack(( test_preds, test_preds, labels_test.values))
for j in range(num_classes):
data[1, data[0].astype(int)==j] = data[0, data[0].astype(int)==j] + offsets[j]
final_test_preds = np.round(np.clip(data[1], 1, 8)).astype(int)
 preds_out = pd.DataFrame({"Id": testId, "Response": final_test_preds})
 preds_out = preds_out.set_index('Id')
 preds_out.to_csv('keras_expv.csv' )<import_modules>
kf = model_selection.KFold(n_splits=5, shuffle=True,random_state=2017)
 templabel = labels.loc[train['parentesco1']==1]
temptrain = train[train['parentesco1']==1]
heads_scores = get_score(lgmodel, temptrain, templabel-1, kf)
 overall_scores = get_score(lgmodel, train, labels-1, kf)
 print("Heads mean score:",heads_scores.mean())
 print("Overall mean score:",overall_scores.mean() )
import pandas as pd
import numpy as np
import xgboost as xgb
from scipy.optimize import fmin_powell
from ml_metrics import quadratic_weighted_kappa<compute_test_metric>
lgmodel.fit(train, labels-1)
 lgpreds = lgmodel.predict(test )
<load_from_csv><EOS> 
sub_df = pd.DataFrame({"Id":test_Id.values})
 sub_df["Target"] = lgpreds + 1
sub_df.to_csv("LightGBM.csv", index=False )