Kaggle competition: Home Credit Default Risk
<SOS> metric: AUC Kaggle data source: home-credit-default-risk<filter>
N_FOLDS = 5
MAX_EVALS = 5
condition1= online_sales['Product SKU'] == 'GGOENEBQ079099'
online_sales[(condition1)]
condition2= online_sales['Quantity'] > 2
online_sales[(condition1)&(condition2)]<feature_engineering>
features = pd.read_csv('.. /input/home-credit-default-risk/application_train.csv')
 features = features.sample(n = 16000, random_state = 42)
 features = features.select_dtypes('number')
 labels = np.array(features['TARGET'].astype(np.int32)).reshape(( -1,))
features = features.drop(columns = ['TARGET', 'SK_ID_CURR'])
 train_features, test_features, train_labels, test_labels = train_test_split(features, labels, test_size = 6000, random_state = 50 )
test = online_sales [['Date', 'Product SKU', 'Quantity', 'Revenue', 'Tax', 'Delivery']]
test['Net_Income'] = test['Revenue'] - test['Tax'] - test['Delivery']
test.head()<feature_engineering>
train_set = lgb.Dataset(data = train_features, label = train_labels)
 test_set = lgb.Dataset(data = test_features, label = test_labels )
kondisi = test['Tax'].isnull()
test.loc[kondisi, ['Tax']] = 1<groupby>
model = lgb.LGBMClassifier()
default_params = model.get_params()
del default_params['n_estimators']
cv_results = lgb.cv(default_params, train_set, num_boost_round = 10000, early_stopping_rounds = 100,
metrics = 'auc', nfold = N_FOLDS, seed = 42 )
test = online_sales.groupby(['Date'])['Quantity'].sum().reset_index()
test.head()<groupby>
print('The maximum validation ROC AUC was: {:.5f} with a standard deviation of {:.5f}.'.format(cv_results['auc-mean'][-1], cv_results['auc-stdv'][-1]))
print('The optimal number of boosting rounds(estimators)was {}.'.format(len(cv_results['auc-mean'])) )
test = online_sales.groupby(['Date', 'Product SKU'])['Quantity'].sum().reset_index()
test.head()<groupby>
from sklearn.metrics import roc_auc_score
test = online_sales.groupby(['Date'] ).agg({'Quantity': 'sum',
'Revenue': 'sum',
'Tax': 'sum',
'Product SKU': 'count',
'Transaction ID': 'count',
} ).reset_index()
test.head()<sort_values>
model.n_estimators = len(cv_results['auc-mean'])
 model.fit(train_features, train_labels)
 preds = model.predict_proba(test_features)[:, 1]
baseline_auc = roc_auc_score(test_labels, preds)
 print('The baseline model scores {:.5f} ROC AUC on the test set.'.format(baseline_auc))
online_sales.sort_values(by=['Quantity'], ascending = False ).head(15 )<drop_column>
def objective(hyperparameters, iteration):

 if 'n_estimators' in hyperparameters.keys() :
del hyperparameters['n_estimators']
cv_results = lgb.cv(hyperparameters, train_set, num_boost_round = 10000, nfold = N_FOLDS,
early_stopping_rounds = 100, metrics = 'auc', seed = 42)
 score = cv_results['auc-mean'][-1]
estimators = len(cv_results['auc-mean'])
 hyperparameters['n_estimators'] = estimators
return [score, hyperparameters, iteration]
test.rename(index=str, columns={"Quantity": "Total Quantity", "Revenue": "Total Revenue"}, inplace = True)
 test.drop(columns=['Product SKU', 'Transaction ID'], inplace = True)
 test.head()<load_from_csv>
score, params, iteration = objective(default_params, 1)
 print('The cross-validation ROC AUC was {:.5f}.'.format(score))
online_sales = online_sales = pd.read_csv('/kaggle/input/uisummerschool/Online_sales.csv')
 backup = online_sales.copy()
daily_online_revenue = online_sales.groupby(['Date'])['Revenue'].sum().reset_index()
daily_online_revenue.tail()<create_dataframe>
model = lgb.LGBMModel()
model.get_params()
add_data = [['2017-12-01', 0], ['2017-12-02', 0], ['2017-12-03', 0],
['2017-12-04', 0], ['2017-12-05', 0], ['2017-12-06', 0],
['2017-12-07', 0], ['2017-12-08', 0], ['2017-12-09', 0],
['2017-12-10', 0], ['2017-12-11', 0], ['2017-12-10', 0],
['2017-12-13', 0], ['2017-12-14', 0]
]
add_data_df = pd.DataFrame(add_data, columns = ['Date', 'Revenue'])
 add_data_df['Date'] = add_data_df['Date'].astype(str)
 add_data_df['Date'] = pd.to_datetime(add_data_df['Date'])
 daily_online_revenue = daily_online_revenue.append(add_data_df)
 df_rev = daily_online_revenue.copy()
daily_online_revenue.tail(20 )<feature_engineering>
param_grid = {
'boosting_type': ['gbdt', 'goss', 'dart'],
'num_leaves': list(range(20, 150)) ,
'learning_rate': list(np.logspace(np.log10(0.005), np.log10(0.5), base = 10, num = 1000)) ,
'subsample_for_bin': list(range(20000, 300000, 20000)) ,
'min_child_samples': list(range(20, 500, 5)) ,
'reg_alpha': list(np.linspace(0, 1)) ,
'reg_lambda': list(np.linspace(0, 1)) ,
'colsample_bytree': list(np.linspace(0.6, 1, 10)) ,
'subsample': list(np.linspace(0.5, 1, 100)) ,
'is_unbalance': [True, False]
}
daily_online_revenue['d-1_rev'] = daily_online_revenue['Revenue'].shift(1)
 daily_online_revenue['d-2_rev'] = daily_online_revenue['Revenue'].shift(2)
 daily_online_revenue['d-3_rev'] = daily_online_revenue['Revenue'].shift(3)
 daily_online_revenue['d-4_rev'] = daily_online_revenue['Revenue'].shift(4)
 daily_online_revenue['d-5_rev'] = daily_online_revenue['Revenue'].shift(5)
 daily_online_revenue = daily_online_revenue.dropna()<split>
a = 0
b = 0
for x in param_grid['learning_rate']:
if x >= 0.005 and x < 0.05:
a += 1
elif x >= 0.05 and x < 0.5:
b += 1
print('There are {} values between 0.005 and 0.05'.format(a))
print('There are {} values between 0.05 and 0.5'.format(b))
train_data = daily_online_revenue.loc[:'2017-11-30']
test_data = daily_online_revenue.loc['2017-12-1':]<prepare_x_and_y>
random_results = pd.DataFrame(columns = ['score', 'params', 'iteration'],
index = list(range(MAX_EVALS)))
 grid_results = pd.DataFrame(columns = ['score', 'params', 'iteration'],
index = list(range(MAX_EVALS)) )
x_train = train_data.drop('Revenue', 1)
 df = train_data[['Revenue']].to_string(index=False ).split('
')
 y_train =pd.DataFrame({'Revenue': df})
 y_train=y_train.drop(y_train.index[[0]])
 x_test = test_data.drop('Revenue', 1)
 df = test_data[['Revenue']].to_string(index=False ).split('
')
 y_test =pd.DataFrame({'Revenue': df})
 y_test=y_test.drop(y_test.index[[0]])
 y_test 
<train_model>
com = 1
for x in param_grid.values() :
com *= len(x)
 print('There are {} combinations'.format(com))
def fit(x_train, y_train):
model = RandomForestRegressor(random_state=1)
 model.fit(x_train, y_train)
 return model
def predict(model, x_test):
y_pred = model.predict(x_test)
 return y_pred
model = fit(x_train, y_train)
<categorify>
print('This would take {:.0f} years to finish.'.format(( 100 * com)/(60 * 60 * 24 * 365)) )
def preprocess(dataset):
processed_dataset = dataset.copy()
processed_dataset['d-1_rev'] = processed_dataset['Revenue'].shift(1)
 processed_dataset['d-2_rev'] = processed_dataset['Revenue'].shift(2)
 processed_dataset['d-3_rev'] = processed_dataset['Revenue'].shift(3)
 processed_dataset['d-4_rev'] = processed_dataset['Revenue'].shift(4)
 processed_dataset['d-5_rev'] = processed_dataset['Revenue'].shift(4)
 processed_dataset = processed_dataset.dropna()
return processed_dataset
def split_label_and_predictor(train_or_test_data):
x_data = train_or_test_data.drop('Revenue', 1)
 df = train_or_test_data[['Revenue']].to_string(index=False ).split('
')
 y_data =pd.DataFrame({'Revenue': df})
 y_data=y_data.drop(y_data.index[[0]])
 return x_data, y_data
def split_train_test(dataset, end_of_training_date):
training_data = dataset.loc[:end_of_training_date]
testing_data = dataset.loc["2017-12-1":]
return training_data, testing_data
df_rev2 = df_rev.copy()
n_iteration = len(x_test)
 result = []
for i in range(n_iteration):
y_pred = predict(model, pd.DataFrame(x_test.iloc[i] ).transpose())
 result.append(y_pred[0])
 df_rev2.loc[df_rev2["Date"]==x_test.index[i],"Revenue"] = y_pred
daily_online_revenue = preprocess(df_rev2 ).set_index('Date')
 _, testing_data = split_train_test(daily_online_revenue,end_of_training_date)
 x_test, _ = split_label_and_predictor(testing_data)
 result<compute_test_metric>
grid_results = grid_search(param_grid)
 print('The best validation score was {:.5f}'.format(grid_results.loc[0, 'score']))
print('
The best hyperparameters were:')
 pprint.pprint(grid_results.loc[0, 'params'] )
comparison = pd.DataFrame({"Prediction":result,"Actual":y_test['Revenue']})
 comparison.index = y_test.index
error = sqrt(mean_squared_error(comparison["Actual"], comparison["Prediction"]))
print("Error Score(RMSE)= {}".format(round(error,2)))
 historical = pd.DataFrame(y_train ).rename(columns={"Revenue":"Actual"} ).tail(14)
 pd.concat([historical,comparison],sort=True ).plot() ;<save_to_csv>
grid_search_params = grid_results.loc[0, 'params']
model = lgb.LGBMClassifier(**grid_search_params, random_state=42)
 model.fit(train_features, train_labels)
 preds = model.predict_proba(test_features)[:, 1]
print('The best model from grid search scores {:.5f} ROC AUC on the test set.'.format(roc_auc_score(test_labels, preds)) )
formatted_result = pd.DataFrame(result ).reset_index().rename(columns={"index":"Id",0:"Revenue"})
 display(formatted_result)
 formatted_result[['Id', 'Revenue']].to_csv("result.csv",index=False )<load_from_csv>
pd.options.display.max_colwidth = 1000
grid_results['params'].values
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 sample = pd.read_csv('.. /input/test.csv' )<prepare_x_and_y>
random.seed(50)
 random_params = {k: random.sample(v, 1)[0] for k, v in param_grid.items() }
random_params['subsample'] = 1.0 if random_params['boosting_type'] == 'goss' else random_params['subsample']
random_params
x_train = train[['LotArea','LotFrontage']].copy()
y_train = train['SalePrice'].copy()<prepare_x_and_y>
def random_search(param_grid, max_evals = MAX_EVALS):

 results = pd.DataFrame(columns = ['score', 'params', 'iteration'],
index = list(range(MAX_EVALS)))
 for i in range(MAX_EVALS):
hyperparameters = {k: random.sample(v, 1)[0] for k, v in param_grid.items() }
hyperparameters['subsample'] = 1.0 if hyperparameters['boosting_type'] == 'goss' else hyperparameters['subsample']
eval_results = objective(hyperparameters, i)
 results.loc[i, :] = eval_results
results.sort_values('score', ascending = False, inplace = True)
 results.reset_index(inplace = True)
 return results
x_test = test[['LotArea','LotFrontage']].copy()<count_missing_values>
random_results = random_search(param_grid)
 print('The best validation score was {:.5f}'.format(random_results.loc[0, 'score']))
print('
The best hyperparameters were:')
 pprint.pprint(random_results.loc[0, 'params'] )
x_train.isnull().sum()<correct_missing_values>
random_search_params = random_results.loc[0, 'params']
model = lgb.LGBMClassifier(**random_search_params, random_state = 42)
 model.fit(train_features, train_labels)
 preds = model.predict_proba(test_features)[:, 1]
print('The best model from random search scores {:.5f} ROC AUC on the test set.'.format(roc_auc_score(test_labels, preds)) )
x_train.fillna(0,inplace=True)
 x_test.fillna(0,inplace=True )<import_modules>
out_file = 'random_search_trials.csv'
of_connection = open(out_file, 'w')
 writer = csv.writer(of_connection)
 headers = ['score', 'hyperparameters', 'iteration']
writer.writerow(headers)
 of_connection.close()
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import StandardScaler<prepare_x_and_y>
def random_search(param_grid, out_file, max_evals = MAX_EVALS):

 results = pd.DataFrame(columns = ['score', 'params', 'iteration'],
index = list(range(MAX_EVALS)))
 for i in range(MAX_EVALS):
random_params = {k: random.sample(v, 1)[0] for k, v in param_grid.items() }
random_params['subsample'] = 1.0 if random_params['boosting_type'] == 'goss' else random_params['subsample']
eval_results = objective(random_params, i)
 results.loc[i, :] = eval_results
of_connection = open(out_file, 'a')
 writer = csv.writer(of_connection)
 writer.writerow(eval_results)
 of_connection.close()
results.sort_values('score', ascending = False, inplace = True)
 results.reset_index(inplace = True)
 return results
x_scaler = StandardScaler()
y_scaler = StandardScaler()<normalization>
def grid_search(param_grid, out_file, max_evals = MAX_EVALS):

 results = pd.DataFrame(columns = ['score', 'params', 'iteration'],
index = list(range(MAX_EVALS)))
 keys, values = zip(*param_grid.items())
 i = 0
for v in itertools.product(*values):
parameters = dict(zip(keys, v))
parameters['subsample'] = 1.0 if parameters['boosting_type'] == 'goss' else parameters['subsample']
eval_results = objective(parameters, i)
 results.loc[i, :] = eval_results
i += 1
of_connection = open(out_file, 'a')
 writer = csv.writer(of_connection)
 writer.writerow(eval_results)
 of_connection.close()
if i > MAX_EVALS:
break
results.sort_values('score', ascending = False, inplace = True)
 results.reset_index(inplace = True)
 return results
x_train_scaler = x_scaler.fit_transform(x_train)
 x_test_scaler = x_scaler.transform(x_test )<choose_model_class>
random_results = pd.read_csv('.. /input/home-credit-model-tuning/random_search_trials_1000.csv')
 grid_results = pd.read_csv('.. /input/home-credit-model-tuning/grid_search_trials_1000.csv' )
lr = LinearRegression()<train_model>
grid_results['hyperparameters'] = grid_results['hyperparameters'].map(ast.literal_eval)
 random_results['hyperparameters'] = random_results['hyperparameters'].map(ast.literal_eval )
lr.fit(x_train_scaler,y_train )<predict_on_test>
def evaluate(results, name):

 results = results.sort_values('score', ascending = False ).reset_index(drop = True)
 print('The highest cross validation score from {} was {:.5f} found on iteration {}.'.format(name, results.loc[0, 'score'], results.loc[0, 'iteration']))
hyperparameters = results.loc[0, 'hyperparameters']
model = lgb.LGBMClassifier(**hyperparameters)
 model.fit(train_features, train_labels)
 preds = model.predict_proba(test_features)[:, 1]
print('ROC AUC from {} on test data = {:.5f}.'.format(name, roc_auc_score(test_labels, preds)))
 hyp_df = pd.DataFrame(columns = list(results.loc[0, 'hyperparameters'].keys()))
for i, hyp in enumerate(results['hyperparameters']):
hyp_df = hyp_df.append(pd.DataFrame(hyp, index = [0]),
ignore_index = True)
 hyp_df['iteration'] = results['iteration']
hyp_df['score'] = results['score']
return hyp_df
pred = lr.predict(x_test_scaler )<create_dataframe>
grid_hyp = evaluate(grid_results, name = 'grid search' )
sub = pd.DataFrame(data = {'Id' : test.Id, 'SalePrice' :pred} )<save_to_csv>
random_hyp = evaluate(random_results, name = 'random search' )
sub.to_csv('submission.csv',index = False )<load_from_csv>
alt.renderers.enable('notebook' )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<import_modules>
random_hyp['search'] = 'random'
grid_hyp['search'] = 'grid'
hyp = random_hyp.append(grid_hyp)
 hyp.head()
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import PolynomialFeatures<drop_column>
best_grid_hyp = grid_hyp.iloc[grid_hyp['score'].idxmax() ].copy()
best_random_hyp = random_hyp.iloc[random_hyp['score'].idxmax() ].copy()
pipe = Pipeline([ 
('scaler', StandardScaler()), 
('poly', PolynomialFeatures()), 
('model', LinearRegression()),
] )<prepare_x_and_y>
print('Average validation score of grid search = {:.5f}.'.format(np.mean(grid_hyp['score'])))
 print('Average validation score of random search = {:.5f}.'.format(np.mean(random_hyp['score'])) )
X_train = train[['LotFrontage','LotArea']].fillna(0)
 X_test = test[['LotFrontage','LotArea']].fillna(0)
 y_train = train['SalePrice']<import_modules>
random_hyp['score'] = random_hyp['score'].astype(float)
 best_random_hyp = random_hyp.loc[0, :].copy()
from sklearn.model_selection import cross_val_score
from sklearn.metrics import mean_squared_error<compute_train_metric>
train = pd.read_csv('.. /input/home-credit-simple-featuers/simple_features_train.csv')
 test = pd.read_csv('.. /input/home-credit-simple-featuers/simple_features_test.csv')
 test_ids = test['SK_ID_CURR']
train_labels = np.array(train['TARGET'].astype(np.int32)).reshape(( -1,))
train = train.drop(columns = ['SK_ID_CURR', 'TARGET'])
 test = test.drop(columns = ['SK_ID_CURR'])
 print('Training shape: ', train.shape)
 print('Testing shape: ', test.shape )
scores = cross_val_score(pipe, X_train, y_train, scoring='neg_mean_squared_error', cv=10 )<feature_engineering>
train_set = lgb.Dataset(train, label = train_labels)
 hyperparameters = dict(**random_results.loc[0, 'hyperparameters'])
 del hyperparameters['n_estimators']
cv_results = lgb.cv(hyperparameters, train_set,
num_boost_round = 10000, early_stopping_rounds = 100,
metrics = 'auc', nfold = N_FOLDS )
np.mean(pow(-scores, 0.5))<train_model>
print('The cross validation score on the full dataset = {:.5f} with std: {:.5f}.'.format(
 cv_results['auc-mean'][-1], cv_results['auc-stdv'][-1]))
print('Number of estimators = {}.'.format(len(cv_results['auc-mean'])) )
pipe.fit(X_train, y_train )<predict_on_test>
model = lgb.LGBMClassifier(n_estimators = len(cv_results['auc-mean']), **hyperparameters)
 model.fit(train, train_labels)
 preds = model.predict_proba(test)[:, 1]
preds = pipe.predict(X_test )<save_to_csv>
submission = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': preds})
 submission.to_csv('submission_simple_features_random.csv', index = False )