Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<drop_column>
pd.set_option('display.max_columns', 200)
 InteractiveShell.ast_node_interactivity = "all"
warnings.filterwarnings('ignore' )
train_data.drop(['key', 'pickup_datetime'],axis =1, inplace = True)
 train_data.head()<prepare_x_and_y>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv' )
test_data = date_time_info(test_data)
 test_data['distance'] = distance(test_data['pickup_latitude'], test_data['pickup_longitude'],
test_data['dropoff_latitude'] , test_data['dropoff_longitude'])
 test_key = test_data['key']
x_pred = test_data.drop(columns=['key', 'pickup_datetime'] )<prepare_x_and_y>
train_df.isnull().sum().sum()
test_df.isnull().sum().sum()
y = train_data['fare_amount']
X = train_data.drop(['fare_amount'],axis=1)
 cv_split = KFold(n_splits=10,random_state=0)
<train_model>
def add_new_feature_row(df,features):
for feature in features:
df[feature+"_pct"] = df[feature].pct_change()
df[feature+"_diff"] = df[feature].diff()
df.drop(feature,axis=1)
 return df
xgb = XGBRegressor(random_state=0)
 base_results = cross_validate(xgb, X,y, cv = cv_split)
 xgb.fit(X,y)
<train_on_grid>
def normalize_df(df,features):
for feature in features:
df[feature+'_norm'] =(df[feature] - df[feature].mean())/df[feature].std()
return df
params = {
'max_depth': [8],
'eta':[.03],
'subsample': [1],
'colsample_bytree': [0.8],
'objective':['reg:linear'],
'eval_metric':['rmse'],
'silent': [1]
}
submit_xgb = GridSearchCV(XGBRegressor() , param_grid=params,
scoring = 'neg_mean_squared_error',
cv = cv_split)
 submit_xgb.fit(X,y)
<predict_on_test>
correlations = train_df[features].corr().abs().unstack().sort_values(kind="quicksort" ).reset_index()
correlations = correlations[correlations['level_0'] != correlations['level_1']]
prediction = submit_xgb.predict(x_pred )<save_to_csv>
gc.collect()
submission = pd.DataFrame({
"key": test_key,
"fare_amount": prediction.round(2)
 })
 submission.to_csv('taxi_fare_submission.csv',index=False)
 submission.head()<split>
test_df['target']= np.nan
combine_df = train_df.append(test_df,ignore_index=True )
def incremental_learning(data, model):
start = time.time()
data["pickup_datetime"] = pd.to_datetime(data["pickup_datetime"], infer_datetime_format=True)
 print("Conversion took", time.time() - start, "s")
 add_travel_vector_features(data)
 data = pd.concat([data, timeExpansion(data["pickup_datetime"])], axis=1)
 X_tr, X_te, y_tr, y_te = train_test_split(
 data.drop(["key", "pickup_datetime", "fare_amount"], axis=1),
data['fare_amount'],
test_size=0.2)
 del data
params = {
'boosting_type': 'gbdt',
'objective': 'regression_l1',
'metric': {'l1'},
'num_leaves': 12,
'learning_rate': 0.1,
'feature_fraction': 0.9,
'bagging_fraction': 0.8,
'random_state': 77,
'n_jobs': 4,
'bagging_freq': 5,
'verbose': 2
}
dset = lgbm.Dataset(X_tr, y_tr, free_raw_data=True)
 model = lgbm.train(params,
init_model = model,
train_set = dset,
keep_training_booster = True,
num_boost_round = 100)
 MAE = mean_absolute_error(y_te, model.predict(X_te))
print("Mean Absolute Error:", MAE)
 return model<load_from_csv>
features = train_df.columns.values[2:]
combine_df = normalize_df(combine_df,features )
def partial_import(filename, skip, rows):
return pd.read_csv('.. input/train.csv', skiprows=skip, nrows=rows)
 def add_travel_vector_features(df):
df['abs_diff_longitude'] =(df.dropoff_longitude - df.pickup_longitude ).abs()
df['abs_diff_latitude'] =(df.dropoff_latitude - df.pickup_latitude ).abs()
def month_translation(number):
month_list = ["jan", "feb", "mar",
"apr", "may", "jun", "jul",
"aug", "sep", "oct", "nov", "dec"]
for num, i in enumerate(month_list):
if num == number-1:
return i
def quarter_translation(number):
quarter_border = [0, 3, 6, 9, 12, 15, 18, 21, 24]
quarter_list = ["11", "12", "13", "14", "21", "22", "23", "24"]
for num, i in enumerate(range(1, len(quarter_border))):
if(number < quarter_border[i])&(number >= quarter_border[i-1]):
return quarter_list[num]
def week_translation(number):
week_border = [0, 7, 14, 21, 31]
week_list = ["1W", "2W", "3W", "4W"]
for num, i in enumerate(range(1, len(week_border))):
if(number <= week_border[i])&(number > week_border[i-1]):
return week_list[num]
def timeExpansion(timeSeries):
additional_cols = ["year", "month", "quarter", "week"]
dummy0 = pd.DataFrame(np.zeros(( len(timeSeries), len(additional_cols)) , 'int'),
columns=additional_cols,
index=timeSeries.index)
 dummy0["year"] = [i.year for i in timeSeries]
dummy0["month"] = [month_translation(i.month)for i in timeSeries]
dummy0["day"] = [week_translation(i.day)for i in timeSeries]
dummy0["quarter"] = [quarter_translation(i.hour)for i in timeSeries]
return pd.get_dummies(dummy0)
 def predict_and_submit(submit=True):
if "model.pkl" in files:
with open('.. /model.pkl', 'rb')as fin:
estimator = pickle.load(fin)
 test_df = pd.read_csv('.. /input/test.csv')
 add_travel_vector_features(test_df)
 test_df["pickup_datetime"] = pd.to_datetime(test_df["pickup_datetime"], infer_datetime_format=True)
 test_X = pd.concat([test_df, timeExpansion(test_df["pickup_datetime"])], axis=1)
 test_X.drop(["key", "pickup_datetime"], axis=1, inplace=True)
 print("Test Shape: ",test_X.shape)
 test_y_predictions = estimator.predict(test_X)
 if submit:
submission = pd.DataFrame(
 {'key': test_df.key, 'fare_amount': test_y_predictions},
columns = ['key', 'fare_amount'])
 submission.to_csv('submission.csv', index = False )<load_from_csv>
train_df = combine_df[combine_df['target'].notnull() ].reset_index(drop=True)
 test_df = combine_df[combine_df['target'].isnull() ].reset_index(drop=True)
training_step = list(np.arange(40_000_000, 55_000_001, 1_000_000))
estimator = None
origin_cols = ['key',
'fare_amount',
'pickup_datetime',
'pickup_longitude',
'pickup_latitude',
'dropoff_longitude',
'dropoff_latitude',
'passenger_count']
for i in range(1, len(training_step)) :
start = time.time()
files = os.listdir(".. /")
 if "model.pkl" in files:
with open('.. /model.pkl', 'rb')as fin:
estimator = pickle.load(fin)
 print(training_step[i-1], training_step[i])
 train_df = pd.read_csv(".. /input/train.csv", skiprows=training_step[i-1], nrows=training_step[i])
 train_df.columns = origin_cols
estimator = incremental_learning(train_df, estimator)
 print("Operation", i, "took ", time.time() - start, "
")
 del train_df
with open('.. /model.pkl', 'wb')as fout:
pickle.dump(estimator, fout)
 del estimator
gc.collect()
predict_and_submit()<import_modules>
predictors = train_df.columns.values.tolist() [2:]
nfold = 10
target = 'target'
import numpy as np
import pandas as pd
from math import sqrt
import os 
<set_options>
param = {
'num_leaves': 18,
'max_bin': 63,
'min_data_in_leaf': 5,
'learning_rate': 0.010614430970330217,
'min_sum_hessian_in_leaf': 0.0093586657313989123,
'feature_fraction': 0.056701788569420042,
'lambda_l1': 0.060222413158420585,
'lambda_l2': 4.6580550589317573,
'min_gain_to_split': 0.29588543202055562,
'max_depth': 49,
'save_binary': True,
'seed': 1337,
'feature_fraction_seed': 1337,
'bagging_seed': 1337,
'drop_seed': 1337,
'data_random_seed': 1337,
'objective': 'binary',
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'is_unbalance': True,
'boost_from_average': False
}
nfold = 10
skf = StratifiedKFold(n_splits=nfold, shuffle=True, random_state=42)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
i = 1
for train_index, valid_index in skf.split(train_df, train_df.target.values):
print("
fold {}".format(i))
xg_train = lgb.Dataset(train_df.iloc[train_index][predictors].values,
label=train_df.iloc[train_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 xg_valid = lgb.Dataset(train_df.iloc[valid_index][predictors].values,
label=train_df.iloc[valid_index][target].values,
feature_name=predictors,
free_raw_data = False 
)
 nround = 8523
clf = lgb.train(param,
xg_train,
nround,
valid_sets = [xg_valid],
early_stopping_rounds=250,
verbose_eval=250)
 oof[valid_index] = clf.predict(train_df.iloc[valid_index][predictors].values, num_iteration=nround)
 predictions += clf.predict(test_df[predictors], num_iteration=nround)/ nfold
i = i + 1
print("
CV AUC: {:<0.4f}".format(metrics.roc_auc_score(train_df.target.values, oof)) )
<define_variables><EOS> 
sub_df = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub_df["target"] = predictions
sub_df.to_csv("sant_lgb.csv", index=False)
 sub_df[:10]