Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<create_dataframe>
!pip install scorecardpy
train_profile = ProfileReport(xtrain, title='Pandas Profiling Report', html={'style':{'full_width':True}})
 train_profile<prepare_x_and_y>
import numpy as np
import pandas as pd
from scipy.special import logit
import lightgbm as lgb
import scorecardpy as sc
xtrain.rename(columns={'Country_Region':'Country'}, inplace=True)
 xtest.rename(columns={'Country_Region':'Country'}, inplace=True)
 xtrain.rename(columns={'Province_State':'State'}, inplace=True)
 xtest.rename(columns={'Province_State':'State'}, inplace=True)
 xtrain['Date'] = pd.to_datetime(xtrain['Date'], infer_datetime_format=True)
 xtest['Date'] = pd.to_datetime(xtest['Date'], infer_datetime_format=True)
 xtrain.info()
xtest.info()
y1_xTrain = xtrain.iloc[:, -2]
y1_xTrain.head()
y2_xTrain = xtrain.iloc[:, -1]
y2_xTrain.head()
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state<data_type_conversions>
train = pd.read_csv(".. /input/santander-customer-transaction-prediction/train.csv")
 test = pd.read_csv(".. /input/santander-customer-transaction-prediction/test.csv")
 train = train.drop('ID_code', axis = 1)
 train.head()
X_xTrain = xtrain.copy()
X_xTrain['State'].fillna(EMPTY_VAL, inplace=True)
 X_xTrain['State'] = X_xTrain.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_xTrain.loc[:, 'Date'] = X_xTrain.Date.dt.strftime("%m%d")
 X_xTrain["Date"] = X_xTrain["Date"].astype(int)
 X_xTrain.head()
X_xTest = xtest.copy()
X_xTest['State'].fillna(EMPTY_VAL, inplace=True)
 X_xTest['State'] = X_xTest.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_xTest.loc[:, 'Date'] = X_xTest.Date.dt.strftime("%m%d")
 X_xTest["Date"] = X_xTest["Date"].astype(int)
 X_xTest.head()<categorify>
test_id = test.ID_code
test = test.drop('ID_code', axis = 1)
 test.head()
le = preprocessing.LabelEncoder()
X_xTrain.Country = le.fit_transform(X_xTrain.Country)
 X_xTrain['State'] = le.fit_transform(X_xTrain['State'])
 X_xTrain.head()
X_xTest.Country = le.fit_transform(X_xTest.Country)
 X_xTest['State'] = le.fit_transform(X_xTest['State'])
 X_xTest.head()
xtrain.head()
xtrain.loc[xtrain.Country == 'Afghanistan', :]
xtest.tail()<categorify>
print(f"The number of missing values in the training set is: {np.sum(np.sum(pd.isnull(train)))}")
 print(f"The number of missing values in the test set is: {np.sum(np.sum(pd.isnull(test)))}" )
filterwarnings('ignore')
 le = preprocessing.LabelEncoder()
countries = X_xTrain.Country.unique() 
<prepare_x_and_y>
correlations = train.drop("target", axis = 1 ).corr().abs().unstack().sort_values(kind = "quicksort" ).reset_index()
correlations = correlations[correlations['level_0'] != correlations['level_1']]
correlations.head(10 )
xout = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = X_xTrain.loc[X_xTrain.Country == country, :].State.unique()
for state in states:
X_xTrain_CS = X_xTrain.loc[(X_xTrain.Country == country)&(X_xTrain.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
y1_xTrain_CS = X_xTrain_CS.loc[:, 'ConfirmedCases']
y2_xTrain_CS = X_xTrain_CS.loc[:, 'Fatalities']
X_xTrain_CS = X_xTrain_CS.loc[:, ['State', 'Country', 'Date']]
X_xTrain_CS.Country = le.fit_transform(X_xTrain_CS.Country)
 X_xTrain_CS['State'] = le.fit_transform(X_xTrain_CS['State'])
 X_xTest_CS = X_xTest.loc[(X_xTest.Country == country)&(X_xTest.State == state), ['State', 'Country', 'Date', 'ForecastId']]
X_xTest_CS_Id = X_xTest_CS.loc[:, 'ForecastId']
X_xTest_CS = X_xTest_CS.loc[:, ['State', 'Country', 'Date']]
X_xTest_CS.Country = le.fit_transform(X_xTest_CS.Country)
 X_xTest_CS['State'] = le.fit_transform(X_xTest_CS['State'])
 xmodel1 = XGBRegressor(n_estimators=1000)
 xmodel1.fit(X_xTrain_CS, y1_xTrain_CS)
 y1_xpred = xmodel1.predict(X_xTest_CS)
 xmodel2 = XGBRegressor(n_estimators=1000)
 xmodel2.fit(X_xTrain_CS, y2_xTrain_CS)
 y2_xpred = xmodel2.predict(X_xTest_CS)
 xdata = pd.DataFrame({'ForecastId': X_xTest_CS_Id, 'ConfirmedCases': y1_xpred, 'Fatalities': y2_xpred})
 xout = pd.concat([xout, xdata], axis=0)
<save_to_csv>
variables = train.drop("target", axis = 1 ).columns.values.tolist()
corr_pre_res = np.zeros(len(variables))
i = 0
for var in variables:
corr_pre_res[i] = np.corrcoef(train[var], train["target"])[0, 1]
i += 1
xout.ForecastId = xout.ForecastId.astype('int')
 xout.tail()
xout.to_csv('submission.csv', index=False )<load_from_csv>
corr_pre_res = abs(pd.DataFrame(corr_pre_res))
corr_pre_res.columns = ['corr_pre_res']
corr_pre_res.sort_values(by = 'corr_pre_res' )
<count_missing_values><EOS> 
features = [x for x in train.columns if x.startswith("var")]
hist_df = pd.DataFrame()
for var in features:
var_stats = train[var].append(test[var] ).value_counts()
hist_df[var] = pd.Series(test[var] ).map(var_stats)
 hist_df[var] = hist_df[var] > 1
ind = hist_df.sum(axis = 1)!= 200
var_stats = {var: train[var].append(test[ind][var] ).value_counts() for var in features}
pred = 0
for var in features:
model = lgb.LGBMClassifier(**{'learning_rate': 0.05,
'max_bin': 165,
'max_depth': 5,
'min_child_samples': 150,
'min_child_weight': 0.1,
'min_split_gain': 0.0018,
'n_estimators': 41,
'num_leaves': 6,
'reg_alpha': 2.0,
'reg_lambda': 2.54,
'objective': 'binary',
'n_jobs': -1})
 model = model.fit(np.hstack([train[var].values.reshape(-1, 1),
train[var].map(var_stats[var] ).values.reshape(-1, 1)]), train["target"].values)
 pred += logit(model.predict_proba(np.hstack([test[var].values.reshape(-1, 1),
test[var].map(var_stats[var] ).values.reshape(-1, 1)])) [:, 1])
 pd.DataFrame({"ID_code": test_id, "target": pred} ).to_csv("submission.csv", index = False )