Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<split>
warnings.filterwarnings('ignore')
 PATH=".. /input/"
N_SPLITS = 10
SEED_SKF = 4221
X_train, x_test, Y_train, y_test= train_test_split(X_train,y_train,test_size=0.1,random_state=0 )<choose_model_class>
def merge_train_test(df_train, df_test):
if "target" not in df_test.columns.values:
df_test["target"] = -1
res = pd.concat([df_train, df_test])
 res.reset_index(inplace=True, drop=True)
 return res
def split_train_test(df):
df_train = df[df["target"] >= 0]
df_test = df[df["target"] <= -1]
df_train.reset_index(inplace=True, drop=True)
 df_test.reset_index(inplace=True, drop=True)
 assert list(df_train["ID_code"].values)== [f"train_{i}" for i in range(200000)]
assert list(df_test["ID_code"].values)== [f"test_{i}" for i in range(200000)]
return df_train, df_test
classifier=Sequential()
classifier.add(Conv2D(64,3,3, input_shape=(28,28,1), activation='relu'))
classifier.add(BatchNormalization())
 classifier.add(Conv2D(64,3,3, activation='relu'))
classifier.add(BatchNormalization())
 classifier.add(Conv2D(64,3,3, activation='relu'))
classifier.add(BatchNormalization())
 classifier.add(MaxPooling2D(pool_size=(2,2)))
 classifier.add(Dropout(0.4))
classifier.add(Conv2D(64,3,3, activation='relu'))
classifier.add(BatchNormalization())
 classifier.add(Conv2D(64,3,3, activation='relu'))
classifier.add(BatchNormalization())
 classifier.add(Conv2D(64,3,3, activation='relu'))
classifier.add(BatchNormalization())
 classifier.add(MaxPooling2D(pool_size=(2,2)))
 classifier.add(Dropout(0.4))
classifier.add(Flatten())
 classifier.add(Dense(output_dim=256,activation='relu'))
classifier.add(Dropout(0.4))
classifier.add(Dense(output_dim=512,activation='relu'))
classifier.add(Dropout(0.4))
classifier.add(Dense(output_dim=1024,activation='relu'))
classifier.add(Dropout(0.5))
classifier.add(Dense(output_dim=10, activation='softmax'))<choose_model_class>
%%time
train_df = pd.read_csv(PATH+"train.csv")
 test_df = pd.read_csv(PATH+"test.csv" )
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0 )<choose_model_class>
class CountEncoder:
def fit(self, series):
self.counts = series.groupby(series ).count()
def transform(self, series):
return series.map(self.counts ).fillna(0 ).astype(np.int16 )
classifier.compile(optimizer=optimizer, loss='categorical_crossentropy',
metrics=['accuracy'] )<set_options>
df_cnt = pd.DataFrame()
for v in range(200):
sr = test_df[f"var_{v}"]
enc = CountEncoder()
enc.fit(sr)
 df_cnt[f"cnt_{v}"] = enc.transform(sr)
 test_df["target"] = -df_cnt.min(1)
 del df_cnt
datagen = ImageDataGenerator(zoom_range = 0.1,
height_shift_range = 0.1,
width_shift_range = 0.1,
rotation_range = 10 )<set_options>
df_merged = merge_train_test(train_df, test_df)
 df_merged.tail()
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.9 ** x )<train_model>
%%time
count_enc = [None] * 200
df_real = df_merged[df_merged["target"]!=-2]
for v in range(200):
enc = CountEncoder()
enc.fit(df_real[f"var_{v}"])
 count_enc[v] = enc.transform(df_merged[f"var_{v}"])
 for v in range(200):
df_merged[f"cnt_{v}"] = count_enc[v]
del df_real
classifier.fit_generator(datagen.flow(X_train, Y_train, batch_size=16),
steps_per_epoch=500,
epochs=40,
verbose=2,
validation_data=(x_test[:400,:], y_test[:400,:]),
callbacks=[annealer] )<predict_on_test>
train_df, test_df = split_train_test(df_merged)
 target = train_df['target']
gc.collect()
print(train_df.shape)
 test_df.head()
result=classifier.predict(X_test)
 result=pd.Series(np.argmax(result, axis=1), name='Label')
 result<save_to_csv>
param = {
"objective": "binary",
"boost": "gbdt",
"metric": "auc",
"boost_from_average": False,
"learning_rate": 0.01,
"num_leaves": 5,
"max_depth": -1,
"tree_learner": "serial",
"feature_fraction": 1.0,
"bagging_freq": 5,
"bagging_fraction": 0.4,
"min_data_in_leaf": 80,
"min_sum_hessian_in_leaf": 10.0,
"verbosity": 1,
"seed": 44000,
}
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),result],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )<load_from_csv>
target = train_df['target']
df_merged_cut = [df_merged[[f"var_{v}",
f"cnt_{v}",
]] for v in range(200)]
gc.collect()
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
print('--------------------------------------')
 print('Train data looks like...')
 trainData = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 print(trainData.head(5))
print('Test data looks like...')
 testData = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 print(testData.head(5))<data_type_conversions>
%%time
skf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED_SKF)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(skf.split(train_df.values, target.values)) :
print("fold n°{}".format(fold_))
df_meta = df_merged[["ID_code", "target"]]
trn_X, trn_y = train_df.iloc[trn_idx], target.iloc[trn_idx]
val_X, val_y = train_df.iloc[val_idx], target.iloc[val_idx]
for v in range(200):
print(f"var {v}")
 features = [f"var_{v}",
f"cnt_{v}",
]
trn_data = lgb.Dataset(trn_X[features], label=trn_y)
 val_data = lgb.Dataset(val_X[features], label=val_y)
 num_round = 1000000
clf = lgb.train(param,
trn_data,
num_round,
valid_sets=[trn_data, val_data],
verbose_eval=1000,
early_stopping_rounds=100)
 df_meta[f"{v}_meta"] = clf.predict(df_merged_cut[v], num_iteration=clf.best_iteration ).astype(np.float32)
 df_meta.to_pickle(f"fold_{fold_}_meta.pickle" )
def kaggle(dfTrain, dfTest):
pd.set_option('display.max_columns', None)
 dfTest['DateNew'] = pd.to_datetime(dfTest['Date'])
 dfTest = dfTest.drop(['Date'], axis=1)
 dfTest = dfTest.rename(columns={"DateNew": "Date"})
 dfTest['Year'] = dfTest['Date'].dt.year
dfTest['Month'] = dfTest['Date'].dt.month
dfTest['Day'] = dfTest['Date'].dt.day
dfTest = dfTest.drop(['Date'], axis=1)
 dfTest = dfTest.fillna('DummyProvince')
 dfTrain['DateNew'] = pd.to_datetime(dfTrain['Date'])
 dfTrain = dfTrain.drop(['Date'], axis=1)
 dfTrain = dfTrain.rename(columns={"DateNew": "Date"})
 dfTrain['Year'] = dfTrain['Date'].dt.year
dfTrain['Month'] = dfTrain['Date'].dt.month
dfTrain['Day'] = dfTrain['Date'].dt.day
dfTrain = dfTrain.drop(['Date'], axis=1)
 dfTrain = dfTrain.fillna('DummyProvince')
 result = pd.merge(dfTest, dfTrain, how='left', on=['Country_Region', 'Province_State', 'Year', 'Month', 'Day'])
 result = result.fillna(-1)
 [rr, cc] = np.shape(result)
 for iQuit in range(rr):
if result.loc[iQuit, 'Day'] == 4 :
result.loc[iQuit, 'ConfirmedCases'] = -1
result.loc[iQuit, 'Fatalities'] = -1
return result
def ErrorCalc(mdl, ref, tag):
relError = np.abs(mdl - ref)/ np.abs(ref+1)
 MeanErrorV = np.mean(relError)
 print(tag + ': Mean Rel Error in %: ', MeanErrorV * 100)
 return MeanErrorV
def AdjustingErrorsOutliers(tempPred, df):
tempPred = np.round(tempPred)
 tempPrev = df['day5'].to_numpy()
for i in range(len(tempPred)) :
if tempPred[i] < tempPrev[i] :
tempPred[i] = tempPrev[i]
return tempPred
def TrainMdl(trainIpData, trainOpData):
testSize = 0.1
print('Training starts...')
 randomState=None
X_train, X_test, y_train, y_test = train_test_split(trainIpData, trainOpData, test_size=testSize, random_state=randomState)
 TrainIP = X_train[['day1', 'day2', 'day3', 'day4', 'day5', 'diff1', 'diff2', 'diff3', 'diff4']]
TrainOP = X_train['gammaFun']
TestIP = X_test[['day1', 'day2', 'day3', 'day4', 'day5', 'diff1', 'diff2', 'diff3', 'diff4']]
TestOP = X_test['gammaFun']
treeDepth = 10
mdl = DecisionTreeRegressor(max_depth=treeDepth)
 param_grid = {
'n_estimators': [100, 250, 500],
'learning_rate': [0.1, 0.01, 0.001]
}
regrMdl = AdaBoostRegressor(base_estimator=mdl)
 clf = RandomizedSearchCV(estimator = regrMdl, param_distributions = param_grid,
n_iter = 100, cv = 3, verbose=0, random_state=42, n_jobs = -1)
 clf.fit(TrainIP, TrainOP)
 y_predictedTrain = clf.predict(TrainIP)
 y_predictedTrain = AdjustingErrorsOutliers(y_predictedTrain * TrainIP['day5'].to_numpy() , TrainIP)
 ErrorCalc(y_predictedTrain, y_train.to_numpy() , 'Train Data-set')
 y_predictedTest = clf.predict(TestIP)
 y_predictedTest = AdjustingErrorsOutliers(y_predictedTest * TestIP['day5'].to_numpy() , TestIP)
 ErrorCalc(y_predictedTest, y_test.to_numpy() , 'Validation Data-set ')
 print('-----------------------------------------------------------')
 dfTrain = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 dfTest = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 df = kaggle(dfTrain, dfTest)
 print('Prediction starts...')
 [rr, cc] = np.shape(df)
 for iP in range(rr):
if df.loc[iP, 'ConfirmedCases'] == -1 :
day5 = df.loc[iP-1, 'ConfirmedCases']
day4 = df.loc[iP-2, 'ConfirmedCases']
day3 = df.loc[iP-3, 'ConfirmedCases']
day2 = df.loc[iP-4, 'ConfirmedCases']
day1 = df.loc[iP-5, 'ConfirmedCases']
diff1 = day5 - day4
diff2 = day4 - day3
diff3 = day3 - day2
diff4 = day2 - day1
data = {'day1': [day1], 'day2': [day2], 'day3': [day3], 'day4': [day4], 'day5': [day5],
'diff1': [diff1], 'diff2': [diff2], 'diff3': [diff3], 'diff4': [diff4]}
dfPredict = pd.DataFrame(data)
 finalPrediction = clf.predict(dfPredict[['day1', 'day2', 'day3', 'day4', 'day5', 'diff1', 'diff2', 'diff3', 'diff4']])* day5
if finalPrediction < day5 :
finalPrediction = day5
df.loc[iP, 'ConfirmedCases'] = np.round(finalPrediction)
 return df
df = pd.read_csv(".. /input/processedtimedata/TrainTest.csv")
 trainIpData = df[['day1', 'day2', 'day3', 'day4', 'day5', 'gammaFun', 'diff1', 'diff2', 'diff3', 'diff4']]
trainOpData = df['dayPredict']
predictions_dF = TrainMdl(trainIpData, trainOpData)
 print('Completed...')
<feature_engineering>
param = {
"objective": "binary",
"boost": "gbdt",
"metric": "auc",
"boost_from_average": "false",
"learning_rate": 0.01,
"num_leaves": 2,
"max_depth": -1,
"tree_learner": "serial",
"feature_fraction": 0.5,
"bagging_freq": 5,
"bagging_fraction": 0.4,
"min_data_in_leaf": 80,
"min_sum_hessian_in_leaf": 10.0,
"verbosity": 1,
"seed": 44000,
}
def kaggle(dfTrain, dfTest):
pd.set_option('display.max_columns', None)
 dfTest['DateNew'] = pd.to_datetime(dfTest['Date'])
 dfTest = dfTest.drop(['Date'], axis=1)
 dfTest = dfTest.rename(columns={"DateNew": "Date"})
 dfTest['Year'] = dfTest['Date'].dt.year
dfTest['Month'] = dfTest['Date'].dt.month
dfTest['Day'] = dfTest['Date'].dt.day
dfTest = dfTest.drop(['Date'], axis=1)
 dfTest = dfTest.fillna('DummyProvince')
 dfTrain['DateNew'] = pd.to_datetime(dfTrain['Date'])
 dfTrain = dfTrain.drop(['Date'], axis=1)
 dfTrain = dfTrain.rename(columns={"DateNew": "Date"})
 dfTrain['Year'] = dfTrain['Date'].dt.year
dfTrain['Month'] = dfTrain['Date'].dt.month
dfTrain['Day'] = dfTrain['Date'].dt.day
dfTrain = dfTrain.drop(['Date'], axis=1)
 dfTrain = dfTrain.fillna('DummyProvince')
 result = pd.merge(dfTest, dfTrain, how='left', on=['Country_Region', 'Province_State', 'Year', 'Month', 'Day'])
 result = result.fillna(-1)
 [rr, cc] = np.shape(result)
 for iQuit in range(rr):
if result.loc[iQuit, 'Day'] == 4 :
result.loc[iQuit, 'ConfirmedCases'] = -1
result.loc[iQuit, 'Fatalities'] = -1
return result
def ErrorCalc(mdl, ref, tag):
relError = np.abs(mdl - ref)/ np.abs(ref+1)
 MeanErrorV = np.mean(relError)
 print(tag + ': Mean Rel Error in %: ', MeanErrorV * 100)
 return MeanErrorV
def AdjustingErrorsOutliers(tempPred, df):
tempPred = np.round(tempPred)
 tempPrev = df['day5'].to_numpy()
for i in range(len(tempPred)) :
if tempPred[i] < tempPrev[i] :
tempPred[i] = tempPrev[i]
return tempPred
def TrainMdl(trainIpData, trainOpData):
testSize = 0.1
print('Training starts...')
 randomState=None
X_train, X_test, y_train, y_test = train_test_split(trainIpData, trainOpData, test_size=testSize, random_state=randomState)
 TrainIP = X_train[['day1', 'day2', 'day3', 'day4', 'day5', 'diff1', 'diff2', 'diff3', 'diff4']]
TrainOP = X_train['gammaFun']
TestIP = X_test[['day1', 'day2', 'day3', 'day4', 'day5', 'diff1', 'diff2', 'diff3', 'diff4']]
TestOP = X_test['gammaFun']
treeDepth = 10
mdl = DecisionTreeRegressor(max_depth=treeDepth)
 param_grid = {
'n_estimators': [100, 250, 500],
'learning_rate': [0.1, 0.01, 0.001]
}
regrMdl = AdaBoostRegressor(base_estimator=mdl)
 clf = RandomizedSearchCV(estimator = regrMdl, param_distributions = param_grid,
n_iter = 100, cv = 3, verbose=0, random_state=42, n_jobs = -1)
 clf.fit(TrainIP, TrainOP)
 y_predictedTrain = clf.predict(TrainIP)
 y_predictedTrain = AdjustingErrorsOutliers(y_predictedTrain * TrainIP['day5'].to_numpy() , TrainIP)
 ErrorCalc(y_predictedTrain, y_train.to_numpy() , 'Train Data-set')
 y_predictedTest = clf.predict(TestIP)
 y_predictedTest = AdjustingErrorsOutliers(y_predictedTest * TestIP['day5'].to_numpy() , TestIP)
 ErrorCalc(y_predictedTest, y_test.to_numpy() , 'Validation Data-set ')
 print('-----------------------------------------------------------')
 dfTrain = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 dfTest = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 df = kaggle(dfTrain, dfTest)
 [rr, cc] = np.shape(df)
 for iP in range(rr):
if df.loc[iP, 'Fatalities'] == -1 :
day5 = df.loc[iP-1, 'Fatalities']
day4 = df.loc[iP-2, 'Fatalities']
day3 = df.loc[iP-3, 'Fatalities']
day2 = df.loc[iP-4, 'Fatalities']
day1 = df.loc[iP-5, 'Fatalities']
diff1 = day5 - day4
diff2 = day4 - day3
diff3 = day3 - day2
diff4 = day2 - day1
data = {'day1': [day1], 'day2': [day2], 'day3': [day3], 'day4': [day4], 'day5': [day5],
'diff1': [diff1], 'diff2': [diff2], 'diff3': [diff3], 'diff4': [diff4]}
dfPredict = pd.DataFrame(data)
 finalPrediction = clf.predict(dfPredict[['day1', 'day2', 'day3', 'day4', 'day5', 'diff1', 'diff2', 'diff3', 'diff4']])* day5
if finalPrediction < day5 :
finalPrediction = day5
df.loc[iP, 'Fatalities'] = np.round(finalPrediction)
 return df
df = pd.read_csv(".. /input/processedtimedata/TrainTest_Fatality.csv")
 trainIpData = df[['day1', 'day2', 'day3', 'day4', 'day5', 'gammaFun', 'diff1', 'diff2', 'diff3', 'diff4']]
trainOpData = df['dayPredict']
fatality_dF = TrainMdl(trainIpData, trainOpData)
 print('Completed...')
<save_to_csv>
%%time
skf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED_SKF)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(skf.split(train_df.values, target.values)) :
print("fold n°{}".format(fold_))
df_meta = pd.read_pickle(f"fold_{fold_}_meta.pickle")
 train_df, test_df = split_train_test(df_meta)
 features = [f"{v}_meta" for v in range(200)]
trn_data = lgb.Dataset(train_df.iloc[trn_idx][features], label=target.iloc[trn_idx])
 val_data = lgb.Dataset(train_df.iloc[val_idx][features], label=target.iloc[val_idx])
 num_round = 1000000
clf = lgb.train(param,
trn_data,
num_round,
valid_sets=[trn_data, val_data],
verbose_eval=1000,
early_stopping_rounds=2000)
 oof[val_idx] = clf.predict(train_df.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test_df[features], num_iteration=clf.best_iteration)/ N_SPLITS
print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)) )
predictions_dF['Fatalities'] = fatality_dF['Fatalities']
predictions_dF[['ForecastId', 'ConfirmedCases', 'Fatalities']].to_csv('submission.csv', index=False)
 print(predictions_dF[['ForecastId', 'ConfirmedCases', 'Fatalities']].head(10))
print(predictions_dF[['ForecastId', 'ConfirmedCases', 'Fatalities']].tail(10))<set_options>
sub_df = pd.DataFrame({"ID_code":test_df["ID_code"].values})
 sub_df["target"] = predictions
sub_df.to_csv("submission.csv", index=False )
<load_from_csv><EOS> 
feature_importance_df.to_csv("feature_importance_df.csv", index=False )