Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<define_variables>
warnings.filterwarnings("ignore")
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 300)
 pd.set_option('display.max_colwidth', 5000)
 pd.options.display.float_format = '{:.3f}'.format
%matplotlib inline
plt.style.use('fivethirtyeight')
 print(os.listdir(".. /input/"))
DIR_NAME = ".. /input"
val2.columns = pd.Index([str(i)for i in range(8,16)])
 val2.head()<concatenate>
from sklearn import preprocessing
from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold, KFold, RepeatedKFold, GridSearchCV
from sklearn.metrics import classification_report, roc_auc_score, roc_curve, auc
import xgboost as xgb
val_pred = pd.concat([val1,val2],axis=1)
 print(val_pred.shape)
 val_pred.head()<load_from_csv>
def load_dir_csv(directory, csv_files=None):
if csv_files is None:
csv_files = sorted([ f for f in os.listdir(directory)if f.endswith(".csv")])
 csv_vars = [ filename[:-4] for filename in csv_files ]
gdict = globals()
for filename, var in zip(csv_files, csv_vars):
print(f"{var:32s} = pd.read_csv({directory}/{filename})")
 gdict[var] = pd.read_csv(f"{directory}/{filename}")
 print(f"{'shape ':32s} = " + str(gdict[var].shape))
display(gdict[var].head() )
y_val = pd.read_csv(dir_prep + 'y_val.csv')
 print(y_val.shape)
 y_val.head()<load_from_csv>
%%time
load_dir_csv(DIR_NAME, ["train_transaction.csv", "test_transaction.csv", "train_identity.csv", "test_identity.csv"] )
items = pd.read_csv(dir_raw + 'items.csv' ).set_index("item_nbr")
 stores_items = pd.read_csv(dir_prep + 'stores_items.csv', index_col=['store_nbr','item_nbr'])
 items = items.reindex(stores_items.index.get_level_values(1))
print(items.shape)
 items.head()<compute_test_metric>
%%time
train = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')
 test = pd.merge(test_transaction, test_identity, on='TransactionID', how='left' )
n_public = 5
weights=pd.concat([items["perishable"]])* 0.25 + 1
print("Unweighted validation mse: ", mean_squared_error(
 np.array(y_val), np.array(val_pred)))
 print("Full validation mse: ", mean_squared_error(
 np.array(y_val), np.array(val_pred), sample_weight=weights))
print("'Public' validation mse: ", mean_squared_error(
 np.array(y_val)[:,:n_public], np.array(val_pred)[:,:n_public], sample_weight=weights))
print("'Private' validation mse: ", mean_squared_error(
 np.array(y_val)[:,n_public:], np.array(val_pred)[:,n_public:], sample_weight=weights))<load_from_csv>
del train_transaction, train_identity, test_transaction, test_identity
sub1 = pd.read_csv(dir_1st_result+'nn_sub_first_half.csv', index_col='id')
 print(sub1.shape)
 sub1.head()<load_from_csv>
gc.collect()
sub2 = pd.read_csv(dir_2nd_result+'nn_sub_second_half.csv', index_col='id')
 print(sub2.shape)
 sub2.head()<merge>
def is_integer_num(n):
if isinstance(n, int):
return True
if isinstance(n, float):
return n.is_integer()
return False
def missing_values_table_specified_value(df, value=0.5):
mis_val = df.isnull().sum()
mis_val_percent = 100 * df.isnull().sum() /len(df)
 mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)
 mis_val_table_ren_columns = mis_val_table.rename(
 columns = {0 : 'Missing Values', 1 : '% of Total Values'})
 if is_integer_num(value):
mis_val_table_ren_columns = mis_val_table_ren_columns[mis_val_table_ren_columns['Missing Values'] >= value]
print('The number of columns with {} counts missing values is {}.'.format(value, len(mis_val_table_ren_columns)))
 else:
value = value * 100
mis_val_table_ren_columns = mis_val_table_ren_columns[mis_val_table_ren_columns['% of Total Values'] >= value]
print('The number of columns with {}% missing values is {}.'.format(value, len(mis_val_table_ren_columns)))
 return mis_val_table_ren_columns
def missing_values_table(data):
total = data.isnull().sum()
percent =(data.isnull().sum() /data.isnull().count() *100)
 tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 tt['Types'] = types
return(np.transpose(tt))
submax = sub1.join(sub2, rsuffix='_2' ).max(axis=1)
 submax.head()<prepare_output>
display(missing_values_table(train), missing_values_table(test))
submax.name = 'unit_sales'
sub = pd.DataFrame(submax ).reset_index()
print(sub.shape)
 sub.head()<load_from_csv>
train['isFraud'].value_counts()
test = pd.read_csv('.. /input/favorita-grocery-sales-forecasting/test.csv',
parse_dates=['date'], index_col=0)
 test.head()<merge>
quantitative = [f for f in train.columns if train.dtypes[f] != 'object']
print(quantitative)
 print('Counts: {}'.format(len(quantitative)) )
check_df = test[['date']].join(sub.set_index("id"))
check_df.head()<define_variables>
qualitative = [f for f in train.columns if train.dtypes[f] == 'object']
print(qualitative)
 print('Counts: {}'.format(len(qualitative)) )
public_dates = ['2017-08-16', '2017-08-17', '2017-08-18', '2017-08-19', '2017-08-20']
private_public_dows = ['2017-08-23', '2017-08-24', '2017-08-25', '2017-08-26', '2017-08-27']
public_dates_nohol = ['2017-08-16', '2017-08-18', '2017-08-19', '2017-08-20']
private_public_dows_nohol = ['2017-08-23', '2017-08-25', '2017-08-26', '2017-08-27']
early_private_dates = ['2017-08-21', '2017-08-22']
private_private_dows = ['2017-08-28', '2017-08-29']
last_private_dates = ['2017-08-30', '2017-08-31']
public_last_dows = ['2017-08-16', '2017-08-17']<compute_test_metric>
def visualization_object_data(df, fig_x=10, fig_y=5):
object_list = unique_data_under_n_columns_list(df, 'object', 65)
 for colomn in object_list:
print('column name:{} / unique value(axis x):{} / max value:{} / min value:{} / missing value:{} '.format(colomn, len(df[colomn].unique()), max(df[colomn].value_counts()), min(df[colomn].value_counts()), df[colomn].isnull().sum()))
fig_x = len(df[colomn].unique())
 order = df[colomn].value_counts(ascending=False ).index
if fig_x <= 8:
fig, ax = plt.subplots(1, 3, figsize=(fig_x*6,fig_y))
sns.countplot(x=colomn, ax=ax[0], data=df, order=order)
 ax[0].set_title('All', fontsize=14)
 sns.countplot(x=colomn, ax=ax[1], data=df.loc[df['isFraud'] == 1], order=order)
 ax[1].set_title('isFraud = 1', fontsize=14)
 sns.countplot(x=colomn, ax=ax[2], data=df.loc[df['isFraud'] == 0], order=order)
 ax[2].set_title('isFraud = 0', fontsize=14)
 else:
fig, ax = plt.subplots(1, 3, figsize=(32,10))
sns.countplot(y=colomn, ax=ax[0], data=df, order=order)
 ax[0].set_title('All', fontsize=14)
 sns.countplot(y=colomn, ax=ax[1], data=df.loc[df['isFraud'] == 1], order=order)
 ax[1].set_title('isFraud = 1', fontsize=14)
 sns.countplot(y=colomn, ax=ax[2], data=df.loc[df['isFraud'] == 0], order=order)
 ax[2].set_title('isFraud = 0', fontsize=14)
 plt.show()
plt.pause(0.05 )
def corr_results(results, earlier_dates, later_dates):
d1 = results[results.date.isin(earlier_dates)].reset_index() [['unit_sales']]
d2 = results[results.date.isin(later_dates)].reset_index() [['unit_sales']]
d = pd.concat([d1, d2], axis=1)
 print(d.corr() )<compute_test_metric>
%%time
X_train = train.drop('isFraud', axis=1)
 y_train = train['isFraud'].copy()
X_test = test.copy()
print('
Comparing results for public test data with subsequent week...')
 compare_results(check_df, public_dates, private_public_dows)
 print('
Comparing results for public test data with non-holidays in subsequent week...')
 compare_results(check_df, public_dates_nohol, private_public_dows_nohol)
 print('
Comparing results for early private test data with a week later...')
 compare_results(check_df, early_private_dates, private_private_dows)
 print('
Comparing results for end of private data with corresponding days in public data...')
 compare_results(check_df, public_last_dows, last_private_dates )<compute_test_metric>
del train, test
gc.collect()
print('
Correlating results for public test data with subsequent week...')
 corr_results(check_df, public_dates, private_public_dows)
 print('
Correlating results for public test data with non-holidays in subsequent week...')
 corr_results(check_df, public_dates_nohol, private_public_dows_nohol)
 print('
Correlating results for early private test data with a week later...')
 corr_results(check_df, early_private_dates, private_private_dows)
 print('
Correlating results for end of private data with corresponding days in public data...')
 corr_results(check_df, public_last_dows, last_private_dates )<save_to_csv>
%%time
for f in qualitative:
lbl = preprocessing.LabelEncoder()
lbl.fit(list(X_train[f].values)+ list(X_test[f].values))
X_train[f] = lbl.transform(list(X_train[f].values))
X_test[f] = lbl.transform(list(X_test[f].values))
sub.to_csv('nn_sub_combined.csv', float_format='%.5f', index=None )<import_modules>
%%time
X_test = reduce_mem_usage(X_test )
import pandas as pd
import subprocess
import torch
import torch.optim as optim
from torch import nn
import torchvision
from torchvision import transforms
import os
from glob import glob
import numpy as np
from tqdm.notebook import tqdm
import random
import cv2
import sys<define_variables>
gc.collect()
EPOCH = 100
BATCH_SIZE = 16
PATIENCE = 3
cum_patience = 0
best_valid_loss = 999
DATA_PATH = '.. /input/state-farm-distracted-driver-detection'
WIDTH = 224
HEIGHT = 224
IMGNET_MEAN = [0.485, 0.456, 0.406]
IMGNET_STD = [0.229, 0.224, 0.225]
classes = [f'c{i}' for i in range(10)]
MODE = 'adv_data_aug'
os.mkdir(MODE)
 MODEL_NAME = 'resnet50'
assert MODEL_NAME in ['resnet50', 'resnet101', 'vgg19', 'densenet161']<load_pretrained>
sub = pd.read_csv(DIR_NAME + '/sample_submission.csv' )
driver_list = pd.read_csv(f'{DATA_PATH}/driver_imgs_list.csv')
 drivers = np.unique(driver_list['subject'].values)
 valid_driver = drivers[0]
prepare_dirs(MODE)
 trn_cnt, val_cnt = softlink_images(valid_driver, driver_list)
 trn_loader, val_loader = load_loaders(MODE)
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 model_conv = get_model(MODEL_NAME)
 criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model_conv.parameters() , lr=0.0001, weight_decay=1e-6, momentum=0.9)
 softmax = nn.Softmax(dim=1)
 for epoch in range(EPOCH):
print(f'
train(model_conv, trn_loader, optimizer, criterion, trn_cnt)
 valid_loss = evaluate(model_conv, val_loader, criterion)
 model_save_path = f'{MODE}.{MODEL_NAME}.{valid_driver}'
cum_patience, best_valid_loss = save_best_model(model_save_path, cum_patience, valid_loss, best_valid_loss)
 if cum_patience == PATIENCE:
break
test_save_path = 'submission.csv'
predict_test(test_save_path, model_save_path, device )<set_options>
NFOLDS = 5
kf = KFold(n_splits = NFOLDS, shuffle = True)
 y_preds = np.zeros(sub.shape[0])
 y_oof = np.zeros(X_train.shape[0] )
import tensorflow as tf 
<set_options>
%%time
for tr_idx, val_idx in kf.split(X_train, y_train):
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=9,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
tree_method='gpu_hist' 
)
 X_tr, X_vl = X_train.iloc[tr_idx, :], X_train.iloc[val_idx, :]
y_tr, y_vl = y_train.iloc[tr_idx], y_train.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
y_oof[val_idx] = y_pred_train
print('ROC AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 y_preds += clf.predict_proba(X_test)[:,1] / NFOLDS
<define_variables><EOS> 
sub['isFraud'] = y_preds
sub.to_csv('submission.csv', index=False )