Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<categorify>
%matplotlib inline
init_notebook_mode(connected=True)
 cufflinks.go_offline(connected=True)
 warnings.filterwarnings("ignore")
 gc.enable()
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
 print(os.listdir(".. /input"))
print("Ready!" )
train_labels = to_categorical(train_labels )<choose_model_class>
def resumetable(df):
print(f"Dataset Shape: {df.shape}")
 summary = pd.DataFrame(df.dtypes,columns=['dtypes'])
 summary = summary.reset_index()
summary['Name'] = summary['index']
summary = summary[['Name','dtypes']]
summary['Missing'] = df.isnull().sum().values
summary['Uniques'] = df.nunique().values
summary['First Value'] = df.loc[0].values
summary['Second Value'] = df.loc[1].values
summary['Third Value'] = df.loc[2].values
for name in summary['Name'].value_counts().index:
summary.loc[summary['Name'] == name, 'Entropy'] = round(stats.entropy(df[name].value_counts(normalize=True), base=2),2)
 return summary
def plot_distribution(df, var_select=None, title=None, bins=1.0):
tmp_fraud = df[df['isFraud'] == 1]
tmp_no_fraud = df[df['isFraud'] == 0]
corr = df['isFraud'].corr(df[var_select])
 corr = np.round(corr,3)
 tmp1 = tmp_fraud[var_select].dropna()
tmp2 = tmp_no_fraud[var_select].dropna()
hist_data = [tmp1, tmp2]
group_labels = ['Fraud', 'No Fraud']
colors = ['seagreen','indianred', ]
fig = ff.create_distplot(hist_data,
group_labels,
colors = colors,
show_hist = True,
curve_type='kde',
bin_size = bins 
)
 fig['layout'].update(title = title+' '+'(corr target ='+ str(corr)+')')
 iplot(fig, filename = 'Density plot')
 def plot_dist_churn(df, col, binary=None):
tmp_churn = df[df[binary] == 1]
tmp_no_churn = df[df[binary] == 0]
tmp_attr = round(tmp_churn[col].value_counts().sort_index() / df[col].value_counts().sort_index() ,2)*100
print(f'Distribution of {col}: ')
 trace1 = go.Bar(
 x=tmp_churn[col].value_counts().sort_index().index,
y=tmp_churn[col].value_counts().sort_index().values,
name='Fraud',opacity = 0.8, marker=dict(
 color='seagreen',
line=dict(color='
trace2 = go.Bar(
 x=tmp_no_churn[col].value_counts().sort_index().index,
y=tmp_no_churn[col].value_counts().sort_index().values,
name='No Fraud', opacity = 0.8,
marker=dict(
 color='indianred',
line=dict(color='
width=1)
)
)
 trace3 = go.Scatter(
 x=tmp_attr.sort_index().index,
y=tmp_attr.sort_index().values,
yaxis = 'y2',
name='% Fraud', opacity = 0.6,
marker=dict(
 color='black',
line=dict(color='
width=2)
)
)
 layout = dict(title = f'Distribution of {str(col)} feature by %Fraud',
xaxis=dict(type='category'),
yaxis=dict(title= 'Count'),
yaxis2=dict(range= [0, 15],
overlaying= 'y',
anchor= 'x',
side= 'right',
zeroline=False,
showgrid= False,
title= 'Percentual Fraud Transactions' 
))
fig = go.Figure(data=[trace1, trace2, trace3], layout=layout)
 iplot(fig )
network = models.Sequential()
network.add(layers.Dense(512, activation='relu', input_shape=(28 * 28,)))
 network.add(layers.Dense(10, activation='softmax'))
network.summary()<choose_model_class>
%%time
train_transaction = pd.read_csv('.. /input/train_transaction.csv', index_col='TransactionID')
 test_transaction = pd.read_csv('.. /input/test_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('.. /input/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/test_identity.csv', index_col='TransactionID')
 print("Data is loaded!" )
network.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'] )<train_model>
missing_values_count = train_transaction.isnull().sum()
print(missing_values_count[0:10])
 total_cells = np.product(train_transaction.shape)
 total_missing = missing_values_count.sum()
print("% of missing data = ",(total_missing/total_cells)* 100 )
network.fit(train_images, train_labels, epochs = 5, batch_size=128 )<save_to_csv>
missing_values_count = train_identity.isnull().sum()
print(missing_values_count[0:10])
 total_cells = np.product(train_identity.shape)
 total_missing = missing_values_count.sum()
print("% of missing data = ",(total_missing/total_cells)* 100 )
submit = pd.DataFrame(np.argmax(network.predict(test_images), axis=1),
columns=['Label'],
index=pd.read_csv('.. /input/digit-recognizer/sample_submission.csv')['ImageId'])
 submit.index.name = 'ImageId'
submit.to_csv('submittion.csv' )<import_modules>
del missing_values_count, total_cells, total_missing
import numpy as np
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
import itertools
import tensorflow as tf
from tensorflow import keras
from keras.preprocessing.image import ImageDataGenerator
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns<load_from_csv>
del x,y
gc.collect()
mnist_train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 mnist_test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )<train_model>
print(np.sum(train_transaction.index.isin(train_identity.index.unique())))
 print(np.sum(test_transaction.index.isin(test_identity.index.unique())) )
mnist_train<drop_column>
train_transaction['TransactionDT'].shape[0] , train_transaction['TransactionDT'].nunique()
x_train = mnist_train.drop(labels = "label", axis = 1)
 x_train<prepare_x_and_y>
train_transaction['TransactionDT'].value_counts().head(10 )
y_train = mnist_train["label"]
y_train<feature_engineering>
del c_features
gc.collect()
x_train = x_train/255.0
mnist_test = mnist_test/255.0<define_search_space>
missing_values_count = train_transaction[d_features].isnull().sum()
missing_values_count
x_train = x_train.values.reshape(-1,28,28,1)
 mnist_test = mnist_test.values.reshape(-1,28,28,1)
 x_train.shape<concatenate>
total_cells = np.product(train_transaction[d_features].shape)
 total_missing = missing_values_count.sum() 
(total_missing/total_cells)* 100
y_train = tf.keras.utils.to_categorical(y_train, num_classes=10 )<train_model>
del d_features, cor
gc.collect()
datagen = ImageDataGenerator(rotation_range=12, zoom_range = 0.15, width_shift_range=0.15, height_shift_range=0.13)
 datagen.fit(x_train)
 datagen.fit(x_test )<split>
del m_features
gc.collect()
x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size = 0.15)
 print(x_train.shape)
 print(x_val.shape )<choose_model_class>
del time_val
input_layer = tf.keras.layers.Input(shape =(28, 28, 1))
hidden_layer_1 = tf.keras.layers.Conv2D(32, kernel_size=(3, 3), activation=tf.keras.activations.relu )(input_layer)
 norm_1 = tf.keras.layers.BatchNormalization()(hidden_layer_1)
 max_1 = tf.keras.layers.MaxPooling2D(pool_size=(3,3), strides=(2,2))(norm_1)
 dropout_layer_1 = tf.keras.layers.Dropout(0.25 )(max_1)
 flatten_layer = tf.keras.layers.Flatten()(dropout_layer_1)
 hidden_layer_3 = tf.keras.layers.Dense(256, activation=tf.keras.activations.relu )(flatten_layer)
 hidden_layer_4 = tf.keras.layers.Dense(128, activation=tf.keras.activations.relu )(hidden_layer_3)
 dropout_layer_3 = tf.keras.layers.Dropout(0.5 )(hidden_layer_4)
 output_layer = tf.keras.layers.Dense(10, activation=tf.keras.activations.sigmoid )(dropout_layer_3 )<choose_model_class>
print("Unique Devices = ",train_identity['DeviceInfo'].nunique())
 train_identity['DeviceInfo'].value_counts().head()
model = tf.keras.Model(inputs = input_layer, outputs = output_layer)
 model.summary()<choose_model_class>
cards = ['card1', 'card2', 'card3', 'card4', 'card5', 'card6']
for i in cards:
print("Unique ",i, " = ",train_transaction[i].nunique() )
model.compile(optimizer = tf.keras.optimizers.Adam() , loss = tf.keras.losses.CategoricalCrossentropy() , metrics = ['accuracy'] )<train_model>
%%time
train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 y_train = train['isFraud'].astype("uint8" ).copy()
print("Tain: ",train.shape)
 del train_transaction, train_identity
test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print("Test: ",test.shape)
 del test_transaction, test_identity
print("Merged!" )
his = model.fit(x_train, y_train, batch_size = 1000, epochs = 40, validation_data=(x_val, y_val))<predict_on_test>
%%time
X_train = train.drop('isFraud', axis=1)
 X_test = test.copy()
del train, test
gc.collect()
for f in X_train.columns:
if X_train[f].dtype=='object' or X_test[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(X_train[f].values)+ list(X_test[f].values))
X_train[f] = lbl.transform(list(X_train[f].values))
X_test[f] = lbl.transform(list(X_test[f].values))
results = model.predict(mnist_test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )<save_to_csv>
def reduce_mem_usage1(props):
start_mem_usg = props.memory_usage().sum() / 1024**2
print("Memory usage of properties dataframe is :",start_mem_usg," MB")
 NAlist = []
for col in props.columns:
if props[col].dtype != object:
print("******************************")
 print("Column: ",col)
 print("dtype before: ",props[col].dtype)
 IsInt = False
mx = props[col].max()
mn = props[col].min()
if not np.isfinite(props[col] ).all() :
NAlist.append(col)
 props[col].fillna(mn-1,inplace=True)
 asint = props[col].fillna(0 ).astype(np.int64)
 result =(props[col] - asint)
 result = result.sum()
if result > -0.01 and result < 0.01:
IsInt = True
if IsInt:
if mn >= 0:
if mx < 255:
props[col] = props[col].astype(np.uint8)
 elif mx < 65535:
props[col] = props[col].astype(np.uint16)
 elif mx < 4294967295:
props[col] = props[col].astype(np.uint32)
 else:
props[col] = props[col].astype(np.uint64)
 else:
if mn > np.iinfo(np.int8 ).min and mx < np.iinfo(np.int8 ).max:
props[col] = props[col].astype(np.int8)
 elif mn > np.iinfo(np.int16 ).min and mx < np.iinfo(np.int16 ).max:
props[col] = props[col].astype(np.int16)
 elif mn > np.iinfo(np.int32 ).min and mx < np.iinfo(np.int32 ).max:
props[col] = props[col].astype(np.int32)
 elif mn > np.iinfo(np.int64 ).min and mx < np.iinfo(np.int64 ).max:
props[col] = props[col].astype(np.int64)
 else:
props[col] = props[col].astype(np.float32)
 print("dtype after: ",props[col].dtype)
 print("******************************")
 print("___MEMORY USAGE AFTER COMPLETION:___")
 mem_usg = props.memory_usage().sum() / 1024**2
print("Memory usage is: ",mem_usg," MB")
 print("This is ",100*mem_usg/start_mem_usg,"% of the initial size")
 return props, NAlist
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("mnist_result.csv",index=False)
 submission<import_modules>
%%time
X_train = reduce_mem_usage2(X_train)
 X_test = reduce_mem_usage2(X_test )
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow import keras
from keras import Sequential
from keras.layers import Conv2D, Dense, Flatten, MaxPooling2D, Dropout
from keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split<load_from_csv>
drop_col = ['TransactionDT']
X_train.drop(drop_col,axis=1, inplace=True)
 X_test.drop(drop_col, axis=1, inplace=True)
 X_train.head()
path = '/kaggle/input/digit-recognizer/'
data = pd.read_csv(path +'train.csv')
 print(data.shape )<categorify>
lin_pca = KernelPCA(n_components = 2, kernel="linear", fit_inverse_transform=True)
 rbf_pca = KernelPCA(n_components = 2, kernel="rbf", gamma=0.0433, fit_inverse_transform=True)
 sig_pca = KernelPCA(n_components = 2, kernel="sigmoid", gamma=0.001, coef0=1, fit_inverse_transform=True)
 plt.figure(figsize=(11, 4))
for subplot, pca, title in(( 131, lin_pca, "Linear kernel"),(132, rbf_pca, "RBF kernel, $\gamma=0.04$"), 
(133, sig_pca, "Sigmoid kernel, $\gamma=10^{-3}, r=1$")) :
PCA_train_x = PCA(2 ).fit_transform(train_scaled)
 plt.subplot(subplot)
 plt.title(title, fontsize=14)
 plt.scatter(PCA_train_x[:, 0], PCA_train_x[:, 1], c=y_train, cmap="nipy_spectral_r")
 plt.xlabel("$z_1$", fontsize=18)
 if subplot == 131:
plt.ylabel("$z_2$", fontsize=18, rotation=0)
 plt.grid(True)
 plt.show()
label = tf.keras.utils.to_categorical(data['label'].values)
 X = data.drop('label',axis=1 ).values
num_images = X.shape[0]
m = X.shape[1]
max_pixel = 255
img_dim = np.sqrt(m ).astype(int)
 out_dim = label.shape[1]
print('Number of different labels/output dimension ' + str(out_dim))
X = X.reshape(( num_images,img_dim,img_dim,1)) / max_pixel
print('Shape of input data X ' + str(X.shape))<choose_model_class>
del train_scaled,PCA_train_x,scaler, lin_pca,rbf_pca, sig_pca
gc.collect()
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu )<choose_model_class>
xgb.XGBClassifier(
 n_estimators=500,
max_depth=9,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
gamma = 0.1,
alpha = 4,
missing = -1,
tree_method='gpu_hist'
)
def create_model() :
my_model = Sequential()
my_model.add(Conv2D(32,
kernel_size =(3,3),
activation = 'relu',
input_shape =(img_dim,img_dim,1)))
 my_model.add(Dropout(0.5))
my_model.add(MaxPooling2D(pool_size=(2, 2), strides=None, padding="valid"))
my_model.add(Conv2D(32, kernel_size =(3,3), activation = 'relu'))
my_model.add(Dropout(0.5))
my_model.add(MaxPooling2D(pool_size=(2, 2), strides=None, padding="valid"))
my_model.add(Flatten())
 my_model.add(Dense(units = 500))
my_model.add(Dense(out_dim, activation = 'softmax'))
return my_model<choose_model_class>
%%time
NFOLDS = 5
kf = StratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=123)
 y_preds = np.zeros(X_test.shape[0])
 y_oof = np.zeros(X_train.shape[0])
 score = 0
for fold,(tr_idx, val_idx)in enumerate(kf.split(X_train, y_train)) :
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=9,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
gamma = 0.2,
alpha = 4,
missing = -1,
tree_method='gpu_hist' 
)
 X_tr, X_vl = X_train.iloc[tr_idx, :], X_train.iloc[val_idx, :]
y_tr, y_vl = y_train.iloc[tr_idx], y_train.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
y_oof[val_idx] = y_pred_train
print("FOLD: ",fold,' AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 score += roc_auc_score(y_vl, y_pred_train)/ NFOLDS
y_preds+= clf.predict_proba(X_test)[:,1] / NFOLDS
del X_tr, X_vl, y_tr, y_vl
gc.collect()
print("
MEAN AUC = {}".format(score))
print("OOF AUC = {}".format(roc_auc_score(y_train, y_oof)) )
with tpu_strategy.scope() :
my_model = create_model()
my_model.compile(optimizer = 'adam',
loss = 'categorical_crossentropy',
metrics = ['accuracy'] )<split>
del clf, importance_df
gc.collect()
X_train, X_val, y_train, y_val = train_test_split(X, label, test_size = 0.1 )<define_variables>
sub = pd.read_csv('.. /input/sample_submission.csv', index_col='TransactionID')
 sub['isFraud'] = y_preds
sub.to_csv('xgboost.csv')
 sub.head()
BATCH_SIZE = 128 * tpu_strategy.num_replicas_in_sync
EPOCHS = 100
STEPS_PER_EPOCH = num_images // BATCH_SIZE<train_model>
print("Predicted {} frauds".format(int(sub[sub['isFraud']==1].sum())) )
<load_from_csv><EOS> 
del sub, X_train, X_test, importance_df
gc.collect()