Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<predict_on_test>
%%time
!cp.. /input/rapids/rapids.0.11.0 /opt/conda/envs/rapids.tar.gz
!cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz
sys.path = ["/opt/conda/envs/rapids/lib"] + ["/opt/conda/envs/rapids/lib/python3.6"] + ["/opt/conda/envs/rapids/lib/python3.6/site-packages"] + sys.path
!cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/
def predict_and_display(net, testset, n=10):

 net.eval()
for i in choice(range(len(testset)) , size=n):
image, label = test_set[i]
output = net(image.unsqueeze(0 ).cuda())
 prob, pred = prediction_from_output(output.unsqueeze(0))
prob, pred = prob.item() , pred.item()
evaluation = 'correct' if pred == label else 'mistake'
plt.figure(figsize=(2, 2))
print(f'
truth: {label} | pred: {pred} | prob: {prob:.2f}')
 print(f'{evaluation}:({class_name(label)} vs.{class_name(pred)})')
 show_image(image, means=channel_means, stdevs=channel_standard_devs)
 def plot_error_curves(training_error, validation_error, error_name='error', ylim=None, save_fig=True):

 assert len(training_error)== len(validation_error)> 1
fig, ax = plt.subplots()
ax.plot(range(len(training_error)) , training_error)
 ax.plot(range(len(validation_error)) , validation_error)
 if ylim:
ax.set_ylim(*ylim)
 ax.set_xlabel('epoch')
 ax.set_ylabel('CE')
 ax.legend(( 'training', 'validation'))
ax.set_title(f'{error_name} over time')
 if save_fig:
fig.savefig(f'{error_name.replace(" ", "_")}_learning_curve', bbox_inches='tight', transparent=True)
 plt.show() ;<save_to_csv>
warnings.filterwarnings("ignore")
 LOCAL_VALIDATION = False
cols = ['TransactionID', 'TransactionDT', 'TransactionAmt',
'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6',
'addr1', 'addr2', 'dist1', 'dist2', 'P_emaildomain', 'R_emaildomain',
'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11',
'C12', 'C13', 'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D7', 'D8',
'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M1', 'M2', 'M3', 'M4',
'M5', 'M6', 'M7', 'M8', 'M9']
v = [1, 3, 4, 6, 8, 11]
v += [13, 14, 17, 20, 23, 26, 27, 30]
v += [36, 37, 40, 41, 44, 47, 48]
v += [54, 56, 59, 62, 65, 67, 68, 70]
v += [76, 78, 80, 82, 86, 88, 89, 91]
v += [107, 108, 111, 115, 117, 120, 121, 123]
v += [124, 127, 129, 130, 136]
v += [138, 139, 142, 147, 156, 162]
v += [165, 160, 166]
v += [178, 176, 173, 182]
v += [187, 203, 205, 207, 215]
v += [169, 171, 175, 180, 185, 188, 198, 210, 209]
v += [218, 223, 224, 226, 228, 229, 235]
v += [240, 258, 257, 253, 252, 260, 261]
v += [264, 266, 267, 274, 277]
v += [220, 221, 234, 238, 250, 271]
v += [294, 284, 285, 286, 291, 297]
v += [303, 305, 307, 309, 310, 320]
v += [281, 283, 289, 296, 301, 314]
cols += ['V'+str(x)for x in v]
dtypes = {'isFraud':'int8'}
for c in cols+['id_0'+str(x)for x in range(1,10)]+['id_'+str(x)for x in range(10,34)]: dtypes[c] = 'float32'
for c in ['id-0'+str(x)for x in range(1,10)]+['id-'+str(x)for x in range(10,34)]: dtypes[c] = 'float32'
str_type = ['ProductCD', 'card4', 'card6', 'P_emaildomain', 'R_emaildomain','M1', 'M2', 'M3', 'M4','M5',
'M6', 'M7', 'M8', 'M9', 'id_12', 'id_15', 'id_16', 'id_23', 'id_27', 'id_28', 'id_29', 'id_30',
'id_31', 'id_33', 'id_34', 'id_35', 'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo']
str_type += ['id-12', 'id-15', 'id-16', 'id-23', 'id-27', 'id-28', 'id-29', 'id-30',
'id-31', 'id-33', 'id-34', 'id-35', 'id-36', 'id-37', 'id-38']
for c in str_type: dtypes[c] = 'category'
start = time.time()
print('RAPIDS =',cudf.__version__ )
def write_submission_file() :
test = HistoPatches(image_dir=os.path.join(DATA_DIR, 'test'), transform=image_trans)
 test_loader = DataLoader(test, batch_size=16)
 prediction_df = pd.DataFrame(columns=['id', 'label'])
 for i,(X, ids)in enumerate(test_loader):
out = net(X.cuda() ).squeeze()
probabilities = torch.sigmoid(out ).detach().cpu().numpy()
df = pd.DataFrame({'id': ids, 'label': probabilities})
 prediction_df = prediction_df.append(df)
 display(prediction_df.head())
 prediction_df.to_csv('submission.csv', index=False)
 os.listdir('.')
 write_submission_file()<import_modules>
%%time
X_train = cudf.read_csv('.. /input/ieee-fraud-detection/train_transaction.csv',index_col='TransactionID', usecols=cols+['isFraud'],dtype=dtypes)
 train_id = cudf.read_csv('.. /input/ieee-fraud-detection/train_identity.csv',index_col='TransactionID',dtype=dtypes)
 X_train = X_train.merge(train_id, how='left', left_index=True, right_index=True)
 X_test = cudf.read_csv('.. /input/ieee-fraud-detection/test_transaction.csv',index_col='TransactionID', usecols=cols,dtype=dtypes)
 test_id = cudf.read_csv('.. /input/ieee-fraud-detection/test_identity.csv',index_col='TransactionID', dtype=dtypes)
 fix = {o:n for o, n in zip(test_id.columns, train_id.columns)}
test_id.rename(columns=fix, inplace=True)
 X_test = X_test.merge(test_id, how='left', left_index=True, right_index=True)
 del train_id, test_id; x = gc.collect()
print('Train shape on GPU',X_train.shape,'Test shape on GPU',X_test.shape )
!pip install livelossplot 
<define_variables>
%%time
for i in range(1,16):
if i in [1,2,3,5,9]: continue
X_train['D'+str(i)] = X_train['D'+str(i)] - X_train.TransactionDT/np.float32(24*60*60)
 X_test['D'+str(i)] = X_test['D'+str(i)] - X_test.TransactionDT/np.float32(24*60*60)
 def encode_LE(df1,df2,col,verbose=True):
df_comb = cudf.concat([df1[col],df2[col]],axis=0)
 df_comb,_ = df_comb.factorize()
df1[col] = df_comb[:len(df1)].astype('int32')
 df2[col] = df_comb[len(df1):].astype('int32')
 if verbose: print(col,', ',end='')
 for i,f in enumerate(X_train.columns):
if(X_train[f].dtype=='object'):
encode_LE(X_train,X_test,f,False)
 elif f in str_type:
X_train[f].fillna(-1,inplace=True)
 X_test[f].fillna(-1,inplace=True)
 elif f not in ['TransactionAmt','TransactionDT','isFraud']:
mn = np.min(( X_train[f].min() ,X_test[f].min()))
X_train[f] -= np.float32(mn)
 X_test[f] -= np.float32(mn)
 X_train[f].fillna(-1,inplace=True)
 X_test[f].fillna(-1,inplace=True )
TRAINING_LOGS_FILE = "training_logs.csv"
MODEL_FILE = "histopathologic.h5"
ROC_PLOT_FILE = "roc.png"
KAGGLE_SUBMISSION_FILE = "kaggle_submission.csv"
INPUT_DIR = '.. /input/'<define_variables>
def add_feature(df1,df2,uid,col,agg,verbose=True):
if agg=='count': func = count2
elif agg=='mean': func = mean2
elif agg=='std' : func = std2
elif agg=='nunique': func = nunique2
else: return
df1['idx'] = np.arange(len(df1))
df2['idx'] = np.arange(len(df2)) +len(df1)
 temp_df = cudf.concat([df1[[uid,col,'idx']], df2[[uid,col,'idx']]])
 tmp = temp_df.groupby(uid,method='cudf' ).apply_grouped(
 func,
incols={col:'x'},
outcols=dict(y_out=np.float32),
tpb=32
).rename({'y_out':'new'})
 tmp = tmp.sort_values('idx')
 df1[uid+'_'+col+'_'+agg] = tmp.iloc[:len(df1)].new
df2[uid+'_'+col+'_'+agg] = tmp.iloc[len(df1):].new
if verbose: print(uid+'_'+col+'_'+agg,', ',end='')
 df1.drop_column('idx')
 df2.drop_column('idx')
 def add_features(df1,df2,uids,cols,aggs,verbose=True):
for col in cols:
for uid in uids:
for agg in aggs:
add_feature(df1,df2,uid,col,agg,verbose)
 def encode_CB(df1,df2,col1,col2,verbose=True):
nm = col1+'_'+col2
df1[nm] = df1[col1].astype(str)+'_'+df1[col2].astype(str)
 df2[nm] = df2[col1].astype(str)+'_'+df2[col2].astype(str)
 encode_LE(df1,df2,nm,verbose=False)
 if verbose: print(nm,', ',end='' )
SAMPLE_COUNT = 50000
TRAINING_RATIO = 0.9
IMAGE_SIZE = 96
EPOCHS = 12
BATCH_SIZE = 128
VERBOSITY = 1
TESTING_BATCH_SIZE = 3000<feature_engineering>
%%time
X_train['cents'] =(X_train['TransactionAmt'] - X_train['TransactionAmt'].floor() ).astype('float32')
 X_test['cents'] =(X_test['TransactionAmt'] - X_test['TransactionAmt'].floor() ).astype('float32')
 print('cents, ', end='')
 add_features(X_train,X_test,['addr1','card1','card2','card3','P_emaildomain'],['TransactionDT'],['count'])
 encode_CB(X_train,X_test,'card1','addr1')
 encode_CB(X_train,X_test,'card1_addr1','P_emaildomain')
 add_features(X_train,X_test,['card1_addr1','card1_addr1_P_emaildomain'],['TransactionDT'],['count'])
 add_features(X_train,X_test,['card1','card1_addr1','card1_addr1_P_emaildomain'],\
['TransactionAmt','D9','D11'],['mean','std'] )
training_dir = INPUT_DIR + 'train/'
data_frame = pd.DataFrame({'path': glob(os.path.join(training_dir,'*.tif')) })
 data_frame['id'] = data_frame.path.map(lambda x: x.split('/')[3].split('.')[0])
 labels = pd.read_csv(INPUT_DIR + 'train_labels.csv')
 data_frame = data_frame.merge(labels, on = 'id')
 negatives = data_frame[data_frame.label == 0].sample(SAMPLE_COUNT)
 positives = data_frame[data_frame.label == 1].sample(SAMPLE_COUNT)
 data_frame = pd.concat([negatives, positives] ).reset_index()
data_frame = data_frame[['path', 'id', 'label']]
data_frame['image'] = data_frame['path'].map(imread)
 training_path = '.. /training1'
validation_path = '.. /validation1'
for folder in [training_path, validation_path]:
for subfolder in ['0', '1']:
path = os.path.join(folder, subfolder)
 os.makedirs(path, exist_ok=True)
 training, validation = train_test_split(data_frame, train_size=TRAINING_RATIO, stratify=data_frame['label'])
 data_frame.set_index('id', inplace=True)
 for images_and_path in [(training, training_path),(validation, validation_path)]:
images = images_and_path[0]
path = images_and_path[1]
for image in images['id'].values:
file_name = image + '.tif'
label = str(data_frame.loc[image,'label'])
 destination = os.path.join(path, label, file_name)
 if not os.path.exists(destination):
source = os.path.join(INPUT_DIR + 'train', file_name)
 shutil.copyfile(source, destination )<load_pretrained>
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d')
 X_train['DT_M'] = X_train['TransactionDT'].to_pandas().map(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 X_train['DT_M'] =(X_train['DT_M'].dt.year-2017)*12 + X_train['DT_M'].dt.month
X_test['DT_M'] = X_test['TransactionDT'].to_pandas().map(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 X_test['DT_M'] =(X_test['DT_M'].dt.year-2017)*12 + X_test['DT_M'].dt.month
training_data_generator = ImageDataGenerator(rescale=1./255,horizontal_flip=True,vertical_flip=True,rotation_range=180,zoom_range=[1, 1.5],
fill_mode='reflect',
width_shift_range=0.3,
height_shift_range=0.3,
shear_range=0.3,
channel_shift_range=0.3 )<train_model>
X_train['day'] = X_train.TransactionDT /(24*60*60)
 X_train['uid'] = X_train.card1_addr1.astype(str)+'_'+(X_train.day-X_train.D1 ).floor().astype(str)
 X_test['day'] = X_test.TransactionDT /(24*60*60)
 X_test['uid'] = X_test.card1_addr1.astype(str)+'_'+(X_test.day-X_test.D1 ).floor().astype(str)
 encode_LE(X_train,X_test,'uid',verbose=False )
training_generator = training_data_generator.flow_from_directory(training_path,
target_size=(IMAGE_SIZE,IMAGE_SIZE),
batch_size=BATCH_SIZE,
class_mode='binary')
 validation_generator = ImageDataGenerator(rescale=1./255 ).flow_from_directory(validation_path,
target_size=(IMAGE_SIZE,IMAGE_SIZE),
batch_size=BATCH_SIZE,
class_mode='binary' )<choose_model_class>
%%time
add_features(X_train,X_test,['uid'],['TransactionDT'],['count'])
 add_features(X_train,X_test,['uid'],['TransactionAmt','D4','D9','D10','D15'],['mean','std'])
 add_features(X_train,X_test,['uid'],['C'+str(x)for x in range(1,15)if x!=3],['mean'])
 add_features(X_train,X_test,['uid'],['M'+str(x)for x in range(1,10)],['mean'])
 add_features(X_train,X_test,['uid'],['P_emaildomain','dist1','DT_M','id_02','cents'],['nunique'])
 add_features(X_train,X_test,['uid'],['C14'],['std'])
 add_features(X_train,X_test,['uid'],['C13','V314'],['nunique'])
 add_features(X_train,X_test,['uid'],['V127','V136','V309','V307','V320'],['nunique'] )
input_shape =(96, 96, 3)
 inputs = Input(input_shape)
 vgg = VGG19(weights='imagenet',include_top=False, input_shape=input_shape)
 outputs = GlobalAveragePooling2D()(vgg(inputs))
outputs = Dropout(0.5 )(outputs)
 outputs = Dense(1, activation='sigmoid' )(outputs)
 model = Model(inputs, outputs)
 model.compile(optimizer=Adam(lr=0.0001, decay=0.00001),
loss='binary_crossentropy',
metrics=['accuracy'])
 model.summary()<train_model>
cols = list(X_train.columns)
 cols.remove('TransactionDT'); cols.remove('isFraud')
 for c in ['D6','D7','D8','D9','D12','D13','D14'] + ['DT_M','day','uid']:
cols.remove(c)
 for c in ['C3','M5','id_08','id_33']:
cols.remove(c)
 for c in ['card4','id_07','id_14','id_21','id_30','id_32','id_34']:
cols.remove(c)
 for c in ['id_'+str(x)for x in range(22,28)]:
cols.remove(c )
history = model.fit_generator(training_generator,
steps_per_epoch=len(training_generator),
validation_data=validation_generator,
validation_steps=len(validation_generator),
epochs=EPOCHS,
verbose=VERBOSITY,shuffle=True,
callbacks=[PlotLossesKeras() ,
ModelCheckpoint(MODEL_FILE,
monitor='val_acc',
verbose=VERBOSITY,
save_best_only=True,
mode='max'),
CSVLogger(TRAINING_LOGS_FILE,
append=False,
separator=';')])
 model.load_weights(MODEL_FILE )<init_hyperparams>
print('NOW USING THE FOLLOWING',len(cols),'FEATURES.')
 print("We read data from disk and created all these features in %.2f seconds!" %(time.time() -start))
np.array(cols )
tta_steps = 5
test_datagen = ImageDataGenerator(
 shear_range=0.1,
zoom_range=[1,1.5],
horizontal_flip=True,
rotation_range=10.,
fill_mode='reflect',
width_shift_range = 0.1,
height_shift_range = 0.1)
 testing_files = glob(os.path.join(INPUT_DIR+'test/','*.tif'))
submission = pd.DataFrame()
for index in range(0, len(testing_files), TESTING_BATCH_SIZE):
predicted_labels=[]
data_frame = pd.DataFrame({'path': testing_files[index:index+TESTING_BATCH_SIZE]})
 data_frame['id'] = data_frame.path.map(lambda x: x.split('/')[3].split(".")[0])
 data_frame['image'] = data_frame['path'].map(imread)
 if(index==57000):
TESTING_BATCH_SIZE=548;
images = np.stack(data_frame.image, axis=0)
 for i in tqdm(range(tta_steps)) :
preds = model.predict_generator(test_datagen.flow(images/255, batch_size=TESTING_BATCH_SIZE, shuffle=False), steps = len(images)/TESTING_BATCH_SIZE)
 predicted_labels.append(preds)
 predictions = np.max(predicted_labels,axis=0)
 data_frame['label'] = predictions
submission = pd.concat([submission, data_frame[["id", "label"]]])
 submission.to_csv(KAGGLE_SUBMISSION_FILE, index=False, header=True )<set_options>
X_train = X_train.sort_index()
split = 3*len(X_train)//4
!pip install -Uqq fastbook
fastbook.setup_book() 
<import_modules>
if LOCAL_VALIDATION:
print("XGBoost version:", xgb.__version__)
 xgb_parms = {
'n_estimators':2000,
'max_depth':12,
'learning_rate':0.02,
'subsample':0.8,
'colsample_bytree':0.4,
'missing':-1,
'eval_metric':'auc',
'objective':'binary:logistic',
'tree_method':'gpu_hist'
}
train = xgb.DMatrix(data=X_train.iloc[:split][cols],label=X_train.iloc[:split]['isFraud'])
 valid = xgb.DMatrix(data=X_train.iloc[split:][cols],label=X_train.iloc[split:]['isFraud'])
 clf = xgb.train(xgb_parms, dtrain=train,
num_boost_round=2000,evals=[(train,'train'),(valid,'valid')],
early_stopping_rounds=100,maximize=True,
verbose_eval=50 )
from pandas.api.types import is_string_dtype, is_numeric_dtype, is_categorical_dtype
from fastai.tabular.all import *
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import train_test_split
from IPython.display import Image, display_svg, SVG<import_modules>
plt.hist(oof,bins=100)
 plt.ylim(( 0,5000))
plt.title('XGB OOF')
 plt.show()
X_train['oof'] = oof
X_train = X_train.reset_index()
X_train[['TransactionID','oof']].to_pandas().to_csv('oof_xgb_96.csv')
 X_train = X_train.set_index('TransactionID',drop=True )
<import_modules><EOS> 
sample_submission = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv')
 sample_submission.isFraud = preds
sample_submission.to_csv('sub_xgb_96.csv',index=False)
 plt.hist(sample_submission.isFraud,bins=100)
 plt.ylim(( 0,5000))
plt.title('XGB96 Submission')
 plt.show()