Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<feature_engineering>
warnings.filterwarnings('ignore')
 pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None)
 warnings.simplefilter('ignore')
 sns.set()
%matplotlib inline
pd.set_option('display.max_columns', 500)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=1.1,max_lighting=0.05, max_warp=0.)<load_from_csv>
train_id= pd.read_csv("/kaggle/input/ieee-fraud-detection/train_identity.csv")
 test_id = pd.read_csv("/kaggle/input/ieee-fraud-detection/test_identity.csv")
 train_tr = pd.read_csv("/kaggle/input/ieee-fraud-detection/train_transaction.csv")
 test_tr = pd.read_csv("/kaggle/input/ieee-fraud-detection/test_transaction.csv" )
data = ImageDataBunch.from_csv(path,folder='train',valid_pct=0.3,csv_labels=train_lbl,ds_tfms=tfms, size=90, suffix='.tif',test=test_folder,bs=64 )<groupby>
train=pd.merge(train_tr, train_id, on = "TransactionID",how='left',left_index=True, right_index=True)
 train.head()
stats=data.batch_stats()
data.normalize(stats)
<define_variables>
test=pd.merge(test_tr, test_id, on = "TransactionID",how="left",left_index=True, right_index=True)
 test.head()
data.show_batch(rows=3, figsize=(8,5))<define_variables>
del train_id, train_tr, test_id, test_tr
model_dir = "/kaggle/working/tmp/models/"
os.makedirs('/kaggle/working/tmp/models/' )<import_modules>
test.columns=train.columns.drop("isFraud" )
dir(fastai.vision.models )<choose_model_class>
train.isna().sum()
learner_resnet50 = cnn_learner(data=data, base_arch=models.resnet50,model_dir=model_dir, metrics=[accuracy,error_rate], ps=0.5 )<find_best_params>
train=train.drop(drop_col,axis=1)
 test=test.drop(drop_col,axis=1)
 del drop_col
lr_find(learner_resnet50 )<set_options>
for col in train.columns:
if sum(train[col].isnull())/float(len(train.index)) > 0.90:
print("'"+col+"'",', ',end='')
 train=train.drop(col,axis=1)
 test=test.drop(col,axis=1 )
defaults.device = torch.device('cuda' )<train_model>
emails = {'gmail': 'google', 'att.net': 'att', 'twc.com': 'spectrum',
'scranton.edu': 'other', 'optonline.net': 'other', 'hotmail.co.uk': 'microsoft',
'comcast.net': 'other', 'yahoo.com.mx': 'yahoo', 'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo', 'charter.net': 'spectrum', 'live.com': 'microsoft',
'aim.com': 'aol', 'hotmail.de': 'microsoft', 'centurylink.net': 'centurylink',
'gmail.com': 'google', 'me.com': 'apple', 'earthlink.net': 'other', 'gmx.de': 'other',
'web.de': 'other', 'cfl.rr.com': 'other', 'hotmail.com': 'microsoft',
'protonmail.com': 'other', 'hotmail.fr': 'microsoft', 'windstream.net': 'other',
'outlook.es': 'microsoft', 'yahoo.co.jp': 'yahoo', 'yahoo.de': 'yahoo',
'servicios-ta.com': 'other', 'netzero.net': 'other', 'suddenlink.net': 'other',
'roadrunner.com': 'other', 'sc.rr.com': 'other', 'live.fr': 'microsoft',
'verizon.net': 'yahoo', 'msn.com': 'microsoft', 'q.com': 'centurylink',
'prodigy.net.mx': 'att', 'frontier.com': 'yahoo', 'anonymous.com': 'other',
'rocketmail.com': 'yahoo', 'sbcglobal.net': 'att', 'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo', 'outlook.com': 'microsoft', 'mail.com': 'other',
'bellsouth.net': 'other', 'embarqmail.com': 'centurylink', 'cableone.net': 'other',
'hotmail.es': 'microsoft', 'mac.com': 'apple', 'yahoo.co.uk': 'yahoo', 'netzero.com': 'other',
'yahoo.com': 'yahoo', 'live.com.mx': 'microsoft', 'ptd.net': 'other', 'cox.net': 'other',
'aol.com': 'aol', 'juno.com': 'other', 'icloud.com': 'apple'}
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
learner_resnet50.fit_one_cycle(1, 1e-02 )<train_model>
def encode_FE(df1, df2, cols):
for col in cols:
df = pd.concat([df1[col],df2[col]])
 vc = df.value_counts(dropna=True, normalize=True ).to_dict()
vc[-1] = -1
nm = col+'_FE'
df1[nm] = df1[col].map(vc)
 df1[nm] = df1[nm].astype('float32')
 df2[nm] = df2[col].map(vc)
 df2[nm] = df2[nm].astype('float32')
 print(nm,', ',end='')
 def encode_LE(col,train=train,test=test,verbose=True):
df_comb = pd.concat([train[col],test[col]],axis=0)
 df_comb,_ = df_comb.factorize(sort=True)
 nm = col
if df_comb.max() >32000:
train[nm] = df_comb[:len(train)].astype('int32')
 test[nm] = df_comb[len(train):].astype('int32')
 else:
train[nm] = df_comb[:len(train)].astype('int16')
 test[nm] = df_comb[len(train):].astype('int16')
 del df_comb; x=gc.collect()
if verbose: print(nm,', ',end='')
 def encode_AG(main_columns, uids, aggregations=['mean'], train_df=train, test_df=test,
fillna=True, usena=False):
for main_column in main_columns:
for col in uids:
for agg_type in aggregations:
new_col_name = main_column+'_'+col+'_'+agg_type
temp_df = pd.concat([train_df[[col, main_column]], test_df[[col,main_column]]])
 if usena: temp_df.loc[temp_df[main_column]==-1,main_column] = np.nan
temp_df = temp_df.groupby([col])[main_column].agg([agg_type] ).reset_index().rename(
 columns={agg_type: new_col_name})
 temp_df.index = list(temp_df[col])
 temp_df = temp_df[new_col_name].to_dict()
train_df[new_col_name] = train_df[col].map(temp_df ).astype('float32')
 test_df[new_col_name] = test_df[col].map(temp_df ).astype('float32')
 if fillna:
train_df[new_col_name].fillna(-1,inplace=True)
 test_df[new_col_name].fillna(-1,inplace=True)
 print("'"+new_col_name+"'",', ',end='')
 def encode_CB(col1,col2,df1=train,df2=test):
nm = col1+'_'+col2
df1[nm] = df1[col1].astype(str)+'_'+df1[col2].astype(str)
 df2[nm] = df2[col1].astype(str)+'_'+df2[col2].astype(str)
 encode_LE(nm,verbose=False)
 def encode_AG2(main_columns, uids, train_df=train, test_df=test):
for main_column in main_columns:
for col in uids:
comb = pd.concat([train_df[[col]+[main_column]],test_df[[col]+[main_column]]],axis=0)
 mp = comb.groupby(col)[main_column].agg(['nunique'])['nunique'].to_dict()
train_df[col+'_'+main_column+'_ct'] = train_df[col].map(mp ).astype('float32')
 test_df[col+'_'+main_column+'_ct'] = test_df[col].map(mp ).astype('float32')
 print(col+'_'+main_column+'_ct, ',end='' )
learner_resnet50.recorder.plot_lr(show_moms=True )<find_best_params>
train['day'] = train.TransactionDT /(24*60*60)
 train['uid'] = train.card1_addr1.astype(str)+'_'+np.floor(train.day-train.D1 ).astype(str)
 test['day'] = test.TransactionDT /(24*60*60)
 test['uid'] = test.card1_addr1.astype(str)+'_'+np.floor(test.day-test.D1 ).astype(str )
learner_resnet50.show_results(alpha=1 )<save_model>
train[(train["card1"]==15775)&(train["addr1"]==251.0)][["TransactionID","isFraud","TransactionDT","TransactionAmt","card1","card2","card3","card4","addr1","uid"]]
learner_resnet50.save('stage-1',return_path=True )<load_pretrained>
na_low=['TransactionAmt' , 'ProductCD' , 'card1' , 'card2' , 'card3' , 'card4' , 'card5' , 'card6' , 'addr1' ,
'addr2' , 'P_emaildomain' , 'C1' , 'C2' , 'C3' , 'C4' , 'C5' , 'C6' , 'C7' , 'C8' , 'C9' , 'C10' ,
'C11' , 'C12' , 'C13' , 'C14' , 'M6' ]
learner_resnet50.unfreeze()<find_best_params>
encode_FE(train,test,['addr1','card1','card2','card3','P_emaildomain'] )
lr_find(learner_resnet50)
 learner_resnet50.recorder.plot()<train_model>
train['day'] = train.TransactionDT /(24*60*60)
 train['uid'] = train.card1_addr1.astype(str)+'_'+np.floor(train.day-train.D1 ).astype(str)
 test['day'] = test.TransactionDT /(24*60*60)
 test['uid'] = test.card1_addr1.astype(str)+'_'+np.floor(test.day-test.D1 ).astype(str )
learner_resnet50.fit_one_cycle(1,slice(1e-06,1e-05),pct_start=0.8 )<save_model>
encode_FE(train,test,['uid'])
 encode_AG(numeric, ['uid'],['mean',"std"], train, test, fillna=True, usena=False)
learner_resnet50.save('stage-2',return_path=True )<choose_model_class>
categorical_columns=test.columns.drop(test._get_numeric_data().columns)
 categorical_columns=categorical_columns.drop('uid')
 categorical_columns
interp = ClassificationInterpretation.from_learner(learner_resnet50 )<compute_test_metric>
encode_AG2(categorical_columns, ['uid'], train, test )
losses,idxs = interp.top_losses()
len(data.valid_ds)==len(losses)==len(idxs )<compute_test_metric>
encode_FE(train,test,['addr1','card1','card2','card3','P_emaildomain'])
 encode_CB('card1_addr1','P_emaildomain')
 encode_FE(train,test,['card1_addr1','card1_addr1_P_emaildomain'])
 encode_AG(['TransactionAmt','D9','D11'],['card1','card1_addr1','card1_addr1_P_emaildomain'],['mean','std'],usena=True )
pred_val ,y_val = learner_resnet50.get_preds()
def auc_score(y_pred,y_true,tens=True):
score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score=tensor(score)
 else:
score=score
return score
pred_score=auc_score(pred_val ,y_val)
 pred_score<compute_test_metric>
del train['uid'], test['uid']
pred_score_acc=accuracy(pred_val ,y_val)
 pred_score_acc<compute_train_metric>
train=reduce_mem_usage2(train)
 test=reduce_mem_usage2(test )
fpr, tpr, thresholds = roc_curve(y_val.numpy() , pred_val.numpy() [:,1], pos_label=1)
 pred_score_auc = auc(fpr, tpr)
 print(f'ROC area: {pred_score_auc}' )<load_pretrained>
categorical_columns=test.columns.drop(test._get_numeric_data().columns)
 categorical_columns
learner_resnet50.export('/kaggle/working/tmp/models/export.pkl' )<load_pretrained>
for i in categorical_columns:
lbe=preprocessing.LabelEncoder()
train[i]=lbe.fit_transform(train[i].astype(str))
loaded_learner = load_learner(Path(model_dir))
loaded_learner.data.classes<predict_on_test>
for i in categorical_columns:
test[i]=lbe.fit_transform(test[i].astype(str))
pred_class,pred_idx,pred_probs = loaded_learner.predict(img)
 print(pred_class, pred_idx,pred_probs )<predict_on_test>
for i in categorical_columns:
if(test[i].max() == train[i].max())&(train[i].max() <8):
test = pd.get_dummies(test, columns = [i])
 train=pd.get_dummies(train, columns = [i])
pred_class,pred_idx,pred_probs = loaded_learner.predict(img)
 print(pred_class, pred_idx,pred_probs )<predict_on_test>
train_TransactionID= train["TransactionID"]
test_TransactionID=test["TransactionID"]
TransactionDT=train["TransactionDT"]
X= train.drop([ 'TransactionDT', 'TransactionID'], axis=1)
 y = train['isFraud']
test = test.sort_values('TransactionDT' ).drop(['TransactionDT', 'TransactionID'], axis=1)
 del train
img = open_image(Path('.. /input/test-image/test_img.tif'))
pred_class,pred_idx,pred_probs = loaded_learner.predict(img)
 img.show()
targets = ['Non-Cancerous','Cancerous']
print("Tissue cell is identified as" , targets[pred_idx] , "with probability of", float(pred_probs[pred_idx]*100))<load_pretrained>
X=X.drop("isFraud", axis=1 )
loaded_learner_val = load_learner(Path(model_dir),test=ImageList.from_folder(Path(test_folder)) )<predict_on_test>
params = {'num_leaves': 491,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.3797454081646243,
'bagging_fraction': 0.4181193142567742,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.006883242363721497,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47
}
pred_test ,y_test = loaded_learner_val.get_preds(ds_type=DatasetType.Test )<load_from_csv>
folds = TimeSeriesSplit(n_splits=5)
 aucs = list()
feature_importances = pd.DataFrame()
feature_importances['feature'] = X.columns
training_start_time = time()
for fold,(trn_idx, test_idx)in enumerate(folds.split(X, y)) :
start_time = time()
print('Training on fold {}'.format(fold + 1))
trn_data = lgb.Dataset(X.iloc[trn_idx], label=y.iloc[trn_idx])
 val_data = lgb.Dataset(X.iloc[test_idx], label=y.iloc[test_idx])
 clf = lgb.train(params, trn_data, 10000, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds=500)
 feature_importances['fold_{}'.format(fold + 1)] = clf.feature_importance()
aucs.append(clf.best_score['valid_1']['auc'])
 print('Fold {} finished in {}'.format(fold + 1, str(datetime.timedelta(seconds=time() - start_time))))
print('-' * 30)
 print('Training has finished.')
 print('Total training time is {}'.format(str(datetime.timedelta(seconds=time() - training_start_time))))
print('Mean AUC:', np.mean(aucs))
print('-' * 30 )
sub=pd.read_csv('.. /input/histopathologic-cancer-detection/sample_submission.csv' ).set_index('id' )<data_type_conversions>
feature_importances['average'] = feature_importances[['fold_{}'.format(fold + 1)for fold in range(folds.n_splits)]].mean(axis=1)
 feature_importances.to_csv('feature_importances_TimeFold.csv')
 plt.figure(figsize=(16, 16))
sns.barplot(data=feature_importances.sort_values(by='average', ascending=False ).head(50), x='average', y='feature');
plt.title('50 TOP feature importance over {} folds average'.format(folds.n_splits)) ;
clean_names = np.vectorize(lambda imgname: str(imgname ).split('/')[-1][:-4])
 cleaned_names = clean_names(data.test_ds.items ).astype(str )<save_to_csv>
best_iter = clf.best_iteration
sub.loc[cleaned_names,'label']=pred_test.numpy() [:,1]
sub.to_csv(f'/kaggle/working/submission_{int(pred_score_auc*100)}auc.csv' )<load_from_csv>
clf = lgb.LGBMClassifier(**params, num_boost_round=best_iter)
 clf.fit(X, y )
<set_options><EOS> 
predictions = clf.predict_proba(test)[:, 1]
output = pd.DataFrame({ "TransactionID" : test_TransactionID, "isFraud": predictions })
 output.to_csv('submission_lgbm.csv', index=False)