Kaggle competition: IEEE-CIS Fraud Detection
train = train.sparse.to_coo().tocsr()
test = test.sparse.to_coo().tocsr()
train = train.astype("float32")
 test = test.astype("float32" )<train_on_grid>
pd.options.display.max_rows = 500
pd.options.display.max_columns = 100
lr = LogisticRegression(C=0.1338,
solver="lbfgs",
tol=0.0003,
max_iter=5000)
 lr.fit(train, labels)
 lr_pred = lr.predict_proba(train)[:, 1]
score = roc_auc_score(labels, lr_pred)
 print("score: ", score )<predict_on_test>
train = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_transaction.csv')
 train_ind = pd.read_csv('/kaggle/input/ieee-fraud-detection/train_identity.csv')
 test = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_transaction.csv')
 test_ind = pd.read_csv('/kaggle/input/ieee-fraud-detection/test_identity.csv' )
submission["id"] = test_id
submission["target"] = lr.predict_proba(test)[:, 1]<save_to_csv>
train_len = len(train)
submission.to_csv("submission.csv", index=False )<save_to_csv>
train = train.merge(train_ind, how = 'left', on ='TransactionID')
 test = test.merge(test_ind, how = 'left', on ='TransactionID')
 del train_ind,test_ind
submission.to_csv("submission.csv", index=False )<load_from_csv>
all_data = pd.concat([train, test])
 del train,test
gc.collect()
file_train = ".. /input/train_v2.csv"
file_test = ".. /input/test_v2.csv"
chunk_size = 10000
def load_data(file,chunk_size,nrows_load=None,test_data=False):
df_res = pd.DataFrame()
df_reader = pd.read_csv(file,
dtype={ 'date': str, 'fullVisitorId': str},
chunksize=10000)
 for cidx, df in enumerate(df_reader):
df.reset_index(drop=True, inplace=True)
 process_df(df,test_data)
 df_res = pd.concat([df_res,df ], axis=0 ).reset_index(drop=True)
 del df
gc.collect()
if cidx % 20 == 0:
print('{}: rows loaded: {}'.format(cidx, df_res.shape[0]))
if nrows_load:
if res.shape[0] >= nrows_load:
break
return df_res<data_type_conversions>
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d')
 all_data['DT_time'] = all_data['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 all_data['count'] = 1
all_data['diff_days_from_first_transaction'] = all_data['D1'].fillna(0 ).apply(lambda x:(datetime.timedelta(days = x)))
 all_data['client_firstdate'] = all_data['DT_time']- all_data['diff_days_from_first_transaction']
all_data['client_firstdate_days'] =(all_data['DT_time']- all_data['diff_days_from_first_transaction'] ).apply(lambda x:str(x.date()))
def parse_json(x,s):
res = json.loads(x)
 try:
return res[s]
except:
return float('NaN')
 def process_df(df,test_data):
df['days'] = df['date'].str[-2:]
df['days'] = df['days'].astype(int)
 df['month'] = df['date'].str[-4:-2]
df['month'] = df['month'].astype(int)
 df['year'] = df['date'].str[:4]
df['year'] = df['year'].astype(int)
 process_dict = {
'totals':['transactionRevenue','newVisits','pageviews','hits'] ,
'trafficSource':['campaign','source','medium'] ,
'device':['browser'],
'geoNetwork': ['country','city','continent','region','subContinent']
}
for c,l in process_dict.items() :
for it in l:
df[it] = df[c].apply(lambda x : parse_json(x,it))
cols = ['country','campaign','source','medium','continent','city','region','socialEngagementType','browser'
,'channelGrouping','subContinent','date']
labelencoder_X=LabelEncoder()
for c in cols:
df.loc[:,c] = labelencoder_X.fit_transform(df.loc[:,c])
 df['transactionRevenue'].fillna(0,inplace=True)
 df['newVisits'].fillna(0,inplace=True)
 df['pageviews'].fillna(0,inplace=True)
 df['transactionRevenue'] = df['transactionRevenue'].astype('float32')
 df['newVisits']= df['newVisits'].astype('uint16')
 df['pageviews'] = df['pageviews'].astype('uint16')
 df['hits'] = df['hits'].astype('uint32')
 rm_col = ['subContinent',
'channelGrouping','date','continent','customDimensions','fullVisitorId']
if test_data:
rm_col = rm_col[:-1]
df.drop(list(process_dict.keys())+ rm_col, axis=1,inplace=True)
 df = load_data(file_train,chunk_size)
 df_test =load_data(file_test,chunk_size,test_data=True )<groupby>
anaysis_fea = [ 'TransactionID',
'isFraud',
'TransactionDT',
'TransactionAmt',
'ProductCD',
'device_hash','card_hash', 'V307','id_30','id_31','id_32','id_33','DeviceType','DeviceInfo',
'card1','card2','card3','card4','card5','card6','client_firstdate_days','dist1','dist2','P_emaildomain','addr1','addr2','train_or_test','count','DT_time','diff_days_from_first_transaction','client_firstdate']
drop_fea = [col for col in all_data.columns if col not in anaysis_fea]
per = sum(df['transactionRevenue'] > 0)/ len(df['transactionRevenue'])
 print('Percentage of transactions greater than 0 : {} %'.format(per*100))<prepare_x_and_y>
all_data = all_data.drop(drop_fea,axis=1 )
X = df[df.columns[df.columns != 'transactionRevenue']]
Y = np.log1p(df['transactionRevenue'])
 X_train, X_val, y_train, y_val = train_test_split(X, Y, test_size=0.4, random_state=42)
 X_test = df_test[df_test.columns[df_test.columns != 'transactionRevenue']]
Y_test = np.log1p(df_test['transactionRevenue'])
 indices_nonzero = np.where(y_train > 0)
 indices_zero = np.where(y_train == 0)
 num_nonzero = len(indices_nonzero[0])
 num_zero = len(indices_zero[0])
 print('Number of non-zero transactions revenues = ', num_nonzero)
 print('Number of zero transactions revenues = ', num_zero)
 all_indx = list(indices_zero[0][0:num_nonzero])+ list(indices_nonzero[0])
 X_sample = X_train.iloc[all_indx,:]
y_sample = y_train.iloc[all_indx]
X_train_sample, X_val_sample, y_train_sample, y_val_sample = train_test_split(X_sample, y_sample, test_size=0.45, random_state=42)
 print('Sample X_Train shape ', X_train_sample.shape )<train_model>
gc.collect()
def create_model_xgboost(X_train,y_train,X_val=None,y_val=None):
params = {'objective': 'reg:linear',
'eval_metric': 'rmse',
'min_child_weight':1.5,
'eta': 0.01,
'max_depth': 4,
'subsample': 0.7,
'colsample_bytree': 0.6,
'reg_alpha':1,
'reg_lambda':0.45,
'random_state': 42,
'silent': True}
xgb_train_data = xgboost.DMatrix(X_train, y_train)
 if not X_val is None:
xgb_val_data = xgboost.DMatrix(X_val, y_val)
 evals=[(xgb_train_data, 'train'),(xgb_val_data, 'valid')]
else:
evals=[(xgb_train_data, 'train')]
model = xgboost.train(params, xgb_train_data,
num_boost_round=1200,
evals= evals,
early_stopping_rounds=50,
verbose_eval=300)
 return model
def create_model_nn(in_dim,layer_size=250):
model = Sequential()
model.add(Dense(layer_size,activation='relu',input_dim=in_dim))
model.add(BatchNormalization())
 model.add(Dense(layer_size,activation='relu'))
model.add(Dense(layer_size,activation='relu'))
model.add(Dense(1, activation='linear'))
adam = optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
 model.compile(optimizer=adam,loss='mse')
 return model
def create_model_lgbm(X_train,y_train,X_val=None,y_val=None):
dtrain = lgb.Dataset(X_train,label=y_train)
 dval = lgb.Dataset(X_val,label=y_val)
 param_up = {"objective" : "regression", "metric" : "rmse",
"max_depth": 2, "min_child_samples": 20,
"reg_alpha": 1.5, "reg_lambda": 1.5,
"num_leaves" : 15, "learning_rate" : 0.1,
"subsample" : 1, "colsample_bytree" : 1,
"verbosity": -1,'data_random_seed':4}
if not X_val is None:
valid_sets =(dtrain,dval)
 valid_names = ['train','valid']
else:
valid_sets =(dtrain)
 valid_names = ['train']
model = lgb.train(param_up,dtrain,num_boost_round=5000,valid_sets=valid_sets,valid_names=['train','valid'],verbose_eval=300,
early_stopping_rounds=50)
 return model<train_model>
all_data['card1'] = all_data['card1'].fillna(0)
 all_data['card2'] = all_data['card2'].fillna(0)
 all_data['card3'] = all_data['card3'].fillna(0)
 all_data['card5'] = all_data['card5'].fillna(0)
 all_data['card4'] = all_data['card4'].fillna('nan')
 all_data['card6'] = all_data['card6'].fillna('nan' )
def fit_train(x,y,X_val,y_val,layer_size=64,mod = 'nn'):
if mod =='nn':
model = create_model_nn(x.shape[1],layer_size)
 history = model.fit(x, y, epochs=3, batch_size=128,validation_data=(X_val,y_val),verbose=1)
 print('Mean RMSE : ',np.sqrt(history.history['val_loss'] ).mean())
 elif mod =='xgboost':
model = create_model_xgboost(x,y,X_val,y_val)
 elif mod =='lgbm':
model = create_model_lgbm(x,y,X_val,y_val)
 else:
raise Exception('invalid model')
 return model
def calc_rmse(pred,y):
diff = pred - y
RMSE =(( diff ** 2 ).mean())**.5
print('RMSE : ',RMSE)
 def eval_set(model,x,y,mod='nn'):
if mod == 'nn':
p = model.predict(x,batch_size=64,verbose=1)
 pred = p[:,0]
elif mod == 'xgboost':
dx = xgboost.DMatrix(x)
 pred = model.predict(dx,ntree_limit=model.best_ntree_limit)
 elif mod == 'lgbm':
pred = model.predict(x, num_iteration=model.best_iteration)
 else:
raise Exception('invalid model')
 calc_rmse(pred,y )<train_model>
all_data['card_hash'] = all_data.apply(lambda x: card_info_hash(x), axis=1 )
print('Training on training/val Dataset')
 model_training = fit_train(X_train,y_train,X_val,y_val)
 print('-'*40)
 print('Training on Sample Dataset')
 model_sample = fit_train(X_train_sample,y_train_sample,X_val_sample,y_val_sample,layer_size=500)
 print('-'*40)
 print('Training on Full training Dataset')
 model_full = create_model_nn(X.shape[1])
 model_full.fit(X, Y, epochs=3, batch_size=128,verbose=1)
 print('-'*40 )<train_model>
def get_data_by_card_hash(data, card_hash):
mask = data['card_hash']==card_hash
return data.loc[mask,:].copy()
def get_data_by_device_hash(data, device_hash):
mask = data['device_hash']==device_hash
return data.loc[mask,:].copy()
def get_data_by_card_and_startdate(data, card_hash, device_hash):
mask =(data['client_firstdate_days']==card_hash)&(data['card_hash']==device_hash)
 return data.loc[mask,:].copy()
print('Training on training/val Dataset')
 model_training = fit_train(X_train,y_train,X_val,y_val,mod='xgboost')
 print('-'*40)
 print('Training on Sample Dataset')
 model_sample = fit_train(X_train_sample,y_train_sample,X_val_sample,y_val_sample,layer_size=500,mod='xgboost')
 print('-'*40)
 print('Training on Full training Dataset')
 model_full = create_model_xgboost(X, Y)
 print('Full Training set -- Using Sample training model')
 eval_set(model_sample,X_train,y_train,mod='xgboost' )<train_model>
all_data['count']=1
grp = all_data.iloc[:train_len].groupby(['client_firstdate_days','card_hash'])['count'].agg('sum')
 display_group = get_data_by_card_and_startdate(all_data,grp[grp>10].index[0][0],grp[grp>10].index[0][1])
 display_group[['DT_time','TransactionAmt','V307','id_30','id_31','id_32','id_33','DeviceType','DeviceInfo','dist1','dist2','P_emaildomain']]
print('Training on training/val Dataset')
 model_training = fit_train(X_train,y_train,X_val,y_val,mod='lgbm')
 print('-'*40)
 print('Training on Sample Dataset')
 model_sample = fit_train(X_train_sample,y_train_sample,X_val_sample,y_val_sample,layer_size=500,mod='lgbm')
 print('-'*40)
 print('Training on Full training Dataset')
 model_full = create_model_lgbm(X, Y)
 print('Full Training set -- Using Sample training model')
 eval_set(model_sample,X_train,y_train,mod='lgbm')
 print('-'*40 )<save_to_csv>
s = all_data.iloc[:train_len].groupby(['client_firstdate_days','card_hash'])['isFraud'].agg(['mean', 'count'] )
df_test['predictions'] = model_full.predict(df_test.loc[:,df_test.columns[1:]],num_iteration=model_full.best_iteration)
 df_test.loc[df_test['predictions']< 0 ,'predictions']= 0
sub_df = pd.DataFrame({'fullVisitorId':df_test['fullVisitorId'] , 'PredictedLogRevenue': np.expm1(df_test['predictions'])})
 sub_df = sub_df.groupby("fullVisitorId")["PredictedLogRevenue"].sum().reset_index()
sub_df.columns = ["fullVisitorId", "PredictedLogRevenue"]
sub_df["PredictedLogRevenue"] = np.log1p(sub_df["PredictedLogRevenue"])
 sub_df.to_csv("baseline_lgb_submission.csv", index=False )<load_pretrained>
Test_ID=[]
for ind in tqdm(s[(s['mean']==1)].index):
very_strange_thing = get_data_by_card_and_startdate(all_data, ind[0],ind[1])
 Test_ID.extend(very_strange_thing[very_strange_thing['isFraud'].isna() ]['TransactionID'].tolist() )
train_df = pd.read_pickle('.. /input/gs-get-src-data/train_df', compression='gzip')
 print(train_df.shape)
 test_df = pd.read_pickle('.. /input/gs-get-src-data/test_df', compression='gzip')
 print(test_df.shape)
 train_df_u = pd.read_pickle('.. /input/gs-get-src-data/train_df_u', compression='gzip')
 print(train_df_u.shape)
 test_df_u = pd.read_pickle('.. /input/gs-get-src-data/test_df_u', compression='gzip')
 print(test_df_u.shape )<count_values>
np.save("test_IDs", Test_ID )
id_col = 'fullVisitorId'
cnt = train_df[id_col].value_counts()
ids = cnt.loc[cnt>1].index.values
print(ids.shape)
 df = train_df.loc[train_df[id_col].isin(ids)]
print(df.shape )<groupby>
submit = pd.read_csv(".. /input/ieeecis-fraud-detection-results/stack_gmean_09_16th_2019_LB0pt9530.csv")
gdf = df.groupby(id_col)['target'].sum()
tids = gdf.loc[gdf>0].index.values
print(tids.shape)
 tdf = df.loc[df[id_col].isin(tids)]
print(tdf.shape )<groupby>
mask = submit['TransactionID'].isin(Test_ID)
 submit.loc[mask,'isFraud'] =1
gdf = tdf.loc[:,[id_col,'target']]
gdf['target'] = gdf['target']>0
gdf = gdf.groupby(id_col)['target'].sum()
tids2 = gdf.loc[gdf>1].index.values
print(tids2.shape)
 tdf2 = tdf.loc[tdf[id_col].isin(tids2)]
print(tdf2.shape )<filter>
submit.loc[mask,:]
<groupby><EOS> 
submit.to_csv('submit_try.csv',index = False )