Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<count_values>
warnings.filterwarnings('ignore')
 sns.set_style('darkgrid')
 pd.options.display.float_format = '{:,.3f}'.format
print(os.listdir(".. /input"))
old_title=new_data['title'].value_counts()<prepare_x_and_y>
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 seed_everything(42 )
a=old_title[old_title.values==3].index
b=old_title[old_title==2].index<feature_engineering>
%%time
train_id = pd.read_csv('.. /input/train_identity.csv')
 train_trn = pd.read_csv('.. /input/train_transaction.csv')
 test_id = pd.read_csv('.. /input/test_identity.csv')
 test_trn = pd.read_csv('.. /input/test_transaction.csv')
 print(train_id.shape, test_id.shape)
 print(train_trn.shape, test_trn.shape )
new_data['fan_pai_2']=new_data['title'].apply(lambda x:1 if x in a else 0)
 new_data['pan_pai_3']=new_data['title'].apply(lambda x:1 if x in b else 0 )<filter>
id_cols = list(train_id.columns.values)
 trn_cols = list(train_trn.drop('isFraud', axis=1 ).columns.values)
 X_train = pd.merge(train_trn[trn_cols + ['isFraud']], train_id[id_cols], how='left')
 X_train = reduce_mem_usage(X_train)
 X_test = pd.merge(test_trn[trn_cols], test_id[id_cols], how='left')
 X_test = reduce_mem_usage(X_test)
 X_train_id = X_train.pop('TransactionID')
 X_test_id = X_test.pop('TransactionID')
 del train_id,train_trn,test_id,test_trn
all_data = X_train.append(X_test, sort=False ).reset_index(drop=True )
new_data.loc[new_data['title'].fillna('un' ).str.contains('Planet of the Apes')]<sort_values>
def encode_loop(df, col, drop=True):
df[col + '_cos'] = np.cos(2 * np.pi * df[col] / df[col].max())
 df[col + '_sin'] = np.sin(2 * np.pi * df[col] / df[col].max())
 if drop:
df.drop(col, axis=1, inplace=True)
 return df
top10 = train.sort_values(by='revenue',ascending=False)[:10]
id10=top10[['title','poster_path','revenue']]<count_values>
ccols = [f'C{i}' for i in range(1,15)]
dcols = [f'D{i}' for i in range(1,16)]
mcols = ['M1','M2','M3','M5','M6','M7','M8','M9']
title_count=train['title'].value_counts()
len(title_count[title_count.values!=1] )<filter>
all_data['_log_dist_1_2'] = np.log1p(np.where(all_data['dist1'].isna() , all_data['dist2'], all_data['dist1']))
train[train['title'].str.contains('Furious')]['title'] 
<filter>
START_DATE = '2017-11-30'
startdate = datetime.datetime.strptime(START_DATE, "%Y-%m-%d")
 trandate = all_data['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds=x)))
 all_data['_days'] = all_data['TransactionDT'] //(24*60*60)
 all_data['_hour'] = trandate.dt.hour
all_data = encode_loop(all_data, '_hour')
 all_data['_weekday__hour'] = trandate.dt.dayofweek.astype(str)+ '_' + trandate.dt.hour.astype(str )
test[test['title'].fillna('Unknow' ).str.contains('Furious')]['title']<drop_column>
all_data['_P_emaildomain__ProductCD'] = all_data['P_emaildomain'] + '__' + all_data['ProductCD']
all_data['_card3__card5'] = all_data['card3'].astype(str)+ '__' + all_data['card5'].astype(str )
d.remove(d[7] )<feature_engineering>
all_data['_uid1'] =(all_data['_days'] - all_data['D1'] ).astype(str)+ '__' + all_data['P_emaildomain'].astype(str)
 all_data['_uid2'] = all_data['card1'].astype(str)+ '__' + all_data['addr1'].astype(str)+ '__' + all_data['_uid1']
group_key = ['_uid2']
all_data = all_data.assign(
 _day_lag_uid2 = all_data['TransactionDT'] - all_data.groupby(group_key)['TransactionDT'].shift(1)
 ,_amount_lag_pct_uid2 = np.abs(all_data.groupby(group_key)['TransactionAmt'].pct_change())
)
 _=
for i in d :
new_data['crew_contains_'+i[0][2]]=new_data['crew'].fillna('' ).apply(lambda x:1 if i[0][2] in x else 0)
<feature_engineering>
all_data['_amount_decimal'] =(( all_data['TransactionAmt'] - all_data['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 all_data['_amount_decimal_len'] = all_data['TransactionAmt'].apply(lambda x: len(re.sub('0+$', '', str(x)).split('.')[1]))
all_data['_amount_fraction'] = all_data['TransactionAmt'].apply(lambda x: float('0.'+re.sub('^[0-9]|\.|0+$', '', str(x))))
mon_is=[1,6,7,9,10,12]
for i in mon_is:
new_data['mon_is_'+str(i)]=new_data['release_month'].apply(lambda x:1 if x==i else 0 )<feature_engineering>
def values_agg(df, periods, columns, aggs=['max']):
for period in periods:
for col in columns:
if col in df.columns:
new_col = f'{col}_{period}'
grouped_col = df.groupby([period])[col]
for a in aggs:
df[f'_{a}_{new_col}'] = df[period].map(grouped_col.agg(a ).to_dict())
 return df
all_data = values_agg(all_data, ['_uid2'], ccols)
new_data['spoken_languages_count']=new_data['spoken_languages'].apply(lambda x:len(x))<groupby>
amt_cols = ['_uid2','_P_emaildomain__ProductCD']
all_data = values_agg(all_data, amt_cols, ['TransactionAmt'], aggs=['max','mean','var'] )
new_data.loc[train.index].groupby('spoken_languages_count' ).revenue.median()<categorify>
for f in ccols + amt_cols:
vc = all_data[f].value_counts(dropna=False)
 all_data[f'_count_full_{f}'] = all_data[f].map(vc )
status_du=pd.get_dummies(new_data['status'] )<concatenate>
all_data['_all_na'] = all_data.isna().sum(axis=1 ).astype(np.int8)
new_data=pd.concat([new_data,status_du],axis=1)
<drop_column>
all_data['_ccols_nonzero'] = all_data[ccols].apply(lambda x: len(x.to_numpy().nonzero() [0]), axis=1)
 all_data['_ccols_sum'] = all_data[ccols].sum(axis=1 ).astype(np.int8)
 all_data['_ccols_0_bin'] = ''
for c in ccols:
all_data['_ccols_0_bin'] +=(all_data[c] == 0 ).astype(int ).astype(str)
 all_data.drop(ccols, axis=1, inplace=True )
new_data=new_data.drop(['status'],axis=1)
 new_data.head()<drop_column>
all_data['_dcol_na'] = all_data[dcols].isna().sum(axis=1 ).astype(np.int8)
 all_data['_dcols_na_bin'] = ''
for c in dcols:
all_data['_dcols_na_bin'] += all_data[c].isna().astype(int ).astype(str)
 for f in ['D1','D2']:
all_data[f] = all_data[f].fillna(0)- all_data['_days']
for f in ['D3','D4','D5','D6','D7','D10','D11','D12','D13','D14','D15']:
all_data[f] = all_data[f].fillna(0)- all_data['_days']
new_data=new_data.drop(['original_language'],axis=1 )<count_missing_values>
all_data['_mcols_na_bin'] = ''
for c in mcols:
all_data['_mcols_na_bin'] += all_data[c].isna().astype(int ).astype(str )
sum(new_data['geres_name'].isna() )<data_type_conversions>
vcols = [f'V{i}' for i in range(1,340)]
sc = preprocessing.MinMaxScaler()
dec = PCA(n_components=2, random_state=42)
 vcol_dec = dec.fit_transform(sc.fit_transform(all_data[vcols].fillna(-1)))
 all_data['_vcols_dec0'] = vcol_dec[:,0]
all_data['_vcols_dec1'] = vcol_dec[:,1]
all_data['_vcols_na'] = all_data[vcols].isna().sum(axis=1 ).astype(np.int8)
 for f in ['V144','V145','V150','V151','V159','V160','V307']:
vcols.remove(f)
 all_data['_vcols_sum'] = all_data[vcols].sum(axis=1 ).astype(np.int8)
 all_data.drop(vcols, axis=1, inplace=True )
new_data['geres_name']=new_data['geres_name'].fillna('Unknow' )<feature_engineering>
_=
cnt_day = all_data['_days'].value_counts()
all_data['_count_trns_day'] = all_data['_days'].map(cnt_day.to_dict())
 daily_cols = ccols
for f in daily_cols:
if f in all_data.columns:
val_day = all_data[f].astype(str)+ '__' + all_data['_days'].astype(str)
 vc = val_day.value_counts(dropna=True)
 all_data[f'_count_day_{f}'] = val_day.map(vc)
 all_data[f'_count_pct_day_{f}'] = all_data[f'_count_day_{f}'] / all_data['_count_trns_day']
all_data.drop('_count_trns_day', axis=1, inplace=True )
new_data['geres_name_count']=new_data['geres_name'].apply(lambda x:len(x))<groupby>
_=
columns_to_drop = ['id_07','id_08','id_10','id_12','id_16',
'id_21','id_22','id_23','id_24','id_25',
'id_26','id_27','id_28','id_29','id_32',
'id_34','id_35','id_36','id_37']
all_data.drop(columns_to_drop, axis=1, inplace=True )
ge_name_count_rev=new_data.loc[train.index].groupby(['geres_name_count'] ).revenue.median().sort_values()<drop_column>
many_same_values_columns = [c for c in all_data.drop('isFraud', axis=1 ).columns if all_data[c].value_counts(normalize=True ).values[0] >= 0.98]
columns_to_drop = list(many_same_values_columns)
 print('drop columns:', columns_to_drop)
 all_data.drop(columns_to_drop, axis=1, inplace=True )
new_data=new_data.drop('homepage_end',axis=1 )<feature_engineering>
all_data.drop(['_days'], axis=1, inplace=True)
 all_data.drop(['TransactionDT'], axis=1, inplace=True)
 uid = all_data['_uid2']
drop_card_cols = ['_uid1','_uid2','card1','card2']
new_data['original_and_new']=new_data['original_and_new'].apply(lambda x: 1 if x else 0 )<define_variables>
_=
cat_cols = ['ProductCD','card1','card2','card3','card4','card5','card6','addr1','addr2','P_emaildomain','R_emaildomain',
'M1','M2','M3','M4','M5','M6','M7','M8','M9','DeviceType','DeviceInfo'] + [f'id_{i}' for i in range(12,39)]
for i in cat_cols:
if i in all_data.columns:
all_data[i] = all_data[i].astype(str)
 enc_cols = []
for i, t in all_data.loc[:, all_data.columns != 'isFraud'].dtypes.iteritems() :
if t == object:
enc_cols.append(i)
 all_data[i] = pd.factorize(all_data[i])[0]
print(enc_cols )
list_geres_name=['Comedy','Thriller','Action','Drama','Romance']<feature_engineering>
X_train = all_data[all_data['isFraud'].notnull() ]
X_test = all_data[all_data['isFraud'].isnull() ].drop('isFraud', axis=1)
 Y_train = X_train.pop('isFraud')
 uid_train = uid[all_data['isFraud'].notnull() ]
uid_fraud = uid_train[Y_train == 1]
uid_test = uid[all_data['isFraud'].isnull() ]
train_group = trandate[:len(X_train)].dt.month
del uid
del all_data
for i in list_geres_name:
new_data['geres_name'+'_'+i]=new_data['geres_name'].apply(lambda x: 1 if i in x else 0 )<drop_column>
_=
new_data=new_data.drop('geres_name',axis=1 )<data_type_conversions>
cat_cols = ['card1','card2','card3','card5','addr1','_uid1','_uid2','_P_emaildomain__ProductCD','_card3__card5']
for f in cat_cols:
if f in X_train.columns:
train_set = set(X_train[f])
 test_set = set(X_test[f])
 tt = train_set.intersection(test_set)
 print(f, '-', 'train:%.3f'%(len(tt)/len(train_set)) , ',test:%.3f'%(len(tt)/len(test_set)))
 X_train[f] = X_train[f].map(lambda x: -999 if x not in tt else x)
 X_test[f] = X_test[f].map(lambda x: -999 if x not in tt else x )
new_data['production_countries']=new_data['production_countries'].fillna('QQ' )<feature_engineering>
def scale_minmax(preds):
return(preds - preds.min())/(preds.max() - preds.min())
 def pred_lgb(X_train, Y_train, X_test, params, num_iterations=1000):
feature_importances = pd.DataFrame()
feature_importances['feature'] = X_train.columns
oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 dtrain = lgb.Dataset(X_train[:312574], label=Y_train.iloc[:312574])
 dvalid = lgb.Dataset(X_train[-175998:], label=Y_train[-175998:])
 clf = lgb.train(params, dtrain, 5000,
valid_sets=[dtrain, dvalid], valid_names=['train', 'valid'],
verbose_eval=1000, early_stopping_rounds=200)
 feature_importances['fold_0'] = clf.feature_importance()
oof_preds = clf.predict(X_train)
 sub_preds = clf.predict(X_test)
 return oof_preds, sub_preds, feature_importances
def pred_lgb_kfold(X_train, Y_train, X_test, params, nfolds=5):
feature_importances = pd.DataFrame()
feature_importances['feature'] = X_train.columns
oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 kf = KFold(n_splits=nfolds, shuffle=False, random_state=42)
 for fold,(train_index, test_index)in enumerate(kf.split(X_train, Y_train)) :
dtrain = lgb.Dataset(X_train.iloc[train_index], label=Y_train.iloc[train_index])
 dvalid = lgb.Dataset(X_train.iloc[test_index], label=Y_train.iloc[test_index])
 clf = lgb.train(params, dtrain, 5000,
valid_sets=[dtrain, dvalid], valid_names=['train', 'valid'],
verbose_eval=1000, early_stopping_rounds=200)
 feature_importances['fold_{}'.format(fold + 1)] = clf.feature_importance()
oof_preds[test_index] = clf.predict(X_train.iloc[test_index])
 sub_preds += clf.predict(X_test)/ nfolds
return oof_preds, sub_preds, feature_importances
new_data['production_countries'].loc[new_data['production_countries']=='Unknow']['production_countries']='QQ'<count_unique_values>
def pred_lgb_LOGO(X_train, Y_train, X_test, groups, params, undersampling=False):
feature_importances = pd.DataFrame()
feature_importances['feature'] = X_train.columns
oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 nfolds = groups.nunique()
kf = LeaveOneGroupOut()
for fold,(train_index, test_index)in enumerate(kf.split(X_train, Y_train, groups)) :
if undersampling:
size1 = sum(Y_train.iloc[train_index]==1)
 X_train_us, Y_train_us = make_imbalance(
 X_train.iloc[train_index].fillna(-9999), Y_train.iloc[train_index],
sampling_strategy={0:size1*5, 1:size1}, random_state=42)
 X_train_us = pd.DataFrame(X_train_us, columns=X_train.columns)
 dtrain = lgb.Dataset(X_train_us, label=Y_train_us)
 else:
dtrain = lgb.Dataset(X_train.iloc[train_index], label=Y_train.iloc[train_index])
 dvalid = lgb.Dataset(X_train.iloc[test_index], label=Y_train.iloc[test_index])
 clf = lgb.train(params, dtrain, 5000,
valid_sets=[dtrain, dvalid], valid_names=['train', 'valid'],
verbose_eval=1000, early_stopping_rounds=200)
 feature_importances['fold_{}'.format(fold + 1)] = clf.feature_importance()
oof_preds[test_index] = clf.predict(X_train.iloc[test_index])
 sub_preds += clf.predict(X_test)/ nfolds
return oof_preds, sub_preds, feature_importances
list_pro_coun=list(new_data['production_countries'])
 d = Counter([j for i in list_pro_coun for j in i] ).most_common(5)
 d<feature_engineering>
def pred_lgb_LOGO_2(X_train, Y_train, X_test, groups, params):
feature_importances = pd.DataFrame()
feature_importances['feature'] = X_train.columns
oof_preds = np.zeros(X_train.shape[0])
 sub_preds = np.zeros(X_test.shape[0])
 nfolds = groups.nunique()
print('groups:', nfolds)
 kf = LeaveOneGroupOut()
pred_count = 0
for fold,(tt_index, out_index)in enumerate(kf.split(X_train, Y_train, groups)) :
train_index = tt_index[tt_index < out_index[0]]
test_index = tt_index[tt_index > out_index[len(out_index)- 1]]
print(f'fold[{fold+1}] - {len(tt_index)}(train:{len(train_index)}, valid:{len(test_index)})')
 if len(train_index)== 0 or len(test_index)== 0:
print('- skip')
 continue
pred_count += 1
dtrain = lgb.Dataset(X_train.iloc[train_index], label=Y_train.iloc[train_index])
 dvalid = lgb.Dataset(X_train.iloc[test_index], label=Y_train.iloc[test_index])
 clf = lgb.train(params, dtrain, 5000,
valid_sets=[dtrain, dvalid], valid_names=['train', 'valid'],
verbose_eval=500, early_stopping_rounds=200)
 feature_importances['fold_{}'.format(pred_count)] = clf.feature_importance()
oof_preds += clf.predict(X_train)
 sub_preds += clf.predict(X_test)
 oof_preds = oof_preds / pred_count
sub_preds = sub_preds / pred_count
return oof_preds, sub_preds, feature_importances
for i in d:
new_data['production_cou_name'+'_'+i[0]]=new_data['production_countries'].apply(lambda x: 1 if i[0] in x else 0 )<feature_engineering>
%%time
params={
'learning_rate': 0.01,
'objective': 'binary',
'boost_from_average': False,
'is_unbalance': False,
'metric': 'auc',
'num_threads': -1,
'num_leaves': 256,
'max_bin': 256,
'verbose': 1,
'random_state': 42,
'bagging_fraction': 0.85,
'bagging_freq': 1,
'feature_fraction': 0.60
}
oof_preds, sub_preds, feature_importances = pred_lgb_LOGO(X_train, Y_train, X_test, train_group, params)
new_data['pro_country_count']=new_data['production_companies'].apply(lambda x:len(x))<groupby>
_=
pro_count_rev=new_data.loc[train.index].groupby('pro_country_count' ).revenue.median()<drop_column>
feature_importances.sort_values(by='average', ascending=False)['feature'].values
new_data=new_data.drop('production_countries',axis=1 )<categorify>
submission = pd.DataFrame()
submission['TransactionID'] = X_test_id
submission['isFraud'] = sub_preds
submission.loc[uid_test.isin(uid_fraud ).values, 'isFraud'] = 1
submission.to_csv('submission.csv', index=False )
<drop_column><EOS> 
np.mean(sub_preds )