Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<load_from_csv>
gc.enable()
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
 %matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
 GradientBoostingClassifier, ExtraTreesClassifier)
pred = np.argmax(pred, axis=-1)
 sub = pd.read_csv('.. /input/digit-recognizer/sample_submission.csv')
 sub['Label'] = pred
sub.head()<save_to_csv>
useXGB = False
useLGBM = True
useEnsemble = False
if(useXGB)and(useLGBM):
useEnsemble = False
sub.to_csv('submission.csv', index=False )<save_to_csv>
%%time
train_transaction = pd.read_csv('.. /input/train_transaction.csv', index_col='TransactionID')
 test_transaction = pd.read_csv('.. /input/test_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('.. /input/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/test_identity.csv', index_col='TransactionID')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv', index_col='TransactionID')
 train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print(train.shape)
 print(test.shape )
sub.to_csv('submission.csv', index=False )<set_options>
def make_day_feature(df, offset=0, tname='TransactionDT'):

 days = df[tname] /(3600*24)
 encoded_days = np.floor(days-1+offset)% 7
return encoded_days
def make_hour_feature(df, tname='TransactionDT'):

 hours = df[tname] /(3600)
 encoded_hours = np.floor(hours)% 24
return encoded_hours
mpl.style.use('ggplot')
 sns.set_style('white')
 pylab.rcParams['figure.figsize'] = 12,8
warnings.filterwarnings('ignore')
 %matplotlib inline<load_from_csv>
train['hours'] = make_hour_feature(train)
 test['hours']= make_hour_feature(test )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 print('train dataset size:', train.shape)
 print('test dataset size:', test.shape)
 train.sample(4 )<categorify>
for f in train.columns:
if train[f].dtype=='object' or train[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train[f].values)+ list(test[f].values))
train[f] = lbl.transform(list(train[f].values))
test[f] = lbl.transform(list(test[f].values))
def proc_json(string, key):
try:
data = eval(string)
 return ",".join([d[key] for d in data])
 except:
return ''
def proc_json_len(string):
try:
data = eval(string)
 return len(data)
 except:
return 0
train.belongs_to_collection = train.belongs_to_collection.apply(lambda x: proc_json(x, 'name'))
test.belongs_to_collection = test.belongs_to_collection.apply(lambda x: proc_json(x, 'name'))
train.genres = train.genres.apply(lambda x: proc_json(x, 'name'))
test.genres = test.genres.apply(lambda x: proc_json(x, 'name'))
train.production_companies = train.production_companies.apply(lambda x: proc_json(x, 'name'))
test.production_companies = test.production_companies.apply(lambda x: proc_json(x, 'name'))
train.production_countries = train.production_countries.apply(lambda x: proc_json(x, 'iso_3166_1'))
test.production_countries = test.production_countries.apply(lambda x: proc_json(x, 'iso_3166_1'))
train.spoken_languages = train.spoken_languages.apply(lambda x: proc_json(x, 'iso_639_1'))
test.spoken_languages = test.spoken_languages.apply(lambda x: proc_json(x, 'iso_639_1'))
train.Keywords = train.Keywords.apply(lambda x: proc_json(x, 'name'))
test.Keywords = test.Keywords.apply(lambda x: proc_json(x, 'name'))
train.cast = train.cast.apply(proc_json_len)
 test.cast = test.cast.apply(proc_json_len)
 train.crew = train.crew.apply(proc_json_len)
 test.crew = test.crew.apply(proc_json_len )<count_missing_values>
y_train = train['isFraud'].copy()
del train_transaction, train_identity, test_transaction, test_identity
X_train = train.drop('isFraud', axis=1)
 X_test = test.copy()
del train, test
train.isnull().sum()<count_missing_values>
drop_col = dtcol +mcol[0:len(mcol)]+ idcol[5:39] + dcol[0:len(dcol)] + ccol[4:len(ccol)]
X_train.drop(drop_col,axis=1, inplace=True)
 X_test.drop(drop_col, axis=1, inplace=True)
 X_train.head()
train.isnull().sum()<feature_engineering>
def make_amt_feature(df, tname='TransactionAmt'):

 logAmt = np.log(df[tname])
 return logAmt
columns = ['original_title', 'title', 'overview', 'tagline']
for col in columns:
new_col = col + '_len'
train[new_col] = train[col].apply(lambda x: 0 if x is np.nan else len(x))
test[new_col] = test[col].apply(lambda x: 0 if x is np.nan else len(x))
columns.extend(['homepage', 'imdb_id', 'poster_path'])
 train.drop(columns, axis=1, inplace=True)
 test.drop(columns, axis=1, inplace=True )<count_missing_values>
X_train['LogAmt'] = make_amt_feature(X_train)
 X_test['LogAmt']= make_amt_feature(X_test )
print('-'*30, '
', train.isnull().sum())
 print('-'*30, '
', test.isnull().sum() )<set_options>
%%time
if useXGB :
NFOLDS = 6
kf = StratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=123)
 y_preds_xgb = np.zeros(X_test.shape[0])
 y_oof_xgb = np.zeros(X_train.shape[0])
 score = 0
for fold,(tr_idx, val_idx)in enumerate(kf.split(X_train, y_train)) :
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=9,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
gamma = 0.2,
alpha = 4,
missing = -1,
tree_method='gpu_hist' 
)
 X_tr, X_vl = X_train.iloc[tr_idx, :], X_train.iloc[val_idx, :]
y_tr, y_vl = y_train.iloc[tr_idx], y_train.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
y_oof_xgb[val_idx] = y_pred_train
print("FOLD: ",fold,' AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 score += roc_auc_score(y_vl, y_pred_train)/ NFOLDS
y_preds_xgb+= clf.predict_proba(X_test)[:,1] / NFOLDS
del X_tr, X_vl, y_tr, y_vl
gc.collect()
y_preds_xgb.reshape(-1,1)
 y_oof_xgb.reshape(-1,1)
 print("
MEAN AUC = {}".format(score))
print("OOF AUC = {}".format(roc_auc_score(y_train, y_oof_xgb)) )
train.runtime = train.runtime.fillna(0)
 test.runtime = test.runtime.fillna(0 )<feature_engineering>
%%time
if useXGB :
NFOLDS = 5
kf = StratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=0)
 y_preds_xgb1 = np.zeros(X_test.shape[0])
 y_oof_xgb1 = np.zeros(X_train.shape[0])
 score = 0
for fold,(tr_idx, val_idx)in enumerate(kf.split(X_train, y_train)) :
clf = xgb.XGBClassifier(
 n_estimators=1500,
max_depth=10,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
gamma = 0.2,
alpha = 4,
missing = -1,
tree_method='gpu_hist' 
)
 X_tr, X_vl = X_train.iloc[tr_idx, :], X_train.iloc[val_idx, :]
y_tr, y_vl = y_train.iloc[tr_idx], y_train.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
y_oof_xgb1[val_idx] = y_pred_train
print("FOLD: ",fold,' AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 score += roc_auc_score(y_vl, y_pred_train)/ NFOLDS
y_preds_xgb1+= clf.predict_proba(X_test)[:,1] / NFOLDS
del X_tr, X_vl, y_tr, y_vl
gc.collect()
y_preds_xgb1.reshape(-1,1)
 y_oof_xgb1.reshape(-1,1)
 print("
MEAN AUC = {}".format(score))
print("OOF AUC = {}".format(roc_auc_score(y_train, y_oof_xgb1)) )
test.loc[test.status.isnull() , 'status'] = 'Released'<feature_engineering>
%%time
if useXGB :
NFOLDS = 5
kf = StratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=42)
 y_preds_xgb2 = np.zeros(X_test.shape[0])
 y_oof_xgb2 = np.zeros(X_train.shape[0])
 score = 0
for fold,(tr_idx, val_idx)in enumerate(kf.split(X_train, y_train)) :
clf = xgb.XGBClassifier(
 n_estimators=1500,
max_depth=10,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
gamma = 0.2,
alpha = 4,
missing = -1,
tree_method='gpu_hist' 
)
 X_tr, X_vl = X_train.iloc[tr_idx, :], X_train.iloc[val_idx, :]
y_tr, y_vl = y_train.iloc[tr_idx], y_train.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
y_oof_xgb2[val_idx] = y_pred_train
print("FOLD: ",fold,' AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 score += roc_auc_score(y_vl, y_pred_train)/ NFOLDS
y_preds_xgb2+= clf.predict_proba(X_test)[:,1] / NFOLDS
del X_tr, X_vl, y_tr, y_vl
gc.collect()
y_preds_xgb2.reshape(-1,1)
 y_oof_xgb2.reshape(-1,1)
 print("
MEAN AUC = {}".format(score))
print("OOF AUC = {}".format(roc_auc_score(y_train, y_oof_xgb2)) )
test.loc[test.release_date.isnull() , 'release_date'] = test.release_date.mode() [0] 
<feature_engineering>
param = {'num_leaves': 30,
'min_data_in_leaf': 30,
'objective':'binary',
'max_depth': -1,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.8,
"bagging_freq": 1,
"bagging_fraction": 0.8 ,
"bagging_seed": 11,
"metric": 'auc',
"lambda_l1": 0.1,
"random_state": 133,
"verbosity": -1}
def expand_release_date(df):
df.release_date = pd.to_datetime(df.release_date)
 df['release_year'] = df.release_date.dt.year
df['release_month'] = df.release_date.dt.month
df['release_day'] = df.release_date.dt.dayofweek
df['release_quarter'] = df.release_date.dt.quarter
return df
train = expand_release_date(train )<feature_engineering>
max_iter = 5
gc.collect()
def expand_release_date(df):
df.release_date = pd.to_datetime(df.release_date)
 df['release_year'] = df.release_date.dt.year
df['release_year'] = df.release_year.apply(lambda x: x-100 if x > 2020 else x)
 df['release_month'] = df.release_date.dt.month
df['release_day'] = df.release_date.dt.dayofweek
df['release_quarter'] = df.release_date.dt.quarter
return df
train = expand_release_date(train)
 test = expand_release_date(test )<data_type_conversions>
%%time
if useLGBM :
folds = TimeSeriesSplit(n_splits =5)
 oof = np.zeros(len(X_train))
features = [c for c in X_train.columns]
predictions = np.zeros(len(X_test))
start = time.time()
feature_importance_df = pd.DataFrame()
start_time= time.time()
score = [0 for _ in range(folds.n_splits)]
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X_train, y_train)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(X_train.iloc[trn_idx][features],
label=y_train.iloc[trn_idx] 
)
 val_data = lgb.Dataset(X_train.iloc[val_idx][features],
label=y_train.iloc[val_idx] 
)
 num_round = 500
clf = lgb.train(param,
trn_data,
num_round,
valid_sets = [trn_data, val_data],
verbose_eval=10,
early_stopping_rounds = 100)
 oof[val_idx] = clf.predict(X_train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance(importance_type='gain')
 fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 initial_idx = 0
chunk_size = 100000
current_pred = np.zeros(len(X_test))
while initial_idx < X_test.shape[0]:
final_idx = min(initial_idx + chunk_size, X_test.shape[0])
 idx = range(initial_idx, final_idx)
 current_pred[idx] = clf.predict(X_test.iloc[idx][features], num_iteration=clf.best_iteration)
 initial_idx = final_idx
predictions += current_pred / min(folds.n_splits, max_iter)
 print("time elapsed: {:<5.2}s".format(( time.time() - start_time)/ 3600))
score[fold_] = metrics.roc_auc_score(y_train.iloc[val_idx], oof[val_idx])
 if fold_ == max_iter - 1: break
if(folds.n_splits == max_iter):
print("CV score: {:<8.5f}".format(metrics.roc_auc_score(y_train, oof)))
 else:
print("CV score: {:<8.5f}".format(sum(score)/ max_iter))
genres = []
for idx, val in train.genres.iteritems() :
gen_list = val.split(',')
 for gen in gen_list:
if gen == '':
continue
if gen not in genres:
genres.append(gen)
 genre_column_names = []
for gen in genres:
col_name = 'genre_' + gen.replace(' ', '_')
 train[col_name] = train.genres.str.contains(gen ).astype('uint8')
 test[col_name] = test.genres.str.contains(gen ).astype('uint8')
 genre_column_names.append(col_name)
 train.sample(5 )<feature_engineering>
feature_importance_df['fold'].value_counts()
train['genre_count'] = train.genres.apply(lambda x: len(x.split(',')))
 test['genre_count'] = test.genres.apply(lambda x: len(x.split(',')) )<data_type_conversions>
df_lose_it = feature_importance_df.loc[feature_importance_df['fold'] == 5].sort_values(by="importance", ascending=True ).head(100 )
train['country_us'] = train.production_countries.str.contains('US' ).astype('uint8')
 train['country_count'] = train.production_countries.apply(lambda x: len(x.split(',')))
 train['company_count'] = train.production_companies.apply(lambda x: len(x.split(',')))
 test['country_us'] = test.production_countries.str.contains('US' ).astype('uint8')
 test['country_count'] = test.production_countries.apply(lambda x: len(x.split(',')))
 test['company_count'] = test.production_companies.apply(lambda x: len(x.split(',')) )<data_type_conversions>
df_lose_it=df_lose_it.loc[df_lose_it['importance'] == 0.0000]
drop_add = []
for vals in df_lose_it['feature'] :
drop_add.append(vals)
train['slc'] = train.spoken_languages.apply(lambda x: len(x.split(',')))
 train['orig_lang_code'] = train.original_language.astype('category' ).cat.codes
test['slc'] = test.spoken_languages.apply(lambda x: len(x.split(',')))
 test['orig_lang_code'] = test.original_language.astype('category' ).cat.codes<categorify>
X_train.drop(drop_add,axis=1, inplace=True)
 X_test.drop(drop_add, axis=1, inplace=True)
 X_train.head()
train['total_staff'] = train.cast + train.crew
test['total_staff'] = test.cast + test.crew<define_variables>
%%time
if useLGBM :
folds = TimeSeriesSplit(n_splits =5)
 oof = np.zeros(len(X_train))
features = [c for c in X_train.columns]
predictions = np.zeros(len(X_test))
start = time.time()
feature_importance_df = pd.DataFrame()
start_time= time.time()
score = [0 for _ in range(folds.n_splits)]
for fold_,(trn_idx, val_idx)in enumerate(folds.split(X_train, y_train)) :
print("fold n°{}".format(fold_))
trn_data = lgb.Dataset(X_train.iloc[trn_idx][features],
label=y_train.iloc[trn_idx] 
)
 val_data = lgb.Dataset(X_train.iloc[val_idx][features],
label=y_train.iloc[val_idx] 
)
 num_round = 10000
clf = lgb.train(param,
trn_data,
num_round,
valid_sets = [trn_data, val_data],
verbose_eval=200,
early_stopping_rounds = 200)
 oof[val_idx] = clf.predict(X_train.iloc[val_idx][features], num_iteration=clf.best_iteration)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance(importance_type='gain')
 fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 initial_idx = 0
chunk_size = 100000
current_pred = np.zeros(len(X_test))
while initial_idx < X_test.shape[0]:
final_idx = min(initial_idx + chunk_size, X_test.shape[0])
 idx = range(initial_idx, final_idx)
 current_pred[idx] = clf.predict(X_test.iloc[idx][features], num_iteration=clf.best_iteration)
 initial_idx = final_idx
predictions += current_pred / min(folds.n_splits, max_iter)
 print("time elapsed: {:<5.2}s".format(( time.time() - start_time)/ 3600))
score[fold_] = metrics.roc_auc_score(y_train.iloc[val_idx], oof[val_idx])
 if fold_ == max_iter - 1: break
if(folds.n_splits == max_iter):
print("CV score: {:<8.5f}".format(metrics.roc_auc_score(y_train, oof)))
 else:
print("CV score: {:<8.5f}".format(sum(score)/ max_iter))
<train_model><EOS> 
sample_submission['isFraud'] = predictions
sample_submission.to_csv('simple_ensemble.csv' )