Kaggle competition: IEEE-CIS Fraud Detection
def create_submission(predictions, test_id, info):
result1 = pd.DataFrame(predictions, columns=['c0', 'c1', 'c2', 'c3', 'c4', 'c5', 'c6', 'c7', 'c8', 'c9'])
 result1.loc[:, 'img'] = pd.Series(test_id, index=result1.index)
 result1 = result1.sort_values(['img'])
 result1.to_csv(index=False)
 return result1<compute_test_metric>
%%time
train_transaction = pd.read_csv('.. /input/train_transaction.csv', index_col='TransactionID')
 test_transaction = pd.read_csv('.. /input/test_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('.. /input/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/test_identity.csv', index_col='TransactionID')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv', index_col='TransactionID')
 train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print(train.shape)
 print(test.shape )
info = '200824'
submission = create_submission(preds, y_test, info )<define_variables>
def reduce_mem_usage(df):
start_mem_usg = df.memory_usage().sum() / 1024**2
print("Memory usage of properties dataframe is :",start_mem_usg," MB")
 NAlist = []
for col in df.columns:
if df[col].dtype != object:
print("******************************")
 print("Column: ",col)
 print("dtype before: ",df[col].dtype)
 IsInt = False
mx = df[col].max()
mn = df[col].min()
print("min for this col: ",mn)
 print("max for this col: ",mx)
 if not np.isfinite(df[col] ).all() :
NAlist.append(col)
 df[col].fillna(mn-1,inplace=True)
 asint = df[col].fillna(0 ).astype(np.int64)
 result =(df[col] - asint)
 result = result.sum()
if result > -0.01 and result < 0.01:
IsInt = True
if IsInt:
if mn >= 0:
if mx < 255:
df[col] = df[col].astype(np.uint8)
 elif mx < 65535:
df[col] = df[col].astype(np.uint16)
 elif mx < 4294967295:
df[col] = df[col].astype(np.uint32)
 else:
df[col] = df[col].astype(np.uint64)
 else:
if mn > np.iinfo(np.int8 ).min and mx < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif mn > np.iinfo(np.int16 ).min and mx < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif mn > np.iinfo(np.int32 ).min and mx < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif mn > np.iinfo(np.int64 ).min and mx < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
df[col] = df[col].astype(np.float32)
 print("dtype after: ",df[col].dtype)
 print("******************************")
 print("___MEMORY USAGE AFTER COMPLETION:___")
 mem_usg = df.memory_usage().sum() / 1024**2
print("Memory usage is: ",mem_usg," MB")
 print("This is ",100*mem_usg/start_mem_usg,"% of the initial size")
 return df, NAlist
now = datetime.datetime.now()
if not os.path.isdir('subm'):
os.mkdir('subm')
 suffix = info + '_' + str(now.strftime("%Y-%m-%d-%H-%M"))
sub_file = os.path.join('subm', 'submission_' + suffix + '.csv' )<save_to_csv>
y_train = train['isFraud'].copy()
del train_transaction, train_identity, test_transaction, test_identity
X_train = train.drop('isFraud', axis=1)
 X_test = test.copy()
del train, test
submission.to_csv('submission.csv', index=False )<save_to_csv>
X_train = X_train.fillna(-999)
 X_test = X_test.fillna(-999)
 for f in X_train.columns:
if X_train[f].dtype=='object' or X_test[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(X_train[f].values)+ list(X_test[f].values))
X_train[f] = lbl.transform(list(X_train[f].values))
X_test[f] = lbl.transform(list(X_test[f].values))
submission.to_csv('submission.csv', index=False )<set_options>
X=pd.concat([X_train,y_train],axis=1)
os.environ['KERAS_BACKEND'] = 'tensorflow'
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
warnings.filterwarnings('ignore')
<load_from_csv>
X['isFraud'].value_counts()
dataset = pd.read_csv('.. /input/driver_imgs_list.csv')
 dataset.head(5 )<define_variables>
not_fraud=X[X.isFraud==0]
fraud=X[X.isFraud==1]
fraud_upsampled = resample(fraud,
replace=True,
n_samples=len(not_fraud),
random_state=27)
 upsampled = pd.concat([not_fraud, fraud_upsampled])
 upsampled.isFraud.value_counts()
activity_map = {'c0': 'Safe driving',
'c1': 'Texting - right',
'c2': 'Talking on the phone - right',
'c3': 'Texting - left',
'c4': 'Talking on the phone - left',
'c5': 'Operating the radio',
'c6': 'Drinking',
'c7': 'Reaching behind',
'c8': 'Hair and makeup',
'c9': 'Talking to passenger'}
plt.figure(figsize =(12, 20))
image_count = 1
BASE_URL = '.. /input/train/'
for directory in os.listdir(BASE_URL):
if directory[0] != '.':
for i, file in enumerate(os.listdir(BASE_URL + directory)) :
if i == 1:
break
else:
fig = plt.subplot(5, 2, image_count)
 image_count += 1
image = mpimg.imread(BASE_URL + directory + '/' + file)
 plt.imshow(image)
 plt.title(activity_map[directory] )<choose_model_class>
columns = []
corr =corr1[np.abs(corr1)> 0.3]
corr = pd.DataFrame(data=corr,columns= ['isFraud'])
 corr = corr.drop_duplicates().sort_values('isFraud',ascending = False)
 corr.shape
classifier = Sequential()
classifier.add(Conv2D(filters = 128, kernel_size =(3, 3), activation = 'relu', input_shape =(240, 240, 3), data_format = 'channels_last'))
classifier.add(MaxPooling2D(pool_size =(2, 2)))
 classifier.add(Conv2D(filters = 64, kernel_size =(3, 3), activation = 'relu'))
classifier.add(MaxPooling2D(pool_size =(2, 2)))
 classifier.add(Conv2D(filters = 32, kernel_size =(3, 3), activation = 'relu'))
classifier.add(MaxPooling2D(pool_size =(2, 2)))
 classifier.add(Flatten())
 classifier.add(Dense(units = 1024, activation = 'relu'))
classifier.add(Dense(units = 256, activation = 'relu'))
classifier.add(Dense(units = 10, activation = 'sigmoid'))
classifier.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])
 classifier.summary()<load_pretrained>
X_train_1 = filter_data
X_train_1 = X_train_1.drop(columns = ['isFraud'] , axis =1)
 y_train_1 = filter_data['isFraud']
train_datagen = ImageDataGenerator(rescale = 1.0/255,
shear_range = 0.2,
zoom_range = 0.2,
horizontal_flip = True,
validation_split = 0.2)
 training_set = train_datagen.flow_from_directory('.. /input/train',
target_size =(240, 240),
batch_size = 32,
subset = 'training')
 validation_set = train_datagen.flow_from_directory('.. /input/train',
target_size =(240, 240),
batch_size = 32,
subset = 'validation' )<train_model>
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=9,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
missing=-999,
random_state=2019,
tree_method='gpu_hist' 
)
classifier.fit_generator(training_set,
steps_per_epoch = 17943/32,
epochs = 10,
validation_data = validation_set,
validation_steps = 4481/32 )<load_from_csv>
%time clf.fit(X_train_1, y_train_1 )
test_file = pd.read_csv('.. /input/sample_submission.csv')
 test_file.head(5 )<predict_on_test>
sample_submission['isFraud'] = clf.predict_proba(X_test_1)[:,1]
sample_submission.to_csv('simple_xgboost_2.csv' )
for i, file in enumerate(test_file['img']):
image = get_data('.. /input/test/' + file)
 image = np.reshape(image,(1, image.shape[0], image.shape[1], image.shape[2]))
result = classifier.predict(image)
 test_file.iloc[i, 1:] = result[0]<save_to_csv>
n_fold = 5
folds = TimeSeriesSplit(n_splits=n_fold)
 folds = KFold(n_splits=5 )
test_file.to_csv('results.csv', index = False )<set_options>
def train_model_classification(X, X_test, y, params, folds, model_type='lgb', eval_metric='auc', columns=None, plot_feature_importance=False, model=None,
verbose=10000, early_stopping_rounds=200, n_estimators=50000, splits=None, n_folds=3, averaging='usual', n_jobs=-1):

 columns = X.columns if columns is None else columns
n_splits = folds.n_splits if splits is None else n_folds
X_test = X_test[columns]
metrics_dict = {'auc': {'lgb_metric_name': eval_auc,
'catboost_metric_name': 'AUC',
'sklearn_scoring_function': metrics.roc_auc_score},
}
result_dict = {}
if averaging == 'usual':
oof = np.zeros(( len(X), 1))
prediction = np.zeros(( len(X_test), 1))
elif averaging == 'rank':
oof = np.zeros(( len(X), 1))
prediction = np.zeros(( len(X_test), 1))
scores = []
feature_importance = pd.DataFrame()
for fold_n,(train_index, valid_index)in enumerate(folds.split(X)) :
print(f'Fold {fold_n + 1} started at {time.ctime() }')
 if type(X)== np.ndarray:
X_train, X_valid = X[columns][train_index], X[columns][valid_index]
y_train, y_valid = y[train_index], y[valid_index]
else:
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
if model_type == 'lgb':
model = lgb.LGBMClassifier(**params, n_estimators=n_estimators, n_jobs = n_jobs)
 model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric=metrics_dict[eval_metric]['lgb_metric_name'],
verbose=verbose, early_stopping_rounds=early_stopping_rounds)
 y_pred_valid = model.predict_proba(X_valid)[:, 1]
y_pred = model.predict_proba(X_test, num_iteration=model.best_iteration_)[:, 1]
if model_type == 'xgb':
train_data = xgb.DMatrix(data=X_train, label=y_train, feature_names=X.columns)
 valid_data = xgb.DMatrix(data=X_valid, label=y_valid, feature_names=X.columns)
 watchlist = [(train_data, 'train'),(valid_data, 'valid_data')]
model = xgb.train(dtrain=train_data, num_boost_round=n_estimators, evals=watchlist, early_stopping_rounds=early_stopping_rounds, verbose_eval=verbose, params=params)
 y_pred_valid = model.predict(xgb.DMatrix(X_valid, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 y_pred = model.predict(xgb.DMatrix(X_test, feature_names=X.columns), ntree_limit=model.best_ntree_limit)
 if model_type == 'sklearn':
model = model
model.fit(X_train, y_train)
 y_pred_valid = model.predict(X_valid ).reshape(-1,)
 score = metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid)
 print(f'Fold {fold_n}.{eval_metric}: {score:.4f}.')
 print('')
 y_pred = model.predict_proba(X_test)
 if model_type == 'cat':
model = CatBoostClassifier(iterations=n_estimators, eval_metric=metrics_dict[eval_metric]['catboost_metric_name'], **params,
loss_function=metrics_dict[eval_metric]['catboost_metric_name'])
 model.fit(X_train, y_train, eval_set=(X_valid, y_valid), cat_features=[], use_best_model=True, verbose=False)
 y_pred_valid = model.predict(X_valid)
 y_pred = model.predict(X_test)
 if averaging == 'usual':
oof[valid_index] = y_pred_valid.reshape(-1, 1)
 scores.append(metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid))
prediction += y_pred.reshape(-1, 1)
 elif averaging == 'rank':
oof[valid_index] = y_pred_valid.reshape(-1, 1)
 scores.append(metrics_dict[eval_metric]['sklearn_scoring_function'](y_valid, y_pred_valid))
prediction += pd.Series(y_pred ).rank().values.reshape(-1, 1)
 if model_type == 'lgb' and plot_feature_importance:
fold_importance = pd.DataFrame()
fold_importance["feature"] = columns
fold_importance["importance"] = model.feature_importances_
fold_importance["fold"] = fold_n + 1
feature_importance = pd.concat([feature_importance, fold_importance], axis=0)
 prediction /= n_splits
print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)))
 result_dict['oof'] = oof
result_dict['prediction'] = prediction
result_dict['scores'] = scores
if model_type == 'lgb':
if plot_feature_importance:
feature_importance["importance"] /= n_splits
cols = feature_importance[["feature", "importance"]].groupby("feature" ).mean().sort_values(
 by="importance", ascending=False)[:50].index
best_features = feature_importance.loc[feature_importance.feature.isin(cols)]
plt.figure(figsize=(16, 12)) ;
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False)) ;
plt.title('LGB Features(avg over folds)');
result_dict['feature_importance'] = feature_importance
result_dict['top_columns'] = cols
return result_dict
warnings.simplefilter('error', SettingWithCopyWarning)
 gc.enable()
%matplotlib inline<load_from_csv>
def eval_auc(y_true, y_pred):

 return 'auc', fast_auc(y_true, y_pred), True
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz',
dtype={'date': str, 'fullVisitorId': str, 'sessionId':str}, nrows=None)
 train.shape, test.shape<prepare_x_and_y>
def fast_auc(y_true, y_prob):

 y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
nfalse = 0
auc = 0
n = len(y_true)
 for i in range(n):
y_i = y_true[i]
nfalse +=(1 - y_i)
 auc += y_i * nfalse
auc /=(nfalse *(n - nfalse))
return auc
y_reg = train['totals.transactionRevenue'].fillna(0)
 del train['totals.transactionRevenue']
if 'totals.transactionRevenue' in test.columns:
del test['totals.transactionRevenue']<feature_engineering>
params = {'num_leaves': 256,
'min_child_samples': 79,
'objective': 'binary',
'max_depth': 13,
'learning_rate': 0.03,
"boosting_type": "gbdt",
"subsample_freq": 3,
"subsample": 0.9,
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3,
'reg_lambda': 0.3,
'colsample_bytree': 0.9,
}
result_dict_lgb = train_model_classification(X=X_train, X_test=X_test, y=y_train, params=params, folds=folds, model_type='lgb', eval_metric='auc', plot_feature_importance=True,
verbose=500, early_stopping_rounds=200, n_estimators=5000, averaging='usual', n_jobs=-1 )