Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<data_type_conversions>
warnings.filterwarnings("ignore" )
def addNewFeatures(data):
data['uid'] = data['card1'].astype(str)+'_'+data['card2'].astype(str)
 data['uid2'] = data['uid'].astype(str)+'_'+data['card3'].astype(str)+'_'+data['card5'].astype(str)
 data['uid3'] = data['uid2'].astype(str)+'_'+data['addr1'].astype(str)+'_'+data['addr2'].astype(str)
 data['D9'] = np.where(data['D9'].isna() ,0,1)
 return data
train = addNewFeatures(train)
 test = addNewFeatures(test )<data_type_conversions>
def seed_everything(seed=0):
random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 np.random.seed(seed )
i_cols = ['card1','card2','card3','card5','uid','uid2','uid3']
for col in i_cols:
for agg_type in ['mean','std']:
new_col_name = col+'_TransactionAmt_'+agg_type
temp_df = pd.concat([train[[col, 'TransactionAmt']], test[[col,'TransactionAmt']]])
 temp_df = temp_df.groupby([col])['TransactionAmt'].agg([agg_type] ).reset_index().rename(
 columns={agg_type: new_col_name})
 temp_df.index = list(temp_df[col])
 temp_df = temp_df[new_col_name].to_dict()
train[new_col_name] = train[col].map(temp_df)
 test[new_col_name] = test[col].map(temp_df)
 train = train.replace(np.inf,999)
 test = test.replace(np.inf,999 )<feature_engineering>
SEED = 1337
seed_everything(SEED )
train['TransactionAmt'] = np.log1p(train['TransactionAmt'])
 test['TransactionAmt'] = np.log1p(test['TransactionAmt'] )<define_variables>
def reduce_memory_usage(df):
numerics = ["int16", "int32", "int64", "float64"]
for col, col_type in df.dtypes.iteritems() :
best_type = None
if col_type == "object":
df[col] = df[col].astype("category")
 best_type = "category"
elif col_type in numerics:
downcast = "integer" if "int" in str(col_type)else "float"
df[col] = pd.to_numeric(df[col], downcast=downcast)
 best_type = df[col].dtype.name
return df
emails = {'gmail': 'google', 'att.net': 'att', 'twc.com': 'spectrum', 'scranton.edu': 'other', 'optonline.net': 'other',
'hotmail.co.uk': 'microsoft', 'comcast.net': 'other', 'yahoo.com.mx': 'yahoo', 'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo', 'charter.net': 'spectrum', 'live.com': 'microsoft', 'aim.com': 'aol', 'hotmail.de': 'microsoft',
'centurylink.net': 'centurylink', 'gmail.com': 'google', 'me.com': 'apple', 'earthlink.net': 'other',
'gmx.de': 'other', 'web.de': 'other', 'cfl.rr.com': 'other', 'hotmail.com': 'microsoft', 'protonmail.com': 'other',
'hotmail.fr': 'microsoft', 'windstream.net': 'other', 'outlook.es': 'microsoft', 'yahoo.co.jp': 'yahoo',
'yahoo.de': 'yahoo', 'servicios-ta.com': 'other', 'netzero.net': 'other', 'suddenlink.net': 'other',
'roadrunner.com': 'other', 'sc.rr.com': 'other', 'live.fr': 'microsoft', 'verizon.net': 'yahoo',
'msn.com': 'microsoft', 'q.com': 'centurylink', 'prodigy.net.mx': 'att', 'frontier.com': 'yahoo',
'anonymous.com': 'other', 'rocketmail.com': 'yahoo', 'sbcglobal.net': 'att', 'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo', 'outlook.com': 'microsoft', 'mail.com': 'other', 'bellsouth.net': 'other',
'embarqmail.com': 'centurylink', 'cableone.net': 'other', 'hotmail.es': 'microsoft', 'mac.com': 'apple',
'yahoo.co.uk': 'yahoo', 'netzero.com': 'other', 'yahoo.com': 'yahoo', 'live.com.mx': 'microsoft', 'ptd.net': 'other',
'cox.net': 'other', 'aol.com': 'aol', 'juno.com': 'other', 'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )<feature_engineering>
%%time
train = pd.read_pickle(".. /input/ieeecis-fraud-detection-merged/train.pkl")
 test = pd.read_pickle(".. /input/ieeecis-fraud-detection-merged/test.pkl" )
p = 'P_emaildomain'
r = 'R_emaildomain'
uknown = 'email_not_provided'
def setDomain(df):
df[p] = df[p].fillna(uknown)
 df[r] = df[r].fillna(uknown)
 df['email_check'] = np.where(( df[p]==df[r])&(df[p]!=uknown),1,0)
 df[p+'_prefix'] = df[p].apply(lambda x: x.split('.')[0])
 df[r+'_prefix'] = df[r].apply(lambda x: x.split('.')[0])
 return df
train=setDomain(train)
 test=setDomain(test )<feature_engineering>
%%time
for df in [train, test]:
df["day"] = np.floor(( df["TransactionDT"] /(3600 * 24)- 1)% 7 ).astype(int)
 df["hour"] =(np.floor(df["TransactionDT"] / 3600)% 24 ).astype(int)
 df["TransactionAmt_log"] = np.log(df["TransactionAmt"])
 df["TransactionAmt_int"] = df["TransactionAmt"].astype(int)
 df["TransactionAmt_dec"] =(1000 *(df["TransactionAmt"] - df["TransactionAmt_int"])).astype(int)
 df["card_id"] = df["card1"].astype(str)+ "__" + df["card2"].astype(str)+ "__" + df["card3"].astype(str)+ "__" + df["card4"].astype(str)+ "__" + df["card5"].astype(str)
 for a, b in itertools.combinations(["card1", "card2", "card3", "card4", "card5", "addr1", "addr2", "dist1", "dist2"], 2):
df["{}__{}".format(a, b)] = df[a].astype(str)+ "__" + df[b].astype(str)
 df[["P_emaildomain_0", "P_emaildomain_1", "P_emaildomain_2"]] = df["P_emaildomain"].str.split(".", expand=True)
 df[["R_emaildomain_0", "R_emaildomain_1", "R_emaildomain_2"]] = df["R_emaildomain"].str.split(".", expand=True)
 df[["operating_system_0", "operating_system_1", "operating_system_2", "operating_system_3"]] = df["id_30"].str.split(" ", expand=True)
 df[["browser_0", "browser_1", "browser_2", "browser_3"]] = df["id_31"].str.split(" ", expand=True)
 df[["resolution_width", "resolution_height"]] = df["id_33"].str.split("x", expand=True)
 df[["DeviceInfo_0", "DeviceInfo_1", "DeviceInfo_2", "DeviceInfo_3", "DeviceInfo_4"]] = df["DeviceInfo"].str.split(r"[ -/_]", expand=True)[[0, 1, 2, 3, 4]]
df["TransactionAmt_to_mean_card1"] = df["TransactionAmt"] / df.groupby(["card1"])["TransactionAmt"].transform("mean")
 df["TransactionAmt_to_mean_card4"] = df["TransactionAmt"] / df.groupby(["card4"])["TransactionAmt"].transform("mean")
 df["TransactionAmt_to_std_card1"] = df["TransactionAmt"] / df.groupby(["card1"])["TransactionAmt"].transform("std")
 df["TransactionAmt_to_std_card4"] = df["TransactionAmt"] / df.groupby(["card4"])["TransactionAmt"].transform("std")
 df["id_02_to_mean_card1"] = df["id_02"] / df.groupby(["card1"])["id_02"].transform("mean")
 df["id_02_to_mean_card4"] = df["id_02"] / df.groupby(["card4"])["id_02"].transform("mean")
 df["id_02_to_std_card1"] = df["id_02"] / df.groupby(["card1"])["id_02"].transform("std")
 df["id_02_to_std_card4"] = df["id_02"] / df.groupby(["card4"])["id_02"].transform("std")
 df["D15_to_mean_card1"] = df["D15"] / df.groupby(["card1"])["D15"].transform("mean")
 df["D15_to_mean_card4"] = df["D15"] / df.groupby(["card4"])["D15"].transform("mean")
 df["D15_to_std_card1"] = df["D15"] / df.groupby(["card1"])["D15"].transform("std")
 df["D15_to_std_card4"] = df["D15"] / df.groupby(["card4"])["D15"].transform("std")
 df["D15_to_mean_addr1"] = df["D15"] / df.groupby(["addr1"])["D15"].transform("mean")
 df["D15_to_mean_addr2"] = df["D15"] / df.groupby(["addr2"])["D15"].transform("mean")
 df["D15_to_std_addr1"] = df["D15"] / df.groupby(["addr1"])["D15"].transform("std")
 df["D15_to_std_addr2"] = df["D15"] / df.groupby(["addr2"])["D15"].transform("std")
 df["device_manufacturer"] = df["DeviceInfo"].map(lambda x: get_device_manufacturer(x))
def setTime(df):
df['TransactionDT'] = df['TransactionDT'].fillna(df['TransactionDT'].median())
 df['DT'] = df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 df['DT_M'] =(df['DT'].dt.year-2017)*12 + df['DT'].dt.month
df['DT_W'] =(df['DT'].dt.year-2017)*52 + df['DT'].dt.weekofyear
df['DT_D'] =(df['DT'].dt.year-2017)*365 + df['DT'].dt.dayofyear
df['DT_hour'] = df['DT'].dt.hour
df['DT_day_week'] = df['DT'].dt.dayofweek
df['DT_day'] = df['DT'].dt.day
return df
train=setTime(train)
 test=setTime(test )<feature_engineering>
default_categorical_columns = ["ProductCD", "card1", "card2", "card3", "card4", "card5", "card6", "addr1", "addr2", "P_emaildomain", "R_emaildomain", "M1", "M2", "M3", "M4", "M5", "M6", "M7", "M8", "M9", "id_12", "id_13", "id_14", "id_15", "id_16", "id_17", "id_18", "id_19", "id_20", "id_21", "id_22", "id_23", "id_24", "id_25", "id_26", "id_27", "id_28", "id_29", "id_30", "id_31", "id_32", "id_33", "id_34", "id_35", "id_36", "id_37", "id_38", "DeviceType", "DeviceInfo"]
train["lastest_browser"] = np.zeros(train.shape[0])
 test["lastest_browser"] = np.zeros(test.shape[0])
 def setBrowser(df):
df.loc[df["id_31"]=="samsung browser 7.0",'lastest_browser']=1
df.loc[df["id_31"]=="opera 53.0",'lastest_browser']=1
df.loc[df["id_31"]=="mobile safari 10.0",'lastest_browser']=1
df.loc[df["id_31"]=="google search application 49.0",'lastest_browser']=1
df.loc[df["id_31"]=="firefox 60.0",'lastest_browser']=1
df.loc[df["id_31"]=="edge 17.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 69.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 67.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 63.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 63.0 for ios",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 64.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 64.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 64.0 for ios",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 65.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 65.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 65.0 for ios",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 66.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 66.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 66.0 for ios",'lastest_browser']=1
return df
train=setBrowser(train)
 test=setBrowser(test )<feature_engineering>
for c in tqdm(set(default_categorical_columns + [c for c in train.columns if train[c].dtype == "object"])) :
labels = {value: index for index, value in enumerate(["__UNKNOWN__"] + list(set(list(train[c].astype(str ).values)+ list(test[c].astype(str ).values)))) }
train[c] = train[c].map(lambda x: labels.get(x)).fillna(labels["__UNKNOWN__"] ).astype(int)
 test[c] = test[c].map(lambda x: labels.get(x)).fillna(labels["__UNKNOWN__"] ).astype(int )
def setDevice(df):
df['DeviceInfo'] = df['DeviceInfo'].fillna('unknown_device' ).str.lower()
df['device_name'] = df['DeviceInfo'].str.split('/', expand=True)[0]
df.loc[df['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
df.loc[df['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
df.loc[df['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
df.loc[df['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
df.loc[df['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
df.loc[df['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
df.loc[df['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
df.loc[df['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
df['had_id'] = 1
gc.collect()
return df
train=setDevice(train)
 test=setDevice(test )<data_type_conversions>
%%time
train = reduce_memory_usage(train)
 test = reduce_memory_usage(test )
i_cols = ['card1','card2','card3','card5',
'C1','C2','C3','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14',
'D1','D2','D3','D4','D5','D6','D7','D8',
'addr1','addr2',
'dist1','dist2',
'P_emaildomain', 'R_emaildomain',
'DeviceInfo','device_name',
'id_30','id_33',
'uid','uid2','uid3',
]
for col in i_cols:
temp_df = pd.concat([train[[col]], test[[col]]])
 fq_encode = temp_df[col].value_counts(dropna=False ).to_dict()
train[col+'_fq_enc'] = train[col].map(fq_encode)
 test[col+'_fq_enc'] = test[col].map(fq_encode)
 for col in ['DT_M','DT_W','DT_D']:
temp_df = pd.concat([train[[col]], test[[col]]])
 fq_encode = temp_df[col].value_counts().to_dict()
train[col+'_total'] = train[col].map(fq_encode)
 test[col+'_total'] = test[col].map(fq_encode)
 periods = ['DT_M','DT_W','DT_D']
i_cols = ['uid']
for period in periods:
for col in i_cols:
new_column = col + '_' + period
temp_df = pd.concat([train[[col,period]], test[[col,period]]])
 temp_df[new_column] = temp_df[col].astype(str)+ '_' +(temp_df[period] ).astype(str)
 fq_encode = temp_df[new_column].value_counts().to_dict()
train[new_column] =(train[col].astype(str)+ '_' + train[period].astype(str)).map(fq_encode)
 test[new_column] =(test[col].astype(str)+ '_' + test[period].astype(str)).map(fq_encode)
 train[new_column] /= train[period+'_total']
test[new_column] /= test[period+'_total']<statistical_test>
lgb_params = {
"boosting_type": "gbdt",
"colsample_bytree": 0.5,
"early_stopping_rounds": 100,
"learning_rate": 0.005,
"max_bin": 255,
"max_depth": -1,
"metric": "auc",
"n_estimators": 20000,
"n_jobs": -1,
"num_leaves": 2**9,
"objective": "binary",
"seed": SEED,
"subsample": 0.7,
"subsample_freq": 1,
"tree_learner": "serial"
}
def get_too_many_null_attr(data):
many_null_cols = [col for col in data.columns if data[col].isnull().sum() / data.shape[0] > 0.9]
return many_null_cols
def get_too_many_repeated_val(data):
big_top_value_cols = [col for col in train.columns if train[col].value_counts(dropna=False, normalize=True ).values[0] > 0.9]
return big_top_value_cols
def get_useless_columns(data):
too_many_null = get_too_many_null_attr(data)
 print("More than 90% null: " + str(len(too_many_null)))
 too_many_repeated = get_too_many_repeated_val(data)
 print("More than 90% repeated value: " + str(len(too_many_repeated)))
 cols_to_drop = list(set(too_many_null + too_many_repeated))
return cols_to_drop<drop_column>
feature_columns = list(train.columns)
 target_column = "isFraud"
columns_to_drop = ["TransactionID", "isFraud", "TransactionDT"]
for c in columns_to_drop:
feature_columns.remove(c )
train = train.drop(cols_to_drop, axis=1)
 test = test.drop(cols_to_drop, axis=1 )<categorify>
X, y = train[feature_columns], train[target_column]
P = test[feature_columns]
for f in train.columns:
if train[f].dtype.name =='object' or test[f].dtype.name =='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train[f].values)+ list(test[f].values))
train[f] = lbl.transform(list(train[f].values))
test[f] = lbl.transform(list(test[f].values))<categorify>
%%time
N_SPLITS = 5
folds = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)
 train_preds = np.zeros(train.shape[0])
 test_preds = np.zeros(test.shape[0])
 del train, test
gc.collect()
feature_importance = pd.DataFrame()
feature_importance["Feature"] = X.columns
feature_importance["Value"] = 0
for fold_idx,(train_idx, valid_idx)in enumerate(folds.split(X, y)) :
print("Fold", fold_idx + 1)
 train_x, train_y = X.iloc[train_idx, :], y[train_idx]
valid_x, valid_y = X.iloc[valid_idx, :], y[valid_idx]
model = lgb.LGBMClassifier(**lgb_params)
 model.fit(train_x, train_y, eval_set=[(train_x, train_y),(valid_x, valid_y)], verbose=100)
 train_preds[valid_idx] = model.predict_proba(valid_x, num_iteration=model.best_iteration_)[:, 1]
test_preds += model.predict_proba(P, num_iteration=model.best_iteration_)[:, 1] / folds.n_splits
print("Fold {} AUC: {:.6f}".format(fold_idx + 1, roc_auc_score(valid_y, train_preds[valid_idx])))
 current_importance = pd.DataFrame(zip(X.columns, model.feature_importances_), columns=["Feature", "Value"])
 feature_importance = pd.concat(( feature_importance, current_importance)).groupby("Feature", as_index=False ).sum()
print("Global AUC: {:.6f}".format(roc_auc_score(y, train_preds)) )
train = train.fillna(-999)
 test = test.fillna(-999 )<count_missing_values>
submission = pd.read_csv(".. /input/ieee-fraud-detection/sample_submission.csv")
 submission[target_column] = test_preds
submission.to_csv("submission.csv", index=False )
print(train.isnull().sum().max())
 print(test.isnull().sum().max() )<drop_column>
feature_importance.sort_values("Value", ascending=False ).to_csv("feature_importance.csv", index=False )
X = train.drop(['TransactionDT', 'DT'], axis=1)
 X_test = test.drop(['TransactionDT', 'DT'], axis=1 )<train_model>
feature_importance.sort_values("Value", ascending=False)[["Feature"]].to_csv("feature_importance_name_only.csv", index=False )