Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<groupby>
warnings.filterwarnings('ignore' )
cdf = tdf.loc[tdf[id_col].isin(cids)].groupby(id_col)['target'].sum()
print(cdf.shape)
 cdf<feature_engineering>
train_transaction = pd.read_csv('.. /input/ieee-fraud-detection/train_transaction.csv')
 train_identity = pd.read_csv('.. /input/ieee-fraud-detection/train_identity.csv')
 test_transaction = pd.read_csv('.. /input/ieee-fraud-detection/test_transaction.csv')
 test_identity = pd.read_csv('.. /input/ieee-fraud-detection/test_identity.csv' )
sub = pd.read_csv('.. /input/gs-lgb-uv-ensemble-1-1/lgb_uv_ensemble_1_1.csv')
 print(sub.shape)
 sub1 = sub.join(cdf, on=id_col)
 ind = sub1.target.notnull()
sub1.loc[ind, 'PredictedLogRevenue'] =(sub1.loc[ind, 'PredictedLogRevenue']+np.log1p(sub1.loc[ind, 'target'])) /2
submission = sub1.drop('target', axis=1)
 submission<save_to_csv>
df_train = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')
 df_test = pd.merge(test_transaction, test_identity, on='TransactionID', how='left' )
submission.to_csv('test_all.csv', index=False, float_format='%.5f' )<set_options>
for df in [train_transaction, train_identity, test_transaction, test_identity]:
del df
!pip install git+https://github.com/fastai/fastai@e85667cfae2e6873b1bb026195b5d09a74dfcff9
%load_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
drop_col = ['V300','V309','V111','V124','V106','V125','V315','V134','V102','V123','V316','V113',
'V136','V305','V110','V299','V289','V286','V318','V103','V304','V116','V29','V284','V293',
'V137','V295','V301','V104','V311','V115','V109','V119','V321','V114','V133','V122','V319',
'V105','V112','V118','V117','V121','V108','V135','V320','V303','V297','V120',
'V1','V14','V41','V65','V88','V107']
for df in [df_train, df_test]:
df = df.drop(drop_col, axis=1 )
from fastai.imports import *
from fastai.structured import *
from pandas_summary import DataFrameSummary
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from IPython.display import display
from sklearn import metrics<load_from_csv>
def make_time_feature(df):
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d')
 df['DT'] = df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 df['DT_M'] =(( df['DT'].dt.year-2017)*12 + df['DT'].dt.month ).astype(np.int8)
 df['DT_W'] =(( df['DT'].dt.year-2017)*52 + df['DT'].dt.weekofyear ).astype(np.int8)
 df['DT_D'] =(( df['DT'].dt.year-2017)*365 + df['DT'].dt.dayofyear ).astype(np.int16)
 df['DT_hour'] =(df['DT'].dt.hour ).astype(np.int8)
 df['DT_day_week'] =(df['DT'].dt.dayofweek ).astype(np.int8)
 df['DT_day_month'] =(df['DT'].dt.day ).astype(np.int8)
 df['is_holiday'] =(df['DT'].dt.date.astype('datetime64' ).isin(us_holidays)).astype(np.int8)
 return df
PATH = ".. /input/"
def display_all(df):
with pd.option_context("display.max_rows", 1000, "display.max_columns", 1000):
display(df)
 def load_df(csv_path, nrows=None):
JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']
types = {
"fullVisitorId": "str",
"channelGrouping": "str",
"date": "str",
"socialEngagementType": "str",
"visitId": "int32",
"visitNumber": "int8",
"visitStartTime": "int32",
}
df = pd.read_csv(csv_path,
converters={column: json.loads for column in JSON_COLUMNS},
dtype=types,
nrows=nrows)
 for column in JSON_COLUMNS:
column_as_df = json_normalize(df[column])
 column_as_df.columns = [f"{column}.{subcolumn}" for subcolumn in column_as_df.columns]
df = df.drop(column, axis=1 ).merge(column_as_df, right_index=True, left_index=True)
 print(f"Loaded {os.path.basename(csv_path)}.Shape: {df.shape}")
 return df<compute_train_metric>
for df in [df_train, df_test]:
df = make_time_feature(df=df )
def split_vals(a,n):
return a[:n].copy() , a[n:].copy()
def rmse(x,y):
return math.sqrt(((x-y)**2 ).mean())
 def print_score(m):
res = [rmse(m.predict(X_train), y_train), rmse(m.predict(X_valid), y_valid),
m.score(X_train, y_train), m.score(X_valid, y_valid)]
if hasattr(m, 'oob_score_'): res.append(m.oob_score_)
 print(res )<load_from_csv>
df_deviceinfo = pd.read_csv('.. /input/ieee-deviceinfo/DeviceInfo.csv' )
df_raw = load_df(f'{PATH}train_v2.csv', nrows=500_000 )<count_unique_values>
df_train = pd.merge(df_train, df_deviceinfo, on='DeviceInfo', how='left')
 df_test = pd.merge(df_test, df_deviceinfo, on='DeviceInfo', how='left' )
const_cols = [c for c in df_raw.columns if df_raw[c].nunique(dropna=False)==1]
df_raw.drop(columns=const_cols + ["customDimensions", "hits"], inplace=True )<load_from_csv>
del df_deviceinfo
df_test = load_df(f'{PATH}test_v2.csv' )<drop_column>
for df in [df_train, df_test]:
df['OS'] = df['id_30'].apply(lambda x: str(x ).split(' ')[0])
 df['OS_version'] = df['id_30'].apply(lambda x: str(x ).split(' ')[-1])
 df['OS_version'] = df['OS_version'].apply(lambda x: str(x ).replace('_', '' ).replace('.', ''))
df['OS_version'] = df['OS_version'].apply(lambda x: int(x)if x.isdigit() else np.nan )
df_raw.drop(columns=['trafficSource.campaignCode'], inplace=True )<drop_column>
browser_release_date = {'android browser 4.0' : '2011-10-18',
'android webview 4.0' : '2011-10-18',
'chrome 43.0 for android' : '2015-04-15',
'chrome 46.0 for android' : '2015-10-14',
'chrome 49.0' : '2016-03-02',
'chrome 49.0 for android' : '2016-03-09',
'chrome 50.0 for android' : '2016-04-26',
'chrome 51.0' : '2016-05-25',
'chrome 51.0 for android' : '2016-06-01',
'chrome 52.0 for android' : '2016-07-27',
'chrome 53.0 for android' : '2016-09-07',
'chrome 54.0 for android' : '2016-10-19',
'chrome 55.0' : '2016-12-01',
'chrome 55.0 for android' : '2016-12-06',
'chrome 56.0' : '2017-01-25',
'chrome 56.0 for android' : '2017-02-01',
'chrome 57.0' : '2017-03-09',
'chrome 57.0 for android' : '2017-03-16',
'chrome 58.0' : '2017-04-19',
'chrome 58.0 for android' : '2017-04-20',
'chrome 59.0' : '2017-06-05',
'chrome 59.0 for android' : '2017-06-06',
'chrome 60.0' : '2017-07-25',
'chrome 60.0 for android' : '2017-07-31',
'chrome 61.0' : '2017-09-05',
'chrome 61.0 for android' : '2017-09-05',
'chrome 62.0' : '2017-10-17',
'chrome 62.0 for android' : '2017-10-19',
'chrome 62.0 for ios' : '2017-10-18',
'chrome 63.0' : '2017-12-06',
'chrome 63.0 for android' : '2017-12-05',
'chrome 63.0 for ios' : '2017-12-05',
'chrome 64.0' : '2018-01-24',
'chrome 64.0 for android' : '2018-01-23',
'chrome 64.0 for ios' : '2018-01-24',
'chrome 65.0' : '2018-03-06',
'chrome 65.0 for android' : '2018-03-06',
'chrome 65.0 for ios' : '2018-03-06',
'chrome 66.0' : '2018-04-17',
'chrome 66.0 for android' : '2018-04-17',
'chrome 66.0 for ios' : '2018-04-17',
'chrome 67.0' : '2018-05-29',
'chrome 67.0 for android' : '2018-05-31',
'chrome 69.0' : '2018-09-04',
'edge 13.0' : '2015-09-15',
'edge 14.0' : '2016-02-18',
'edge 15.0' : '2016-10-07',
'edge 16.0' : '2017-09-26',
'edge 17.0' : '2018-04-30',
'firefox 47.0' : '2016-06-07',
'firefox 48.0' : '2016-08-02',
'firefox 52.0' : '2017-03-07',
'firefox 55.0' : '2017-08-08',
'firefox 56.0' : '2017-09-28',
'firefox 57.0' : '2017-11-14',
'firefox 58.0' : '2018-01-23',
'firefox 59.0' : '2018-03-13',
'firefox 60.0' : '2018-05-09',
'firefox mobile 61.0' : '2018-06-26',
'google search application 48.0' : '2016-01-20',
'google search application 49.0' : '2016-03-02',
'ie 11.0 for desktop' : '2013-10-17',
'ie 11.0 for tablet' : '2013-10-17',
'mobile safari 10.0' : '2016-09-14',
'mobile safari 11.0' : '2017-09-20',
'mobile safari 8.0' : '2014-09-17',
'mobile safari 9.0' : '2015-09-16',
'opera 49.0' : '2017-11-08',
'opera 51.0' : '2018-02-07',
'opera 52.0' : '2018-03-14',
'opera 53.0' : '2018-05-10',
'safari 10.0' : '2016-09-20',
'safari 11.0' : '2017-09-19',
'safari 9.0' : '2015-09-30',
'samsung browser 3.3' : '2015-08-01',
'samsung browser 4.0' : '2016-03-11',
'samsung browser 4.2' : '2016-08-19',
'samsung browser 5.2' : '2016-11-01',
'samsung browser 5.4' : '2017-05-01',
'samsung browser 6.2' : '2017-08-01',
'samsung browser 6.4' : '2018-02-19',
'samsung browser 7.0' : '2018-03-01',
}
df_test.drop(columns=const_cols + ["customDimensions", "hits"], inplace=True )<data_type_conversions>
def time_split(val):
try:
return datetime.datetime.strptime(str(x), '%Y-%m-%d')
 except:
return pd.NaT
df_raw['totals.transactionRevenue'].fillna(0, inplace=True)
 df_raw['totals.transactionRevenue'] = df_raw['totals.transactionRevenue'].astype(float)
 df_raw['totals.transactionRevenue'] = np.log1p(df_raw['totals.transactionRevenue'] )<data_type_conversions>
for df in [df_train, df_test]:
df['browser_elapsed_time'] = df['id_31'].map(browser_release_date)
 df['browser_elapsed_time'].apply(lambda x: time_split(x))
df['browser_elapsed_time'] = df['DT'] - df['browser_elapsed_time'].astype('datetime64[D]')
 df['browser_elapsed_time'] = df['browser_elapsed_time'].dt.days
del df['DT']
df_raw["visitStartTime"] = pd.to_datetime(df_raw["visitStartTime"], infer_datetime_format=True, unit="s")
 df_raw["date"] = pd.to_datetime(df_raw["date"], infer_datetime_format=True, format="%Y%m%d")
 add_datepart(df_raw, 'date')
 add_datepart(df_raw, 'visitStartTime' )<data_type_conversions>
emails = {'gmail': 'google', 'att.net': 'att', 'twc.com': 'spectrum',
'scranton.edu': 'other', 'optonline.net': 'other', 'hotmail.co.uk': 'microsoft',
'comcast.net': 'other', 'yahoo.com.mx': 'yahoo', 'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo', 'charter.net': 'spectrum', 'live.com': 'microsoft',
'aim.com': 'aol', 'hotmail.de': 'microsoft', 'centurylink.net': 'centurylink',
'gmail.com': 'google', 'me.com': 'apple', 'earthlink.net': 'other', 'gmx.de': 'other',
'web.de': 'other', 'cfl.rr.com': 'other', 'hotmail.com': 'microsoft',
'protonmail.com': 'other', 'hotmail.fr': 'microsoft', 'windstream.net': 'other',
'outlook.es': 'microsoft', 'yahoo.co.jp': 'yahoo', 'yahoo.de': 'yahoo',
'servicios-ta.com': 'other', 'netzero.net': 'other', 'suddenlink.net': 'other',
'roadrunner.com': 'other', 'sc.rr.com': 'other', 'live.fr': 'microsoft',
'verizon.net': 'yahoo', 'msn.com': 'microsoft', 'q.com': 'centurylink',
'prodigy.net.mx': 'att', 'frontier.com': 'yahoo', 'anonymous.com': 'other',
'rocketmail.com': 'yahoo', 'sbcglobal.net': 'att', 'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo', 'outlook.com': 'microsoft', 'mail.com': 'other',
'bellsouth.net': 'other', 'embarqmail.com': 'centurylink', 'cableone.net': 'other',
'hotmail.es': 'microsoft', 'mac.com': 'apple', 'yahoo.co.uk': 'yahoo', 'netzero.com': 'other',
'yahoo.com': 'yahoo', 'live.com.mx': 'microsoft', 'ptd.net': 'other', 'cox.net': 'other',
'aol.com': 'aol', 'juno.com': 'other', 'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']
df_raw['totals.totalTransactionRevenue'].fillna(0, inplace=True)
 df_raw['totals.totalTransactionRevenue'] = df_raw['totals.totalTransactionRevenue'].astype('int32')
 df_raw['totals.transactions'].fillna(0, inplace=True)
 df_raw['totals.transactions'] = df_raw['totals.transactions'].astype('int8' )<data_type_conversions>
def make_email_feature(df):
for c in ['P_emaildomain', 'R_emaildomain']:
df[c + '_bin'] = df[c].map(emails)
 df[c + '_suffix'] = df[c].map(lambda x: str(x ).split('.')[-1])
 df[c + '_suffix'] = df[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )
df_test['totals.transactionRevenue'].fillna(0, inplace=True)
 df_test['totals.transactionRevenue'] = df_test['totals.transactionRevenue'].astype(float)
 df_test['totals.transactionRevenue'] = np.log1p(df_test['totals.transactionRevenue'] )<data_type_conversions>
for df in [df_train, df_test]:
df = make_email_feature(df=df )
df_test["visitStartTime"] = pd.to_datetime(df_test["visitStartTime"], infer_datetime_format=True, unit="s")
 df_test["date"] = pd.to_datetime(df_test["date"], infer_datetime_format=True, format="%Y%m%d")
 add_datepart(df_test, 'date')
 add_datepart(df_test, 'visitStartTime' )<concatenate>
def siev_card_feature(df_train, df_test, col):
valid_card = pd.concat([df_train[[col]], df_test[[col]]])
 valid_card = valid_card[col].value_counts()
valid_card_std = valid_card.values.std()
invalid_cards = valid_card[valid_card < 2]
print('{0}{1}
Number of Rare card : '.format(col, '*'*10), len(invalid_cards))
valid_card = valid_card[valid_card >= 2]
valid_card = list(valid_card.index)
 print('Number of not intersection in Train : ', len(df_train[~df_train[col].isin(df_test[col])]))
print('Nmber of intersection in Train : ', len(df_train[df_train[col].isin(df_test[col])]))
df_train[col] = np.where(df_train[col].isin(df_test[col]), df_train[col], np.nan)
 df_test[col] = np.where(df_test[col].isin(df_train[col]), df_test[col], np.nan)
 df_train[col] = np.where(df_train[col].isin(valid_card), df_train[col], np.nan)
 df_test[col] = np.where(df_test[col].isin(valid_card), df_test[col], np.nan )
train_cats(df_raw)
 train_cats(df_test )<prepare_x_and_y>
for col in ['card1','card2','card3','card4','card5','card6']:
siev_card_feature(df_train, df_test, col )
df_trn, y_trn, nas = proc_df(df_raw, 'totals.transactionRevenue' )<train_model>
def make_composite_id(new_id, id1, id2):
df_train[new_id] = df_train[id1].astype(str)+'_' + df_train[id2].astype(str)
 df_test[new_id] = df_test[id1].astype(str)+'_' + df_test[id2].astype(str )
m = RandomForestRegressor(n_jobs=-1)
 m.fit(df_trn, y_trn)
 m.score(df_trn, y_trn )<split>
make_composite_id('uid1', 'card1', 'card2')
 make_composite_id('uid2', 'uid1', 'card3')
 make_composite_id('uid2', 'uid2', 'card5')
 make_composite_id('uid3', 'uid2', 'addr1')
 make_composite_id('uid3', 'uid3', 'addr2')
 make_composite_id('uid4', 'uid3', 'P_emaildomain')
 make_composite_id('uid5', 'uid3', 'R_emaildomain')
 make_composite_id('bank_type', 'card3', 'card5' )
train_required_ratio = 0.8
n_trn = int(len(df_trn)* train_required_ratio)
 X_train, X_valid = split_vals(df_trn, n_trn)
 y_train, y_valid = split_vals(y_trn, n_trn)
 X_train.shape, X_valid.shape<train_model>
gc.collect()
m = RandomForestRegressor(n_estimators=10, n_jobs=-1)
 m.fit(X_train, y_train)
 print_score(m )<predict_on_test>
for df in [df_train, df_test]:
df = reduce_mem_usage(df=df )
preds = np.stack([t.predict(X_valid)for t in m.estimators_])
 preds[:,0], np.mean(preds[:,0]), y_valid[0]
preds.shape<compute_test_metric>
def create_new_columns(key, aggs):
return [key + '_' + k + '_' + agg for k in aggs.keys() for agg in aggs[k]]
plt.plot([metrics.r2_score(y_valid, np.mean(preds[:i+1], axis=0)) for i in range(10)]);<train_model>
aggs = {}
for col in ['DT_D','DT_W','DT_M']:
aggs[col] = ['nunique', 'mean', 'median', 'var', 'skew']
aggs
m = RandomForestRegressor(n_estimators=40, n_jobs=-1, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )<set_options>
df_temp = pd.concat([df_train, df_test])
 for key in tqdm_notebook(['card3', 'card5', 'bank_type']):
new_columns = create_new_columns(key, aggs)
 df_grouped = df_temp.groupby(key ).agg(aggs)
 df_grouped.columns = new_columns
df_grouped.reset_index(drop=False,inplace=True)
 df_train = df_train.merge(df_grouped,on=key,how='left')
 df_test = df_test.merge(df_grouped,on=key,how='left' )
set_rf_samples(50_000 )<train_model>
aggs = {}
for col in ['TransactionAmt', 'D15']:
aggs[col] = ['nunique', 'mean', 'median', 'var', 'skew', 'min', 'max']
aggs
m = RandomForestRegressor(n_estimators=40, n_jobs=-1)
 m.fit(X_train, y_train)
 print_score(m )<train_model>
for key in tqdm_notebook(['card1','card2','card3','card5','uid1','uid2','uid3','uid4','uid5','bank_type']):
new_columns = create_new_columns(key, aggs)
 df_grouped = df_temp.groupby(key ).agg(aggs)
 df_grouped.columns = new_columns
df_grouped.reset_index(drop=False,inplace=True)
 df_train = df_train.merge(df_grouped,on=key,how='left')
 df_test = df_test.merge(df_grouped,on=key,how='left' )
m = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, n_jobs=-1, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )<split>
aggs = {}
for col in ['TransactionAmt', 'D15']:
aggs[col] = ['mean', 'median', 'min', 'max']
for key in tqdm_notebook(['card1','card2','card3','card5','uid1','uid2','uid3','uid4','uid5','bank_type']):
columns = create_new_columns(key, aggs)
 for i in columns:
diff_column = 'diff_' + i
ratio_column = 'ratio_' + i
df_train[diff_column] = df_train['TransactionAmt'] - df_train[i]
df_test[diff_column] = df_test['TransactionAmt'] - df_test[i]
df_train[ratio_column] = df_train['TransactionAmt'] / df_train[i]
df_test[ratio_column] = df_test['TransactionAmt'] / df_test[i]
df_trn, y_trn, nas = proc_df(df_raw, 'totals.transactionRevenue')
 train_required_ratio = 0.8
n_trn = int(len(df_trn)* train_required_ratio)
 X_train, X_valid = split_vals(df_trn, n_trn)
 y_train, y_valid = split_vals(y_trn, n_trn)
 X_test, y_test, _ = proc_df(df_test, 'totals.transactionRevenue', nas)
 X_train.shape, X_valid.shape, X_test.shape<train_model>
del df_temp
del df_grouped
gc.collect()
m = RandomForestRegressor(n_estimators=100, min_samples_leaf=3, n_jobs=-1, oob_score=True)
 m.fit(X_train, y_train )<compute_test_metric>
df_train['nulls1'] = df_train.isna().sum(axis=1)
 df_test['nulls1'] = df_test.isna().sum(axis=1 )
print_score(m )<predict_on_test>
categorical_features = list()
for i in df_train.columns:
if(df_train[i].dtype == object):
categorical_features.append(i)
 print(categorical_features )
predictions = m.predict(X_test )<set_options>
def label_encoding(df_train, df_test, feature):
le = preprocessing.LabelEncoder()
le.fit(list(df_train[feature].astype(str ).values)+ list(df_test[feature].astype(str ).values))
df_train[feature] = le.transform(list(df_train[feature].astype(str ).values))
df_test[feature] = le.transform(list(df_test[feature].astype(str ).values))
set_rf_samples(50000 )<train_model>
for feature in tqdm_notebook(categorical_features):
label_encoding(df_train, df_test, feature )
m = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )<split>
def frequency_encoding(df_train, df_test, feature, self_encoding=False):
df_temp = pd.concat([df_train[[feature]], df_test[[feature]]])
 fq_encode = df_temp[feature].value_counts(dropna=False ).to_dict()
if self_encoding:
df_train[feature] = df_train[feature].map(fq_encode)
 df_test[feature] = df_test[feature].map(fq_encode)
 else:
df_train[feature+'_fq_enc'] = df_train[feature].map(fq_encode)
 df_test[feature+'_fq_enc'] = df_test[feature].map(fq_encode )
raw_train, raw_valid = split_vals(df_raw, n_trn )<compute_test_metric>
for feature in tqdm_notebook(categorical_features):
frequency_encoding(df_train, df_test, feature, self_encoding=False )
fi = rf_feat_importance(m, df_trn); fi[:30]<filter>
for feature in tqdm_notebook(['DT_M','DT_W','DT_D']):
frequency_encoding(df_train, df_test, feature, self_encoding=False )
fi[fi.imp>0.0001]<features_selection>
for feature in tqdm_notebook(['card1','card2','card3','card5','uid1','uid2','uid3','uid4','uid5']):
frequency_encoding(df_train, df_test, feature, self_encoding=False )
to_keep = fi[fi.imp>0.0001].cols; len(to_keep )<split>
for feature in tqdm_notebook(['general_vendor','general_platform','general_type']):
frequency_encoding(df_train, df_test, feature, self_encoding=False )
df_keep = df_trn[to_keep].copy()
X_train, X_valid = split_vals(df_keep, n_trn )<train_model>
for df in [df_train, df_test]:
df = reduce_mem_usage(df=df )
m = RandomForestRegressor(n_estimators=40, min_samples_leaf=3, max_features=0.5,
n_jobs=-1, oob_score=True)
 m.fit(X_train, y_train)
 print_score(m )<train_model>
gc.collect()
<import_modules><EOS> 
df_train.to_pickle('df_train.pkl')
 df_test.to_pickle('df_test.pkl' )