Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<define_variables>
warnings.simplefilter('ignore')
 sns.set()
%matplotlib inline
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 SEED = 1993
seed_everything(SEED)
 START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d' )
target = df_train["target"] 
<drop_column>
files = ['.. /input/ieee-fraud-detection/test_identity.csv',
'.. /input/ieee-fraud-detection/test_transaction.csv',
'.. /input/ieee-fraud-detection/train_identity.csv',
'.. /input/ieee-fraud-detection/train_transaction.csv',
'.. /input/ieee-fraud-detection/sample_submission.csv']
df_train.drop("target", inplace=True, axis=1 )<choose_model_class>
%%time
def load_data(file):
return pd.read_csv(file)
 with multiprocessing.Pool() as pool:
test_identity, test_transaction, train_identity, train_transaction, sample_submission = pool.map(load_data, files )
catb_opt = CatboostOptimizer()
catb_opt.n_trials = N_ROUNDS_CATB
catb_opt.n_folds = N_FOLDS
catb_opt.use_gpu = True
catb_opt.cat_features = categorical_indices(df_train)
 X_cols = df_train.columns
catb_results = catb_opt.classify_binary(X_cols, df_train, df_test, target)
<load_from_csv>
train = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')
 test = pd.merge(test_transaction, test_identity, on='TransactionID', how='left')
 print("Train shape: ", train.shape)
 print("Test shape:", test.shape)
 y = train['isFraud'].copy()
del train_transaction, train_identity, test_transaction, test_identity
gc.collect()
train = train.drop('isFraud', axis=1 )
submission = pd.read_csv("/kaggle/input/cat-in-the-dat/sample_submission.csv")
 submission.head()<prepare_output>
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )
submission["target"] = catb_results.predictions[:, 1]
submission.head()<save_to_csv>
useful_features = ['TransactionAmt', 'ProductCD', 'card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'addr1', 'addr2', 'dist1',
'P_emaildomain', 'R_emaildomain', 'C1', 'C2', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13',
'C14', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D8', 'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'M2', 'M3',
'M4', 'M5', 'M6', 'M7', 'M8', 'M9', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V17',
'V19', 'V20', 'V29', 'V30', 'V33', 'V34', 'V35', 'V36', 'V37', 'V38', 'V40', 'V44', 'V45', 'V46', 'V47', 'V48',
'V49', 'V51', 'V52', 'V53', 'V54', 'V56', 'V58', 'V59', 'V60', 'V61', 'V62', 'V63', 'V64', 'V69', 'V70', 'V71',
'V72', 'V73', 'V74', 'V75', 'V76', 'V78', 'V80', 'V81', 'V82', 'V83', 'V84', 'V85', 'V87', 'V90', 'V91', 'V92',
'V93', 'V94', 'V95', 'V96', 'V97', 'V99', 'V100', 'V126', 'V127', 'V128', 'V130', 'V131', 'V138', 'V139', 'V140',
'V143', 'V145', 'V146', 'V147', 'V149', 'V150', 'V151', 'V152', 'V154', 'V156', 'V158', 'V159', 'V160', 'V161',
'V162', 'V163', 'V164', 'V165', 'V166', 'V167', 'V169', 'V170', 'V171', 'V172', 'V173', 'V175', 'V176', 'V177',
'V178', 'V180', 'V182', 'V184', 'V187', 'V188', 'V189', 'V195', 'V197', 'V200', 'V201', 'V202', 'V203', 'V204',
'V205', 'V206', 'V207', 'V208', 'V209', 'V210', 'V212', 'V213', 'V214', 'V215', 'V216', 'V217', 'V219', 'V220',
'V221', 'V222', 'V223', 'V224', 'V225', 'V226', 'V227', 'V228', 'V229', 'V231', 'V233', 'V234', 'V238', 'V239',
'V242', 'V243', 'V244', 'V245', 'V246', 'V247', 'V249', 'V251', 'V253', 'V256', 'V257', 'V258', 'V259', 'V261',
'V262', 'V263', 'V264', 'V265', 'V266', 'V267', 'V268', 'V270', 'V271', 'V272', 'V273', 'V274', 'V275', 'V276',
'V277', 'V278', 'V279', 'V280', 'V282', 'V283', 'V285', 'V287', 'V288', 'V289', 'V291', 'V292', 'V294', 'V303',
'V304', 'V306', 'V307', 'V308', 'V310', 'V312', 'V313', 'V314', 'V315', 'V317', 'V322', 'V323', 'V324', 'V326',
'V329', 'V331', 'V332', 'V333', 'V335', 'V336', 'V338', 'id_01', 'id_02', 'id_03', 'id_05', 'id_06', 'id_09',
'id_11', 'id_12', 'id_13', 'id_14', 'id_15', 'id_17', 'id_19', 'id_20', 'id_30', 'id_31', 'id_32', 'id_33',
'id_36', 'id_37', 'id_38', 'DeviceType', 'DeviceInfo']
submission.to_csv("submission_catb.csv", index=False )<import_modules>
cols_to_drop = [col for col in train.columns if col not in useful_features]
cols_to_drop.remove('TransactionID')
 cols_to_drop.remove('TransactionDT' )
import numpy as np
import pandas as pd
from sklearn.linear_model import LogisticRegression
import scipy
from tqdm import tqdm_notebook as tqdm<load_from_csv>
print('{} features are going to be dropped for being useless'.format(len(cols_to_drop)))
 train = train.drop(cols_to_drop, axis=1)
 test = test.drop(cols_to_drop, axis=1 )
dd0=pd.read_csv("/kaggle/input/cat-in-the-dat/train.csv")
 ddtest0=pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv")
 ddall=dd0.append(ddtest0, sort=False)
 num_train=len(dd0)
 ddall.head()<feature_engineering>
def addNewFeatures(data):
data['uid1'] = data['card1'].astype(str)+'_'+data['card2'].astype(str)
 data['uid2'] = data['uid1'].astype(str)+'_'+data['card3'].astype(str)+'_'+data['card5'].astype(str)
 data['uid3'] = data['uid2'].astype(str)+'_'+data['addr1'].astype(str)+'_'+data['addr2'].astype(str)
 data['D9'] = np.where(data['D9'].isna() ,0,1)
 return data
train = addNewFeatures(train)
 test = addNewFeatures(test)
 train['Transaction_day_of_week'] = np.floor(( train['TransactionDT'] /(3600 * 24)- 1)% 7)
 test['Transaction_day_of_week'] = np.floor(( test['TransactionDT'] /(3600 * 24)- 1)% 7)
 train['Transaction_hour_of_day'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour_of_day'] = np.floor(test['TransactionDT'] / 3600)% 24
train['Transaction_hour'] = np.floor(train['TransactionDT'] / 3600)% 24
test['Transaction_hour'] = np.floor(test['TransactionDT'] / 3600)% 24
train['TransactionAmt_decimal'] =(( train['TransactionAmt'] - train['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 test['TransactionAmt_decimal'] =(( test['TransactionAmt'] - test['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 for feature in ['id_02__id_20', 'id_02__D8', 'D11__DeviceInfo', 'DeviceInfo__P_emaildomain', 'P_emaildomain__C2',
'card2__dist1', 'card1__card5', 'card2__id_20', 'card5__P_emaildomain', 'addr1__card1']:
f1, f2 = feature.split('__')
 train[feature] = train[f1].astype(str)+ '_' + train[f2].astype(str)
 test[feature] = test[f1].astype(str)+ '_' + test[f2].astype(str)
 le = LabelEncoder()
le.fit(list(train[feature].astype(str ).values)+ list(test[feature].astype(str ).values))
train[feature] = le.transform(list(train[feature].astype(str ).values))
test[feature] = le.transform(list(test[feature].astype(str ).values))
for feature in ['id_34', 'id_36']:
if feature in useful_features:
train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
for feature in ['id_01', 'id_31', 'id_33', 'id_35', 'id_36']:
if feature in useful_features:
train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))
test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))
drop_cols=["bin_0"]
ddall["ord_5a"]=ddall["ord_5"].str[0]
ddall["ord_5b"]=ddall["ord_5"].str[1]
drop_cols.append("ord_5" )<drop_column>
train['card1_count_full'] = train['card1'].map(pd.concat([train['card1'], test['card1']], ignore_index=True ).value_counts(dropna=False))
test['card1_count_full'] = test['card1'].map(pd.concat([train['card1'], test['card1']], ignore_index=True ).value_counts(dropna=False))
train['card2_count_full'] = train['card2'].map(pd.concat([train['card2'], test['card2']], ignore_index=True ).value_counts(dropna=False))
test['card2_count_full'] = test['card2'].map(pd.concat([train['card2'], test['card2']], ignore_index=True ).value_counts(dropna=False))
train['card3_count_full'] = train['card3'].map(pd.concat([train['card3'], test['card3']], ignore_index=True ).value_counts(dropna=False))
test['card3_count_full'] = test['card3'].map(pd.concat([train['card3'], test['card3']], ignore_index=True ).value_counts(dropna=False))
train['card4_count_full'] = train['card4'].map(pd.concat([train['card4'], test['card4']], ignore_index=True ).value_counts(dropna=False))
test['card4_count_full'] = test['card4'].map(pd.concat([train['card4'], test['card4']], ignore_index=True ).value_counts(dropna=False))
train['card5_count_full'] = train['card5'].map(pd.concat([train['card5'], test['card5']], ignore_index=True ).value_counts(dropna=False))
test['card5_count_full'] = test['card5'].map(pd.concat([train['card5'], test['card5']], ignore_index=True ).value_counts(dropna=False))
train['card6_count_full'] = train['card6'].map(pd.concat([train['card6'], test['card6']], ignore_index=True ).value_counts(dropna=False))
test['card6_count_full'] = test['card6'].map(pd.concat([train['card6'], test['card6']], ignore_index=True ).value_counts(dropna=False))
train['addr1_count_full'] = train['addr1'].map(pd.concat([train['addr1'], test['addr1']], ignore_index=True ).value_counts(dropna=False))
test['addr1_count_full'] = test['addr1'].map(pd.concat([train['addr1'], test['addr1']], ignore_index=True ).value_counts(dropna=False))
train['addr2_count_full'] = train['addr2'].map(pd.concat([train['addr2'], test['addr2']], ignore_index=True ).value_counts(dropna=False))
test['addr2_count_full'] = test['addr2'].map(pd.concat([train['addr2'], test['addr2']], ignore_index=True ).value_counts(dropna=False))
X=ddall[ddall.columns.difference(["id", "target"] + drop_cols)]<categorify>
train['TransactionAmt_to_mean_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_mean_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_std_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('std')
 train['TransactionAmt_to_std_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_mean_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_mean_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_std_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_std_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('std')
 train['id_02_to_mean_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('mean')
 train['id_02_to_mean_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('mean')
 train['id_02_to_std_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('std')
 train['id_02_to_std_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('std')
 test['id_02_to_mean_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('mean')
 test['id_02_to_mean_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('mean')
 test['id_02_to_std_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('std')
 test['id_02_to_std_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('std')
 train['D15_to_mean_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['D15_to_mean_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std' )
X_oh=X[X.columns.difference(["ord_1", "ord_4", "ord_5a", "ord_5b", "day", "month"])]
oh1=pd.get_dummies(X_oh, columns=X_oh.columns, drop_first=True, sparse=True)
 ohc1=oh1.sparse.to_coo()<categorify>
train['TransactionAmt'] = np.log1p(train['TransactionAmt'])
 test['TransactionAmt'] = np.log1p(test['TransactionAmt'] )
class ThermometerEncoder(TransformerMixin):

 def __init__(self, sort_key=None):
self.sort_key = sort_key
self.value_map_ = None
def fit(self, X, y=None):
self.value_map_ = {val: i for i, val in enumerate(sorted(X.unique() , key=self.sort_key)) }
return self
def transform(self, X, y=None):
values = X.map(self.value_map_)
 possible_values = sorted(self.value_map_.values())
 idx1 = []
idx2 = []
all_indices = np.arange(len(X))
for idx, val in enumerate(possible_values[:-1]):
new_idxs = all_indices[values > val]
idx1.extend(new_idxs)
 idx2.extend(repeat(idx, len(new_idxs)))
 result = scipy.sparse.coo_matrix(( [1] * len(idx1),(idx1, idx2)) , shape=(len(X), len(possible_values)) , dtype="int8")
 return result<categorify>
dates_range = pd.date_range(start='2017-10-01', end='2019-01-01')
 us_holidays = calendar().holidays(start=dates_range.min() , end=dates_range.max())
 def setTime(df):
df['DT'] = df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 df['DT_M'] =(( df['DT'].dt.year-2017)*12 + df['DT'].dt.month ).astype(np.int8)
 df['DT_W'] =(( df['DT'].dt.year-2017)*52 + df['DT'].dt.weekofyear ).astype(np.int8)
 df['DT_D'] =(( df['DT'].dt.year-2017)*365 + df['DT'].dt.dayofyear ).astype(np.int16)
 df['DT_hour'] =(df['DT'].dt.hour ).astype(np.int8)
 df['DT_day_week'] =(df['DT'].dt.dayofweek ).astype(np.int8)
 df['DT_day_month'] =(df['DT'].dt.day ).astype(np.int8)
 df['is_december'] = df['DT'].dt.month
df['is_december'] =(df['is_december']==12 ).astype(np.int8)
 df['is_holiday'] =(df['DT'].dt.date.astype('datetime64' ).isin(us_holidays)).astype(np.int8)
 return df
train = setTime(train)
 test = setTime(test )
thermos=[]
for col in ["ord_1", "ord_2", "ord_3", "ord_4", "ord_5a", "day", "month"]:
if col=="ord_1":
sort_key=['Novice', 'Contributor', 'Expert', 'Master', 'Grandmaster'].index
elif col=="ord_2":
sort_key=['Freezing', 'Cold', 'Warm', 'Hot', 'Boiling Hot', 'Lava Hot'].index
elif col in ["ord_3", "ord_4", "ord_5a"]:
sort_key=str
elif col in ["day", "month"]:
sort_key=int
else:
raise ValueError(col)
 enc=ThermometerEncoder(sort_key=sort_key)
 thermos.append(enc.fit_transform(X[col]))<prepare_x_and_y>
i_cols = ['card1','card2','card3','card5','uid1','uid2','uid3']
for col in i_cols:
for agg_type in ['mean','std']:
new_col_name = col+'_TransactionAmt_'+agg_type
temp_df = pd.concat([train[[col, 'TransactionAmt']], test[[col,'TransactionAmt']]])
 temp_df = temp_df.groupby([col])['TransactionAmt'].agg([agg_type] ).reset_index().rename(
 columns={agg_type: new_col_name})
 temp_df.index = list(temp_df[col])
 temp_df = temp_df[new_col_name].to_dict()
train[new_col_name] = train[col].map(temp_df)
 test[new_col_name] = test[col].map(temp_df)
 train = train.replace(np.inf,999)
 test = test.replace(np.inf,999 )
ohc=scipy.sparse.hstack([ohc1] + thermos ).tocsr()
display(ohc)
 X_train = ohc[:num_train]
X_test = ohc[num_train:]
y_train = dd0["target"].values<save_to_csv>
train['P_isproton'] =(train['P_emaildomain']=='protonmail.com')
 train['R_isproton'] =(train['R_emaildomain']=='protonmail.com')
 test['P_isproton'] =(test['P_emaildomain']=='protonmail.com')
 test['R_isproton'] =(test['R_emaildomain']=='protonmail.com' )
clf=LogisticRegression(C=0.123456789, solver="lbfgs", max_iter=5000)
 clf.fit(X_train, y_train)
 pred=clf.predict_proba(X_test)[:,1]
pd.DataFrame({"id": ddtest0["id"], "target": pred} ).to_csv("submission.csv", index=False )<compute_train_metric>
train['nulls1'] = train.isna().sum(axis=1)
 test['nulls1'] = test.isna().sum(axis=1 )
score=cross_validate(clf, X_train, y_train, cv=3, scoring="roc_auc")["test_score"].mean()
print(f"{score:.6f}" )<load_from_csv>
emails = {'gmail': 'google', 'att.net': 'att', 'twc.com': 'spectrum', 'scranton.edu': 'other', 'optonline.net': 'other',
'hotmail.co.uk': 'microsoft', 'comcast.net': 'other', 'yahoo.com.mx': 'yahoo', 'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo', 'charter.net': 'spectrum', 'live.com': 'microsoft', 'aim.com': 'aol', 'hotmail.de': 'microsoft',
'centurylink.net': 'centurylink', 'gmail.com': 'google', 'me.com': 'apple', 'earthlink.net': 'other',
'gmx.de': 'other', 'web.de': 'other', 'cfl.rr.com': 'other', 'hotmail.com': 'microsoft', 'protonmail.com': 'other',
'hotmail.fr': 'microsoft', 'windstream.net': 'other', 'outlook.es': 'microsoft', 'yahoo.co.jp': 'yahoo',
'yahoo.de': 'yahoo', 'servicios-ta.com': 'other', 'netzero.net': 'other', 'suddenlink.net': 'other',
'roadrunner.com': 'other', 'sc.rr.com': 'other', 'live.fr': 'microsoft', 'verizon.net': 'yahoo',
'msn.com': 'microsoft', 'q.com': 'centurylink', 'prodigy.net.mx': 'att', 'frontier.com': 'yahoo',
'anonymous.com': 'other', 'rocketmail.com': 'yahoo', 'sbcglobal.net': 'att', 'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo', 'outlook.com': 'microsoft', 'mail.com': 'other', 'bellsouth.net': 'other',
'embarqmail.com': 'centurylink', 'cableone.net': 'other', 'hotmail.es': 'microsoft', 'mac.com': 'apple',
'yahoo.co.uk': 'yahoo', 'netzero.com': 'other', 'yahoo.com': 'yahoo', 'live.com.mx': 'microsoft', 'ptd.net': 'other',
'cox.net': 'other', 'aol.com': 'aol', 'juno.com': 'other', 'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']
submission = pd.read_csv('.. /input/cat-in-the-dat/sample_submission.csv', index_col='id' )<load_from_csv>
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )
kernels = pd.read_csv('.. /input/cat-in-dat-kernels/kernels.csv', index_col='id' )<load_from_csv>
p = 'P_emaildomain'
r = 'R_emaildomain'
uknown = 'email_not_provided'
def setDomain(df):
df[p] = df[p].fillna(uknown)
 df[r] = df[r].fillna(uknown)
 df['email_check'] = np.where(( df[p]==df[r])&(df[p]!=uknown),1,0)
 df[p+'_prefix'] = df[p].apply(lambda x: x.split('.')[0])
 df[r+'_prefix'] = df[r].apply(lambda x: x.split('.')[0])
 return df
train=setDomain(train)
 test=setDomain(test )
def make_filename(idx):
return glob.glob('.. /input/cat-in-dat-kernels/' + str(idx)+ '__submission__*.csv')[0]
def read_predictions(idx):
temp = pd.read_csv(make_filename(idx), index_col='id')
 temp.columns = [str(idx)]
return temp
predictions = pd.concat([read_predictions(idx)for idx in kernels.index], axis=1)
 predictions.shape<save_to_csv>
train["lastest_browser"] = np.zeros(train.shape[0])
 test["lastest_browser"] = np.zeros(test.shape[0])
 def setBrowser(df):
df.loc[df["id_31"] == "samsung browser 7.0",'lastest_browser']=1
df.loc[df["id_31"] == "opera 53.0",'lastest_browser']=1
df.loc[df["id_31"] == "mobile safari 10.0",'lastest_browser']=1
df.loc[df["id_31"] == "google search application 49.0",'lastest_browser']=1
df.loc[df["id_31"] == "firefox 60.0",'lastest_browser']=1
df.loc[df["id_31"] == "edge 17.0",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 69.0",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 67.0 for android",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 63.0 for android",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 63.0 for ios",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 64.0",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 64.0 for android",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 64.0 for ios",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 65.0",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 65.0 for android",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 65.0 for ios",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 66.0",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 66.0 for android",'lastest_browser']=1
df.loc[df["id_31"] == "chrome 66.0 for ios",'lastest_browser']=1
return df
train = setBrowser(train)
 test = setBrowser(test )
submission['target'] = predictions.mean(axis=1)
 submission.to_csv('stack-mean.csv' )<define_variables>
def setDevice(df):
df['DeviceInfo'] = df['DeviceInfo'].fillna('unknown_device' ).str.lower()
df['device_name'] = df['DeviceInfo'].str.split('/', expand=True)[0]
df.loc[df['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
df.loc[df['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
df.loc[df['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
df.loc[df['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
df.loc[df['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
df.loc[df['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
df.loc[df['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
df.loc[df['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
df.loc[df.device_name.isin(df.device_name.value_counts() [df.device_name.value_counts() < 200].index), 'device_name'] = "Others"
df['had_id'] = 1
gc.collect()
return df
train = setDevice(train)
 test = setDevice(test )
scores = kernels['score']
sum_scores = sum(scores)
 weights = [x / sum_scores for x in scores]<save_to_csv>
i_cols = ['card1','card2','card3','card5',
'C1','C2','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14',
'D1','D2','D3','D4','D5','D6','D8',
'addr1','addr2',
'dist1',
'P_emaildomain', 'R_emaildomain',
'DeviceInfo','device_name',
'id_30','id_33',
'uid1','uid2','uid3',
]
for col in i_cols:
temp_df = pd.concat([train[[col]], test[[col]]])
 fq_encode = temp_df[col].value_counts(dropna=False ).to_dict()
train[col+'_fq_enc'] = train[col].map(fq_encode)
 test[col+'_fq_enc'] = test[col].map(fq_encode)
 for col in ['DT_M','DT_W','DT_D']:
temp_df = pd.concat([train[[col]], test[[col]]])
 fq_encode = temp_df[col].value_counts().to_dict()
train[col+'_total'] = train[col].map(fq_encode)
 test[col+'_total'] = test[col].map(fq_encode)
 periods = ['DT_M','DT_W','DT_D']
i_cols = ['uid1']
for period in periods:
for col in i_cols:
new_column = col + '_' + period
temp_df = pd.concat([train[[col,period]], test[[col,period]]])
 temp_df[new_column] = temp_df[col].astype(str)+ '_' +(temp_df[period] ).astype(str)
 fq_encode = temp_df[new_column].value_counts().to_dict()
train[new_column] =(train[col].astype(str)+ '_' + train[period].astype(str)).map(fq_encode)
 test[new_column] =(test[col].astype(str)+ '_' + test[period].astype(str)).map(fq_encode)
 train[new_column] /= train[period+'_total']
test[new_column] /= test[period+'_total']
submission['target'] = sum_predictions
submission.to_csv('stack-weighted-sum.csv' )<sort_values>
def get_too_many_null_attr(data):
many_null_cols = [col for col in data.columns if data[col].isnull().sum() / data.shape[0] > 0.9]
return many_null_cols
def get_too_many_repeated_val(data):
big_top_value_cols = [col for col in train.columns if train[col].value_counts(dropna=False, normalize=True ).values[0] > 0.9]
return big_top_value_cols
def get_useless_columns(data):
too_many_null = get_too_many_null_attr(data)
 print("More than 90% null: " + str(len(too_many_null)))
 too_many_repeated = get_too_many_repeated_val(data)
 print("More than 90% repeated value: " + str(len(too_many_repeated)))
 cols_to_drop = list(set(too_many_null + too_many_repeated))
return cols_to_drop
N = 3
selected = kernels.sort_values('score', ascending=False ).head(N )<data_type_conversions>
for f in train.columns:
if train[f].dtype.name =='object' or test[f].dtype.name =='object':
lbl = LabelEncoder()
lbl.fit(list(train[f].values)+ list(test[f].values))
train[f] = lbl.transform(list(train[f].values))
test[f] = lbl.transform(list(test[f].values))
filter_predictions = predictions.loc[:,selected.index.values.astype(str)]<save_to_csv>
train = train.fillna(-999)
 test = test.fillna(-999 )
submission['target'] = filter_predictions.mean(axis=1)
 submission.to_csv('stack-filtered.csv' )<load_from_csv>
print(train.isnull().sum().max())
 print(test.isnull().sum().max() )
submission = pd.read_csv("/kaggle/input/cat-in-the-dat/sample_submission.csv")
 train = pd.read_csv("/kaggle/input/cat-in-the-dat/train.csv")
 test = pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv")
 labels = train.pop('target')
 train_id = train.pop("id")
 test_id = test.pop("id" )<define_variables>
X = train.drop(['TransactionID', 'TransactionDT', 'DT'], axis=1)
 X_test = test.drop(['TransactionID', 'TransactionDT', 'DT'], axis=1 )
labels = labels.values<concatenate>
params = {'num_leaves': 546,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.1797454081646243,
'bagging_fraction': 0.2181193142567742,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.005883242363721497,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3299927210061127,
'reg_lambda': 0.3885237330340494,
'random_state': SEED,
}
data = pd.concat([train, test] )<feature_engineering>
%%time
NFOLDS = 5
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds = np.zeros(X_test.shape[0])
 y_oof = np.zeros(X.shape[0])
 score = 0
feature_importances = pd.DataFrame()
feature_importances['feature'] = columns
for fold_n,(train_index, valid_index)in enumerate(splits):
X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
dtrain = lgb.Dataset(X_train, label=y_train)
 dvalid = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(
 params,
dtrain,
10000,
valid_sets = [dtrain, dvalid],
verbose_eval=200,
early_stopping_rounds=500 
)
 feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()
y_pred_valid = clf.predict(X_valid)
 y_oof[valid_index] = y_pred_valid
print(f"Fold {fold_n + 1} | AUC: {roc_auc_score(y_valid, y_pred_valid)}")
 score += roc_auc_score(y_valid, y_pred_valid)/ NFOLDS
y_preds += clf.predict(X_test)/ NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
print(f"
Mean AUC = {score}")
 print(f"Out of folds AUC = {roc_auc_score(y, y_oof)}" )
data["ord_5a"] = data["ord_5"].str[0]
data["ord_5b"] = data["ord_5"].str[1]<drop_column>
sample_submission['isFraud'] = y_preds
data.drop(["bin_0", "ord_5"], axis=1, inplace=True )<categorify>
sample_submission.to_csv('submission.csv', index=False )
columns = [i for i in data.columns]
dummies = pd.get_dummies(data,
columns=columns,
drop_first=True,
sparse=True)
 del data<categorify>
feature_importances['average'] = feature_importances[[f'fold_{fold_n + 1}' for fold_n in range(folds.n_splits)]].mean(axis=1)
 feature_importances.to_csv('feature_importances.csv')
 plt.figure(figsize=(16, 16))
sns.barplot(data=feature_importances.sort_values(by='average', ascending=False ).head(50), x='average', y='feature');
plt.title('50 TOP feature importance over {} folds average'.format(folds.n_splits)) ;
train = dummies.iloc[:train.shape[0], :]
test = dummies.iloc[train.shape[0]:, :]
del dummies<data_type_conversions>
FileLinks('.' )