Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<define_variables>
!pip install -q 'pandas==0.25' --force-reinstall
!pip install GPyOpt
img_size1 = 64
img_size2 = 64<train_model>
print("GPyOpt version:", GPyOpt.__version__)
 print("LGBM version:", xgb.__version__)
 print("Pandas version:", pd.__version__ )
new_img = cv2.resize(test_array,(img_size2,img_size1))
plt.imshow(new_img,cmap='gray')
 plt.show()<normalization>
notebookstart = time.time()
warnings.filterwarnings("ignore")
 seed = 24
np.random.seed(seed)
 pd.set_option('display.max_columns', 500)
 pd.options.display.max_rows = 999
pd.set_option('max_colwidth', 500 )
kernel = np.ones(( 5,5), np.uint8)
 img_dl = cv2.dilate(new_img, kernel, iterations=1)
 plt.imshow(img_dl,cmap='gray')
 plt.show()<normalization>
print("Define DF Schema.. ")
 target_var = 'isFraud'
schema = {
"TransactionDT": "int32",
"TransactionAmt": "float32",
"ProductCD": "object",
"card1": "int16",
"card2": "float32",
"card3": "float32",
"card4": "object",
"card5": "float32",
"card6": "object",
"addr1": "float32",
"addr2": "float32",
"dist1": "float32",
"dist2": "float32",
"P_emaildomain": "object",
"R_emaildomain": "object",
"C1": "float32",
"C2": "float32",
"C3": "float32",
"C4": "float32",
"C5": "float32",
"C6": "float32",
"C7": "float32",
"C8": "float32",
"C9": "float32",
"C10": "float32",
"C11": "float32",
"C12": "float32",
"C13": "float32",
"C14": "float32",
"D1": "float32",
"D2": "float32",
"D3": "float32",
"D4": "float32",
"D5": "float32",
"D6": "float32",
"D7": "float32",
"D8": "float32",
"D9": "float32",
"D10": "float32",
"D11": "float32",
"D12": "float32",
"D13": "float32",
"D14": "float32",
"D15": "float32",
"M1": "object",
"M2": "object",
"M3": "object",
"M4": "object",
"M5": "object",
"M6": "object",
"M7": "object",
"M8": "object",
"M9": "object",
"V1": "float32",
"V2": "float32",
"V3": "float32",
"V4": "float32",
"V5": "float32",
"V6": "float32",
"V7": "float32",
"V8": "float32",
"V9": "float32",
"V10": "float32",
"V11": "float32",
"V12": "float32",
"V13": "float32",
"V14": "float32",
"V15": "float32",
"V16": "float32",
"V17": "float32",
"V18": "float32",
"V19": "float32",
"V20": "float32",
"V21": "float32",
"V22": "float32",
"V23": "float32",
"V24": "float32",
"V25": "float32",
"V26": "float32",
"V27": "float32",
"V28": "float32",
"V29": "float32",
"V30": "float32",
"V31": "float32",
"V32": "float32",
"V33": "float32",
"V34": "float32",
"V35": "float32",
"V36": "float32",
"V37": "float32",
"V38": "float32",
"V39": "float32",
"V40": "float32",
"V41": "float32",
"V42": "float32",
"V43": "float32",
"V44": "float32",
"V45": "float32",
"V46": "float32",
"V47": "float32",
"V48": "float32",
"V49": "float32",
"V50": "float32",
"V51": "float32",
"V52": "float32",
"V53": "float32",
"V54": "float32",
"V55": "float32",
"V56": "float32",
"V57": "float32",
"V58": "float32",
"V59": "float32",
"V60": "float32",
"V61": "float32",
"V62": "float32",
"V63": "float32",
"V64": "float32",
"V65": "float32",
"V66": "float32",
"V67": "float32",
"V68": "float32",
"V69": "float32",
"V70": "float32",
"V71": "float32",
"V72": "float32",
"V73": "float32",
"V74": "float32",
"V75": "float32",
"V76": "float32",
"V77": "float32",
"V78": "float32",
"V79": "float32",
"V80": "float32",
"V81": "float32",
"V82": "float32",
"V83": "float32",
"V84": "float32",
"V85": "float32",
"V86": "float32",
"V87": "float32",
"V88": "float32",
"V89": "float32",
"V90": "float32",
"V91": "float32",
"V92": "float32",
"V93": "float32",
"V94": "float32",
"V95": "float32",
"V96": "float32",
"V97": "float32",
"V98": "float32",
"V99": "float32",
"V100": "float32",
"V101": "float32",
"V102": "float32",
"V103": "float32",
"V104": "float32",
"V105": "float32",
"V106": "float32",
"V107": "float32",
"V108": "float32",
"V109": "float32",
"V110": "float32",
"V111": "float32",
"V112": "float32",
"V113": "float32",
"V114": "float32",
"V115": "float32",
"V116": "float32",
"V117": "float32",
"V118": "float32",
"V119": "float32",
"V120": "float32",
"V121": "float32",
"V122": "float32",
"V123": "float32",
"V124": "float32",
"V125": "float32",
"V126": "float32",
"V127": "float32",
"V128": "float32",
"V129": "float32",
"V130": "float32",
"V131": "float32",
"V132": "float32",
"V133": "float32",
"V134": "float32",
"V135": "float32",
"V136": "float32",
"V137": "float32",
"V138": "float32",
"V139": "float32",
"V140": "float32",
"V141": "float32",
"V142": "float32",
"V143": "float32",
"V144": "float32",
"V145": "float32",
"V146": "float32",
"V147": "float32",
"V148": "float32",
"V149": "float32",
"V150": "float32",
"V151": "float32",
"V152": "float32",
"V153": "float32",
"V154": "float32",
"V155": "float32",
"V156": "float32",
"V157": "float32",
"V158": "float32",
"V159": "float32",
"V160": "float32",
"V161": "float32",
"V162": "float32",
"V163": "float32",
"V164": "float32",
"V165": "float32",
"V166": "float32",
"V167": "float32",
"V168": "float32",
"V169": "float32",
"V170": "float32",
"V171": "float32",
"V172": "float32",
"V173": "float32",
"V174": "float32",
"V175": "float32",
"V176": "float32",
"V177": "float32",
"V178": "float32",
"V179": "float32",
"V180": "float32",
"V181": "float32",
"V182": "float32",
"V183": "float32",
"V184": "float32",
"V185": "float32",
"V186": "float32",
"V187": "float32",
"V188": "float32",
"V189": "float32",
"V190": "float32",
"V191": "float32",
"V192": "float32",
"V193": "float32",
"V194": "float32",
"V195": "float32",
"V196": "float32",
"V197": "float32",
"V198": "float32",
"V199": "float32",
"V200": "float32",
"V201": "float32",
"V202": "float32",
"V203": "float32",
"V204": "float32",
"V205": "float32",
"V206": "float32",
"V207": "float32",
"V208": "float32",
"V209": "float32",
"V210": "float32",
"V211": "float32",
"V212": "float32",
"V213": "float32",
"V214": "float32",
"V215": "float32",
"V216": "float32",
"V217": "float32",
"V218": "float32",
"V219": "float32",
"V220": "float32",
"V221": "float32",
"V222": "float32",
"V223": "float32",
"V224": "float32",
"V225": "float32",
"V226": "float32",
"V227": "float32",
"V228": "float32",
"V229": "float32",
"V230": "float32",
"V231": "float32",
"V232": "float32",
"V233": "float32",
"V234": "float32",
"V235": "float32",
"V236": "float32",
"V237": "float32",
"V238": "float32",
"V239": "float32",
"V240": "float32",
"V241": "float32",
"V242": "float32",
"V243": "float32",
"V244": "float32",
"V245": "float32",
"V246": "float32",
"V247": "float32",
"V248": "float32",
"V249": "float32",
"V250": "float32",
"V251": "float32",
"V252": "float32",
"V253": "float32",
"V254": "float32",
"V255": "float32",
"V256": "float32",
"V257": "float32",
"V258": "float32",
"V259": "float32",
"V260": "float32",
"V261": "float32",
"V262": "float32",
"V263": "float32",
"V264": "float32",
"V265": "float32",
"V266": "float32",
"V267": "float32",
"V268": "float32",
"V269": "float32",
"V270": "float32",
"V271": "float32",
"V272": "float32",
"V273": "float32",
"V274": "float32",
"V275": "float32",
"V276": "float32",
"V277": "float32",
"V278": "float32",
"V279": "float32",
"V280": "float32",
"V281": "float32",
"V282": "float32",
"V283": "float32",
"V284": "float32",
"V285": "float32",
"V286": "float32",
"V287": "float32",
"V288": "float32",
"V289": "float32",
"V290": "float32",
"V291": "float32",
"V292": "float32",
"V293": "float32",
"V294": "float32",
"V295": "float32",
"V296": "float32",
"V297": "float32",
"V298": "float32",
"V299": "float32",
"V300": "float32",
"V301": "float32",
"V302": "float32",
"V303": "float32",
"V304": "float32",
"V305": "float32",
"V306": "float32",
"V307": "float32",
"V308": "float32",
"V309": "float32",
"V310": "float32",
"V311": "float32",
"V312": "float32",
"V313": "float32",
"V314": "float32",
"V315": "float32",
"V316": "float32",
"V317": "float32",
"V318": "float32",
"V319": "float32",
"V320": "float32",
"V321": "float32",
"V322": "float32",
"V323": "float32",
"V324": "float32",
"V325": "float32",
"V326": "float32",
"V327": "float32",
"V328": "float32",
"V329": "float32",
"V330": "float32",
"V331": "float32",
"V332": "float32",
"V333": "float32",
"V334": "float32",
"V335": "float32",
"V336": "float32",
"V337": "float32",
"V338": "float32",
"V339": "float32",
"id_01": "float32",
"id_02": "float32",
"id_03": "float32",
"id_04": "float32",
"id_05": "float32",
"id_06": "float32",
"id_07": "float32",
"id_08": "float32",
"id_09": "float32",
"id_10": "float32",
"id_11": "float32",
"id_12": "object",
"id_13": "float32",
"id_14": "float32",
"id_15": "object",
"id_16": "object",
"id_17": "float32",
"id_18": "float32",
"id_19": "float32",
"id_20": "float32",
"id_21": "float32",
"id_22": "float32",
"id_23": "object",
"id_24": "float32",
"id_25": "float32",
"id_26": "float32",
"id_27": "object",
"id_28": "object",
"id_29": "object",
"id_30": "object",
"id_31": "object",
"id_32": "float32",
"id_33": "object",
"id_34": "object",
"id_35": "object",
"id_36": "object",
"id_37": "object",
"id_38": "object",
"DeviceType": "object",
"DeviceInfo": "object",
"is_fraud":			 "int8"
}
emails = {'gmail': 'google',
'att.net': 'att',
'twc.com': 'spectrum',
'scranton.edu': 'other',
'optonline.net': 'other',
'hotmail.co.uk': 'microsoft',
'comcast.net': 'other',
'yahoo.com.mx': 'yahoo',
'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo',
'charter.net': 'spectrum',
'live.com': 'microsoft',
'aim.com': 'aol',
'hotmail.de': 'microsoft',
'centurylink.net': 'centurylink',
'gmail.com': 'google',
'me.com': 'apple',
'earthlink.net': 'other',
'gmx.de': 'other',
'web.de': 'other',
'cfl.rr.com': 'other',
'hotmail.com': 'microsoft',
'protonmail.com': 'other',
'hotmail.fr': 'microsoft',
'windstream.net': 'other',
'outlook.es': 'microsoft',
'yahoo.co.jp': 'yahoo',
'yahoo.de': 'yahoo',
'servicios-ta.com': 'other',
'netzero.net': 'other',
'suddenlink.net': 'other',
'roadrunner.com': 'other',
'sc.rr.com': 'other',
'live.fr': 'microsoft',
'verizon.net': 'yahoo',
'msn.com': 'microsoft',
'q.com': 'centurylink',
'prodigy.net.mx': 'att',
'frontier.com': 'yahoo',
'anonymous.com': 'other',
'rocketmail.com': 'yahoo',
'sbcglobal.net': 'att',
'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo',
'outlook.com': 'microsoft',
'mail.com': 'other',
'bellsouth.net': 'other',
'embarqmail.com': 'centurylink',
'cableone.net': 'other',
'hotmail.es': 'microsoft',
'mac.com': 'apple',
'yahoo.co.uk': 'yahoo',
'netzero.com': 'other',
'yahoo.com': 'yahoo',
'live.com.mx': 'microsoft',
'ptd.net': 'other',
'cox.net': 'other',
'aol.com': 'aol',
'juno.com': 'other',
'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']
def addGaussianNoise(new_img):
row,col= new_img.shape
mean = 0
var = 0.1
sigma = 100
gauss = np.random.normal(mean,sigma,(row,col))
gauss = gauss.reshape(row,col)
 noisy = new_img + gauss
return noisy
gau_noi_img = addGaussianNoise(new_img)
 plt.imshow(gau_noi_img,cmap='gray')
 plt.show()<categorify>
@contextmanager
def timer(name):

 t0 = time.time()
yield
print('
[{}] done in {} Minutes
'.format(name, round(( time.time() - t0)/60,2)))
 def id_split(dataframe):
dataframe['device_name'] = dataframe['DeviceInfo'].str.split('/', expand=True)[0]
dataframe['device_version'] = dataframe['DeviceInfo'].str.split('/', expand=True)[1]
dataframe['OS_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[0]
dataframe['version_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[1]
dataframe['browser_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[0]
dataframe['version_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[1]
dataframe['screen_width'] = dataframe['id_33'].str.split('x', expand=True)[0]
dataframe['screen_height'] = dataframe['id_33'].str.split('x', expand=True)[1]
dataframe['id_34'] = dataframe['id_34'].str.split(':', expand=True)[1]
dataframe['id_23'] = dataframe['id_23'].str.split(':', expand=True)[1]
dataframe.loc[dataframe['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
dataframe.loc[dataframe['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
dataframe.loc[dataframe['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
dataframe.loc[dataframe['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
dataframe.loc[dataframe['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
dataframe.loc[dataframe['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
dataframe.loc[dataframe.device_name.isin(dataframe.device_name.value_counts() [dataframe.device_name.value_counts() < 200].index), 'device_name'] = "Others"
dataframe['had_id'] = 1
gc.collect()
return dataframe
def fraud_preprocessing(debug = None):
print("Starting Pre-Processing.. ")
 with timer("Load Tables"):
train_transaction = pd.read_csv('.. /input/train_transaction.csv',
index_col='TransactionID', nrows= debug, dtype = schema)
 test_transaction = pd.read_csv('.. /input/test_transaction.csv',
index_col='TransactionID', nrows= debug, dtype = schema)
 train_identity = pd.read_csv('.. /input/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/test_identity.csv', index_col='TransactionID')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv',
index_col='TransactionID',
nrows= debug)
 with timer("Merge Tables"):
train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print("Train Shape: {} Rows, {} Columns".format(*train.shape))
print("Test Shape: {} Rows, {} Columns".format(*test.shape))
y = train[target_var].copy()
del train_transaction, train_identity, test_transaction, test_identity
traindex = train.index
testdex = test.index
with timer("Train/Test Split Feature Engineering"):
train['TransactionAmt_to_mean_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_mean_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_std_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('std')
 train['TransactionAmt_to_std_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_mean_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_mean_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_std_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_std_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('std')
 train['id_02_to_mean_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('mean')
 train['id_02_to_mean_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('mean')
 train['id_02_to_std_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('std')
 train['id_02_to_std_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('std')
 test['id_02_to_mean_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('mean')
 test['id_02_to_mean_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('mean')
 test['id_02_to_std_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('std')
 test['id_02_to_std_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('std')
 train['D15_to_mean_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['D15_to_mean_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['TransactionAmt_Log'] = np.log(train['TransactionAmt'])
 test['TransactionAmt_Log'] = np.log(test['TransactionAmt'])
 for feature in ['card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'id_36']:
train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
for feature in ['id_01', 'id_31', 'id_33', 'id_36']:
train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))
test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 train = id_split(train)
 test = id_split(test)
 df = pd.concat([train.drop(target_var,axis=1),test],axis = 0)
 del train, test
with timer("Whole Feature Engineering"):
START_DATE = '2017-12-01'
startdate = datetime.datetime.strptime(START_DATE, '%Y-%m-%d')
 df = df.assign(
 TransactionAmt_decimal =(( df['TransactionAmt'] - df['TransactionAmt'].astype(int)) * 1000 ).astype(int),
card1_count_full = df['card1'].map(df['card1'].value_counts(dropna=False)) ,
Transaction_day_of_week = np.floor(( df['TransactionDT'] /(3600 * 24)- 1)% 7),
Transaction_hour = np.floor(df['TransactionDT'] / 3600)% 24,
TransactionDT = df['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds = x))), 
)
 df = df.assign(
 year = df['TransactionDT'].dt.year,
month = df['TransactionDT'].dt.month,
dow = df['TransactionDT'].dt.dayofweek,
quarter = df['TransactionDT'].dt.quarter,
hour = df['TransactionDT'].dt.hour,
day = df['TransactionDT'].dt.day,
all_group_nan_sum = df.isnull().sum(axis=1)/ df.shape[1],
all_group_0_count =(df == 0 ).astype(int ).sum(axis=1)/(df.shape[1] - df.isnull().sum(axis=1)) 
)
 prefix = ['C','D','Device','M','Transaction','V','addr','card','dist','id']
for i, p in enumerate(prefix):
column_set = [x for x in df.columns.tolist() if x.startswith(prefix[i])]
df[p + "group_nan_sum"] = df[column_set].isnull().sum(axis=1)/ df[column_set].shape[1]
numeric_cols = [x for x in column_set if df[x].dtype != object]
if numeric_cols:
df[p + "group_sum"] = df[column_set].sum(axis=1)
 df[p + "group_mean"] = df[column_set].mean(axis=1)
 df[p + "group_0_count"] =(df[column_set] == 0 ).astype(int ).sum(axis=1)/(df[column_set].shape[1] - df[p + "group_nan_sum"])
 with timer("Label Encode"):
categorical_cols = []
for f in df.columns:
if df[f].dtype=='object':
categorical_cols += [f]
lbl = preprocessing.LabelEncoder()
df[f] = lbl.fit_transform(df[f].astype(str))
print("Total Shape: {} Rows, {} Columns".format(*df.shape))
return df, y, traindex, testdex, categorical_cols, sample_submission
def add_Salt_Pepper_Noise(new_img, s_vs_p = 0.5, amount = 0.05):
row,col = new_img.shape
s_and_p = np.copy(new_img)
 num_salt = np.ceil(amount * new_img.size * s_vs_p)
 coords = [np.random.randint(0, i - 1, int(num_salt))
for i in new_img.shape]
s_and_p[coords] = 1
num_pepper = np.ceil(amount* new_img.size *(1.- s_vs_p))
coords = [np.random.randint(0, i - 1, int(num_pepper))
for i in new_img.shape]
s_and_p[coords] = 0
return s_and_p
sap_noi_img = add_Salt_Pepper_Noise(new_img, s_vs_p = 0.5, amount = 0.05)
 plt.imshow(sap_noi_img,cmap='gray')
 plt.show()<normalization>
DEBUG = None
df, y, traindex, testdex, cat_cols, sample_submission = fraud_preprocessing(debug = DEBUG)
 df.fillna(-999, inplace=True)
 X = df.loc[traindex,:]
feature_subset = [x for x in X.columns.tolist() if x not in ['TransactionDT','Fraud', 'traintest', 'yrmth']]
test = df.loc[testdex,:]
del df; gc.collect() ;
height,width = new_img.shape[:2]
center =(int(width/2), int(height/2))
angle = 45
M = cv2.getRotationMatrix2D(center, angle, 1)
 rotated_img = cv2.warpAffine(new_img, M,(width, height))
plt.imshow(rotated_img,cmap='gray')
 plt.show()<define_variables>
print("None Fraud: {}%, Fraud: {}%".format(*y.value_counts(normalize=True)))
 print("Randomness Score AUC: {}".format(
 metrics.roc_auc_score(y,np.array([y.value_counts(normalize=True)[0]]*y.shape[0]))))
try_img_size1 = 128
try_img_size2 = 128<train_on_grid>
def XGB_score(para):
parameters = para[0]
params = {
'objective': 'binary:logistic',
'eval_metric': metric,
'learning_rate': 0.1,
'tree_method': 'gpu_hist',
'importance_type': 'weight',
'missing': -999,
'max_depth': int(parameters[0]),
'reg_alpha': parameters[1],
'reg_lambda': parameters[2],
'subsample': parameters[3],
'colsample_bytree': parameters[4],
'seed': seed
}
experiment_clf = xgb.XGBClassifier(**params,
n_estimators = n_estimators)
 modelstart= time.time()
experiment_clf.fit(X_train,
y_train,
eval_set=[(X_valid,y_valid)],
early_stopping_rounds=100,
verbose=0)
 runtime =(time.time() - modelstart)/60
rounds = experiment_clf.best_ntree_limit
val_pred = experiment_clf.predict_proba(X_valid)[:,1]
score = experiment_clf.best_score
loss = metrics.log_loss(y_valid, val_pred)
 params['num_boost_round'] = rounds
gpyopt_output.append(
 [
loss,
score,
rounds,
params,
runtime,
experiment_clf.feature_importances_
] 
)
 return score
img = 'img_58997.jpg'
test_array = []
img_array = cv2.imread(os.path.join(test_directory,img),cv2.IMREAD_GRAYSCALE)
 test_array = img_array
plt.imshow(img_array, cmap='gray')
 plt.show()<train_model>
split_size = 0.4
n_estimators = 10000
metric = 'auc'
ESR = 175
X_train, X_valid, y_train, y_valid = train_test_split(
 X[feature_subset], y, test_size=split_size,
random_state=seed, shuffle=True,stratify=y)
 bds = [ {'name': 'max_depth', 'type': 'discrete', 'domain':(5, 15)},
{'name': 'reg_alpha', 'type': 'continuous', 'domain': (.05, 0.8)},
{'name': 'reg_lambda', 'type': 'continuous', 'domain': (.05, 0.8)},
{'name': 'subsample', 'type': 'continuous', 'domain':(0.1, 0.80)},
{'name': 'colsample_bytree', 'type': 'continuous', 'domain':(0.1, 0.80)},
]
sb_cat_cols = [x for x in cat_cols if x in feature_subset]
max_iter = 30
initial_iter = 5
gpyopt_output = []
gpy_importance = pd.DataFrame()
optimizer = BayesianOptimization(f=XGB_score,
domain=bds,
model_type='GP',
optimize_restarts = 1,
initial_design_numdata = initial_iter,
acquisition_type ='EI',
acquisition_jitter = 0.05,
exact_feval=True,
maximize=True)
 with timer("Bayesian Optimisation - {} Iterations".format(max_iter + initial_iter)) :
optimizer.run_optimization(max_iter=max_iter)
 results = pd.DataFrame(gpyopt_output,
columns = ['logloss','valid_auc',
'boosting_rounds','parameters', 'runtime', 'imp'] 
)
 gpyimp = results.imp.apply(lambda x: pd.Series(x)).T
gpyimp.index = feature_subset
og_cols = gpyimp.columns
gpyimp['mean'] = gpyimp[og_cols].mean(axis = 1)
 gpyimp['std'] = gpyimp[og_cols].std(axis = 1)
 del gpyopt_output; results.drop('imp', axis =1, inplace=True)
 results.to_csv("gpyopt_iterations_output.csv")
 best_params = results['parameters'].iloc[np.argmax(results.valid_auc)]
optimizer.plot_convergence()
results['TPM'] = results['boosting_rounds'] / results['runtime']
plt.plot(results['TPM'], '-rx')
 plt.xlabel("Bayesian Search Iteration")
 plt.ylabel("TPM")
 plt.title("Tree per Minute through Bayesian Search")
 plt.show()
print("Best AUC: {}".format(optimizer.fx_opt))
print("Best Parameters")
 pprint.pprint(best_params)
 results = pd.concat([results.drop('parameters',axis=1 ).reset_index(drop=True),
json_normalize(results['parameters'] ).reset_index(drop=True)
 ], axis = 1)
 del X_train, X_valid, y_train, y_valid
new_img = cv2.resize(test_array,(try_img_size2,try_img_size1))
plt.imshow(new_img,cmap='gray')
 plt.show()<train_model>
del results
training_data = []
i = 0
def create_training_data() :
for category in classes:
path = os.path.join(directory,category)
 class_num = classes.index(category)
 for img in os.listdir(path):
img_array = cv2.imread(os.path.join(path,img),cv2.IMREAD_GRAYSCALE)
 new_img = cv2.resize(img_array,(img_size2,img_size1))
ret, bin_img = cv2.threshold(new_img, 128, 255, cv2.THRESH_BINARY)
 training_data.append([
bin_img,class_num])
 sap_img = add_Salt_Pepper_Noise(bin_img, s_vs_p = 0.5, amount = 0.01)
 training_data.append([
sap_img,class_num])
 sap_img = add_Salt_Pepper_Noise(bin_img, s_vs_p = 0.5, amount = 0.03)
 training_data.append([
sap_img,class_num])
 sap_img = add_Salt_Pepper_Noise(bin_img, s_vs_p = 0.5, amount = 0.05)
 training_data.append([
sap_img,class_num])
 height,width = new_img.shape[:2]
center =(int(width/2), int(height/2))
angle = 20
M = cv2.getRotationMatrix2D(center, angle, 1)
 rotated_img = cv2.warpAffine(bin_img, M,(width, height))
training_data.append([
rotated_img,class_num])
 moving_x = -10
moving_y = -10
M = np.float32([[1, 0, moving_x], [0, 1, moving_y]])
 shifted_img = cv2.warpAffine(bin_img, M,(width, height))
training_data.append([
shifted_img,class_num] )<train_model>
feature_subset = keep_features
drop_cols = [x for x in X.columns if x not in keep_features]
X.drop(drop_cols, axis = 1, inplace= True)
 test.drop(drop_cols, axis = 1, inplace= True)
 gc.collect()
print("Train Shape: {} Rows, {} Cols
".format(*X[feature_subset].shape))
allmodelstart= time.time()
EPOCHS = 3
best_params['learning_rate'] = 0.1
kf = KFold(n_splits = EPOCHS, shuffle = True)
 y_preds = np.zeros(sample_submission.shape[0])
 y_oof = np.zeros(X.shape[0])
 f,ax = plt.subplots(1,3,figsize = [15,6])
 sb_cat_cols = [x for x in cat_cols if x in feature_subset]
all_feature_importance_df = pd.DataFrame()
mean_fpr = np.linspace(0,1,100)
 cms, tprs, aucs, y_real, y_proba,recalls, roc_aucs,f1_scores, accuracies, precisions = [],[],[],[],[],[],[],[],[],[]
for i,(tr_idx, val_idx)in enumerate(kf.split(X, y)) :
i += 1
modelstart = time.time()
sb_cat_cols = [x for x in cat_cols if x in feature_subset]
clf = xgb.XGBClassifier(**best_params,
n_estimators = n_estimators)
 modelstart= time.time()
clf.fit(X.iloc[tr_idx, :][feature_subset],
y.iloc[tr_idx],
eval_set=[(X.iloc[val_idx, :][feature_subset], y.iloc[val_idx])],
early_stopping_rounds=ESR,
verbose=500)
 runtime =(time.time() - modelstart)/60
rounds = clf.best_ntree_limit
y_oof[val_idx] = clf.predict_proba(X.iloc[val_idx, :][feature_subset])[:,1]
y_preds += clf.predict_proba(test[feature_subset])[:,1] / EPOCHS
label = 'valid_{}'.format(i)
 ax[0].plot(clf.evals_result_['validation_0']['auc'], label = label)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feature_subset
fold_importance_df["importance"] = clf.feature_importances_
all_feature_importance_df = pd.concat([all_feature_importance_df, fold_importance_df.sort_values(by = 'feature')], axis=0)
 print("Model Runtime: %0.2f Minutes"%(( time.time() - modelstart)/60))
del clf
roc_aucs.append(roc_auc_score(y.iloc[val_idx].values,y_oof[val_idx]))
accuracies.append(accuracy_score(y.iloc[val_idx].values,y_oof[val_idx].round()))
recalls.append(recall_score(y.iloc[val_idx].values,y_oof[val_idx].round()))
precisions.append(precision_score(y.iloc[val_idx].values,y_oof[val_idx].round()))
f1_scores.append(f1_score(y.iloc[val_idx].values,y_oof[val_idx].round()))
fpr, tpr, t = roc_curve(y.iloc[val_idx].values,y_oof[val_idx])
 tprs.append(interp(mean_fpr, fpr, tpr))
roc_auc = auc(fpr, tpr)
 aucs.append(roc_auc)
 ax[1].plot(fpr, tpr, lw=2, alpha=0.3, label='ROC fold %d(AUC = %0.4f)' %(i,roc_auc))
precision, recall, _ = precision_recall_curve(y.iloc[val_idx].values,y_oof[val_idx])
 y_real.append(y.iloc[val_idx].values)
 y_proba.append(y_oof[val_idx])
 ax[2].plot(recall, precision, lw=2, alpha=0.3, label='P|R fold %d' %(i))
cms.append(confusion_matrix(y.iloc[val_idx].values,y_oof[val_idx].round()))
ax[0].set_title("GPU XGB Metric Convergence over {} Folds".format(EPOCHS))
ax[0].set_ylabel("AUC")
 ax[0].set_xlabel("Boosting Rounds")
 ax[1].plot([0,1],[0,1],linestyle = '--',lw = 2,color = 'grey')
 mean_tpr = np.mean(tprs, axis=0)
 mean_auc = auc(mean_fpr, mean_tpr)
 ax[1].plot(mean_fpr, mean_tpr, color='blue',
label=r'Mean ROC(AUC = %0.4f)' %(np.mean(roc_aucs)) ,lw=2, alpha=1)
 ax[1].set_xlabel('False Positive Rate')
 ax[1].set_ylabel('True Positive Rate')
 ax[1].set_title('LGB ROC curve by folds')
 ax[1].legend(loc="lower right")
 ax[2].plot([0,1],[1,0],linestyle = '--',lw = 2,color = 'grey')
 y_real = np.concatenate(y_real)
 y_proba = np.concatenate(y_proba)
 precision, recall, _ = precision_recall_curve(y_real, y_proba)
 ax[2].plot(recall, precision, color='blue',
label=r'Mean P|R')
 ax[2].set_xlabel('Recall')
 ax[2].set_ylabel('Precision')
 ax[2].set_title('P|R curve by folds')
 ax[2].legend(loc="lower left")
 plt.tight_layout(pad=0)
 plt.savefig('model_eval.png')
 plt.show()
print(
 'CV roc score : {0:.4f}, std: {1:.4f}.'.format(np.mean(roc_aucs), np.std(roc_aucs)) ,
'
CV accuracy score : {0:.4f}, std: {1:.4f}.'.format(np.mean(accuracies), np.std(accuracies)) ,
'
CV recall score : {0:.4f}, std: {1:.4f}.'.format(np.mean(recalls), np.std(recalls)) ,
'
CV precision score : {0:.4f}, std: {1:.4f}.'.format(np.mean(precisions), np.std(precisions)) ,
'
CV f1 score : {0:.4f}, std: {1:.4f}.'.format(np.mean(f1_scores), np.std(f1_scores))
)
for category in classes:
path = os.path.join(directory,category)
 class_num = classes.index(category)
 for img in os.listdir(path):
img_array = cv2.imread(os.path.join(path,img),cv2.IMREAD_GRAYSCALE)
 new_img = cv2.resize(img_array,(img_size2,img_size1))
training_data.append([
new_img,class_num])
 print('path :', path)
 print('img :', img)
 print('img_array:', img_array)
 print('img_array_shape:', img_array.shape)
 print('new_img :', new_img)
 print('new_img_shape:', new_img.shape)
 print('class_num:', class_num)
 break<define_variables>
assert X[feature_subset].shape[1] == test[feature_subset].shape[1]
sample_submission['isFraud'] = y_preds
sample_submission.to_csv('{}_feats_{}fold_lgbm_gpu.csv'.format(len(feature_subset),EPOCHS))
training_data = []
i = 0<train_on_grid>
print("Notebook Runtime: %0.2f Hours"%(( time.time() - notebookstart)/60/60))