Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<train_model>
!rm -r /opt/conda/lib/python3.6/site-packages/lightgbm
!git clone --recursive https://github.com/Microsoft/LightGBM
!apt-get install -y -qq libboost-all-dev
for index, row in df_test.iterrows() :
filePath = f'.. /test/{index+1}.jpg'
saveDigit(row.values, filePath )<feature_engineering>
%%bash
cd LightGBM
rm -r build
mkdir build
cd build
cmake -DUSE_GPU=1 -DOpenCL_LIBRARY=/usr/local/cuda/lib64/libOpenCL.so -DOpenCL_INCLUDE_DIR=/usr/local/cuda/include/..
make -j$(nproc )
imgs = get_image_files(path_train )<set_options>
!cd LightGBM/python-package/;python3 setup.py install --precompile
Image.open(imgs[0] )<load_pretrained>
!mkdir -p /etc/OpenCL/vendors && echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
!rm -r LightGBM
!pip install -q 'pandas==0.25' --force-reinstall
!pip install GPyOpt
dls = db.dataloaders(path_train)
 dls.show_batch()<define_variables>
print("GPyOpt version:", GPyOpt.__version__)
 print("LGBM version:", lgb.__version__)
 print("Pandas version:", pd.__version__ )
xb,yb = dls.one_batch()
xb.shape,yb.shape<define_variables>
notebookstart = time.time()
warnings.filterwarnings("ignore")
 seed = 24
np.random.seed(seed)
 pd.set_option('display.max_columns', 500)
 pd.options.display.max_rows = 999
pd.set_option('max_colwidth', 500 )
dls.bs = 32<choose_model_class>
print("Define DF Schema.. ")
 target_var = 'isFraud'
schema = {
"TransactionDT": "int32",
"TransactionAmt": "float32",
"ProductCD": "object",
"card1": "int16",
"card2": "float32",
"card3": "float32",
"card4": "object",
"card5": "float32",
"card6": "object",
"addr1": "float32",
"addr2": "float32",
"dist1": "float32",
"dist2": "float32",
"P_emaildomain": "object",
"R_emaildomain": "object",
"C1": "float32",
"C2": "float32",
"C3": "float32",
"C4": "float32",
"C5": "float32",
"C6": "float32",
"C7": "float32",
"C8": "float32",
"C9": "float32",
"C10": "float32",
"C11": "float32",
"C12": "float32",
"C13": "float32",
"C14": "float32",
"D1": "float32",
"D2": "float32",
"D3": "float32",
"D4": "float32",
"D5": "float32",
"D6": "float32",
"D7": "float32",
"D8": "float32",
"D9": "float32",
"D10": "float32",
"D11": "float32",
"D12": "float32",
"D13": "float32",
"D14": "float32",
"D15": "float32",
"M1": "object",
"M2": "object",
"M3": "object",
"M4": "object",
"M5": "object",
"M6": "object",
"M7": "object",
"M8": "object",
"M9": "object",
"V1": "float32",
"V2": "float32",
"V3": "float32",
"V4": "float32",
"V5": "float32",
"V6": "float32",
"V7": "float32",
"V8": "float32",
"V9": "float32",
"V10": "float32",
"V11": "float32",
"V12": "float32",
"V13": "float32",
"V14": "float32",
"V15": "float32",
"V16": "float32",
"V17": "float32",
"V18": "float32",
"V19": "float32",
"V20": "float32",
"V21": "float32",
"V22": "float32",
"V23": "float32",
"V24": "float32",
"V25": "float32",
"V26": "float32",
"V27": "float32",
"V28": "float32",
"V29": "float32",
"V30": "float32",
"V31": "float32",
"V32": "float32",
"V33": "float32",
"V34": "float32",
"V35": "float32",
"V36": "float32",
"V37": "float32",
"V38": "float32",
"V39": "float32",
"V40": "float32",
"V41": "float32",
"V42": "float32",
"V43": "float32",
"V44": "float32",
"V45": "float32",
"V46": "float32",
"V47": "float32",
"V48": "float32",
"V49": "float32",
"V50": "float32",
"V51": "float32",
"V52": "float32",
"V53": "float32",
"V54": "float32",
"V55": "float32",
"V56": "float32",
"V57": "float32",
"V58": "float32",
"V59": "float32",
"V60": "float32",
"V61": "float32",
"V62": "float32",
"V63": "float32",
"V64": "float32",
"V65": "float32",
"V66": "float32",
"V67": "float32",
"V68": "float32",
"V69": "float32",
"V70": "float32",
"V71": "float32",
"V72": "float32",
"V73": "float32",
"V74": "float32",
"V75": "float32",
"V76": "float32",
"V77": "float32",
"V78": "float32",
"V79": "float32",
"V80": "float32",
"V81": "float32",
"V82": "float32",
"V83": "float32",
"V84": "float32",
"V85": "float32",
"V86": "float32",
"V87": "float32",
"V88": "float32",
"V89": "float32",
"V90": "float32",
"V91": "float32",
"V92": "float32",
"V93": "float32",
"V94": "float32",
"V95": "float32",
"V96": "float32",
"V97": "float32",
"V98": "float32",
"V99": "float32",
"V100": "float32",
"V101": "float32",
"V102": "float32",
"V103": "float32",
"V104": "float32",
"V105": "float32",
"V106": "float32",
"V107": "float32",
"V108": "float32",
"V109": "float32",
"V110": "float32",
"V111": "float32",
"V112": "float32",
"V113": "float32",
"V114": "float32",
"V115": "float32",
"V116": "float32",
"V117": "float32",
"V118": "float32",
"V119": "float32",
"V120": "float32",
"V121": "float32",
"V122": "float32",
"V123": "float32",
"V124": "float32",
"V125": "float32",
"V126": "float32",
"V127": "float32",
"V128": "float32",
"V129": "float32",
"V130": "float32",
"V131": "float32",
"V132": "float32",
"V133": "float32",
"V134": "float32",
"V135": "float32",
"V136": "float32",
"V137": "float32",
"V138": "float32",
"V139": "float32",
"V140": "float32",
"V141": "float32",
"V142": "float32",
"V143": "float32",
"V144": "float32",
"V145": "float32",
"V146": "float32",
"V147": "float32",
"V148": "float32",
"V149": "float32",
"V150": "float32",
"V151": "float32",
"V152": "float32",
"V153": "float32",
"V154": "float32",
"V155": "float32",
"V156": "float32",
"V157": "float32",
"V158": "float32",
"V159": "float32",
"V160": "float32",
"V161": "float32",
"V162": "float32",
"V163": "float32",
"V164": "float32",
"V165": "float32",
"V166": "float32",
"V167": "float32",
"V168": "float32",
"V169": "float32",
"V170": "float32",
"V171": "float32",
"V172": "float32",
"V173": "float32",
"V174": "float32",
"V175": "float32",
"V176": "float32",
"V177": "float32",
"V178": "float32",
"V179": "float32",
"V180": "float32",
"V181": "float32",
"V182": "float32",
"V183": "float32",
"V184": "float32",
"V185": "float32",
"V186": "float32",
"V187": "float32",
"V188": "float32",
"V189": "float32",
"V190": "float32",
"V191": "float32",
"V192": "float32",
"V193": "float32",
"V194": "float32",
"V195": "float32",
"V196": "float32",
"V197": "float32",
"V198": "float32",
"V199": "float32",
"V200": "float32",
"V201": "float32",
"V202": "float32",
"V203": "float32",
"V204": "float32",
"V205": "float32",
"V206": "float32",
"V207": "float32",
"V208": "float32",
"V209": "float32",
"V210": "float32",
"V211": "float32",
"V212": "float32",
"V213": "float32",
"V214": "float32",
"V215": "float32",
"V216": "float32",
"V217": "float32",
"V218": "float32",
"V219": "float32",
"V220": "float32",
"V221": "float32",
"V222": "float32",
"V223": "float32",
"V224": "float32",
"V225": "float32",
"V226": "float32",
"V227": "float32",
"V228": "float32",
"V229": "float32",
"V230": "float32",
"V231": "float32",
"V232": "float32",
"V233": "float32",
"V234": "float32",
"V235": "float32",
"V236": "float32",
"V237": "float32",
"V238": "float32",
"V239": "float32",
"V240": "float32",
"V241": "float32",
"V242": "float32",
"V243": "float32",
"V244": "float32",
"V245": "float32",
"V246": "float32",
"V247": "float32",
"V248": "float32",
"V249": "float32",
"V250": "float32",
"V251": "float32",
"V252": "float32",
"V253": "float32",
"V254": "float32",
"V255": "float32",
"V256": "float32",
"V257": "float32",
"V258": "float32",
"V259": "float32",
"V260": "float32",
"V261": "float32",
"V262": "float32",
"V263": "float32",
"V264": "float32",
"V265": "float32",
"V266": "float32",
"V267": "float32",
"V268": "float32",
"V269": "float32",
"V270": "float32",
"V271": "float32",
"V272": "float32",
"V273": "float32",
"V274": "float32",
"V275": "float32",
"V276": "float32",
"V277": "float32",
"V278": "float32",
"V279": "float32",
"V280": "float32",
"V281": "float32",
"V282": "float32",
"V283": "float32",
"V284": "float32",
"V285": "float32",
"V286": "float32",
"V287": "float32",
"V288": "float32",
"V289": "float32",
"V290": "float32",
"V291": "float32",
"V292": "float32",
"V293": "float32",
"V294": "float32",
"V295": "float32",
"V296": "float32",
"V297": "float32",
"V298": "float32",
"V299": "float32",
"V300": "float32",
"V301": "float32",
"V302": "float32",
"V303": "float32",
"V304": "float32",
"V305": "float32",
"V306": "float32",
"V307": "float32",
"V308": "float32",
"V309": "float32",
"V310": "float32",
"V311": "float32",
"V312": "float32",
"V313": "float32",
"V314": "float32",
"V315": "float32",
"V316": "float32",
"V317": "float32",
"V318": "float32",
"V319": "float32",
"V320": "float32",
"V321": "float32",
"V322": "float32",
"V323": "float32",
"V324": "float32",
"V325": "float32",
"V326": "float32",
"V327": "float32",
"V328": "float32",
"V329": "float32",
"V330": "float32",
"V331": "float32",
"V332": "float32",
"V333": "float32",
"V334": "float32",
"V335": "float32",
"V336": "float32",
"V337": "float32",
"V338": "float32",
"V339": "float32",
"id_01": "float32",
"id_02": "float32",
"id_03": "float32",
"id_04": "float32",
"id_05": "float32",
"id_06": "float32",
"id_07": "float32",
"id_08": "float32",
"id_09": "float32",
"id_10": "float32",
"id_11": "float32",
"id_12": "object",
"id_13": "float32",
"id_14": "float32",
"id_15": "object",
"id_16": "object",
"id_17": "float32",
"id_18": "float32",
"id_19": "float32",
"id_20": "float32",
"id_21": "float32",
"id_22": "float32",
"id_23": "object",
"id_24": "float32",
"id_25": "float32",
"id_26": "float32",
"id_27": "object",
"id_28": "object",
"id_29": "object",
"id_30": "object",
"id_31": "object",
"id_32": "float32",
"id_33": "object",
"id_34": "object",
"id_35": "object",
"id_36": "object",
"id_37": "object",
"id_38": "object",
"DeviceType": "object",
"DeviceInfo": "object",
"is_fraud":			 "int8"
}
emails = {'gmail': 'google',
'att.net': 'att',
'twc.com': 'spectrum',
'scranton.edu': 'other',
'optonline.net': 'other',
'hotmail.co.uk': 'microsoft',
'comcast.net': 'other',
'yahoo.com.mx': 'yahoo',
'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo',
'charter.net': 'spectrum',
'live.com': 'microsoft',
'aim.com': 'aol',
'hotmail.de': 'microsoft',
'centurylink.net': 'centurylink',
'gmail.com': 'google',
'me.com': 'apple',
'earthlink.net': 'other',
'gmx.de': 'other',
'web.de': 'other',
'cfl.rr.com': 'other',
'hotmail.com': 'microsoft',
'protonmail.com': 'other',
'hotmail.fr': 'microsoft',
'windstream.net': 'other',
'outlook.es': 'microsoft',
'yahoo.co.jp': 'yahoo',
'yahoo.de': 'yahoo',
'servicios-ta.com': 'other',
'netzero.net': 'other',
'suddenlink.net': 'other',
'roadrunner.com': 'other',
'sc.rr.com': 'other',
'live.fr': 'microsoft',
'verizon.net': 'yahoo',
'msn.com': 'microsoft',
'q.com': 'centurylink',
'prodigy.net.mx': 'att',
'frontier.com': 'yahoo',
'anonymous.com': 'other',
'rocketmail.com': 'yahoo',
'sbcglobal.net': 'att',
'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo',
'outlook.com': 'microsoft',
'mail.com': 'other',
'bellsouth.net': 'other',
'embarqmail.com': 'centurylink',
'cableone.net': 'other',
'hotmail.es': 'microsoft',
'mac.com': 'apple',
'yahoo.co.uk': 'yahoo',
'netzero.com': 'other',
'yahoo.com': 'yahoo',
'live.com.mx': 'microsoft',
'ptd.net': 'other',
'cox.net': 'other',
'aol.com': 'aol',
'juno.com': 'other',
'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']
learn = cnn_learner(dls, resnet50, metrics=accuracy ).to_fp16()<set_options>
@contextmanager
def timer(name):

 t0 = time.time()
yield
print('
[{}] done in {} Minutes
'.format(name, round(( time.time() - t0)/60,2)))
 def id_split(dataframe):
dataframe['device_name'] = dataframe['DeviceInfo'].str.split('/', expand=True)[0]
dataframe['device_version'] = dataframe['DeviceInfo'].str.split('/', expand=True)[1]
dataframe['OS_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[0]
dataframe['version_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[1]
dataframe['browser_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[0]
dataframe['version_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[1]
dataframe['screen_width'] = dataframe['id_33'].str.split('x', expand=True)[0]
dataframe['screen_height'] = dataframe['id_33'].str.split('x', expand=True)[1]
dataframe['id_34'] = dataframe['id_34'].str.split(':', expand=True)[1]
dataframe['id_23'] = dataframe['id_23'].str.split(':', expand=True)[1]
dataframe.loc[dataframe['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
dataframe.loc[dataframe['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
dataframe.loc[dataframe['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
dataframe.loc[dataframe['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
dataframe.loc[dataframe['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
dataframe.loc[dataframe['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
dataframe.loc[dataframe.device_name.isin(dataframe.device_name.value_counts() [dataframe.device_name.value_counts() < 200].index), 'device_name'] = "Others"
dataframe['had_id'] = 1
gc.collect()
return dataframe
def fraud_preprocessing(debug = None):
print("Starting Pre-Processing.. ")
 with timer("Load Tables"):
train_transaction = pd.read_csv('.. /input/train_transaction.csv',
index_col='TransactionID', nrows= debug, dtype = schema)
 test_transaction = pd.read_csv('.. /input/test_transaction.csv',
index_col='TransactionID', nrows= debug, dtype = schema)
 train_identity = pd.read_csv('.. /input/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/test_identity.csv', index_col='TransactionID')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv',
index_col='TransactionID',
nrows= debug)
 with timer("Merge Tables"):
train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print("Train Shape: {} Rows, {} Columns".format(*train.shape))
print("Test Shape: {} Rows, {} Columns".format(*test.shape))
y = train[target_var].copy()
del train_transaction, train_identity, test_transaction, test_identity
traindex = train.index
testdex = test.index
with timer("Train/Test Split Feature Engineering"):
train['TransactionAmt_to_mean_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_mean_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_std_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('std')
 train['TransactionAmt_to_std_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_mean_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_mean_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_std_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_std_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('std')
 train['id_02_to_mean_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('mean')
 train['id_02_to_mean_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('mean')
 train['id_02_to_std_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('std')
 train['id_02_to_std_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('std')
 test['id_02_to_mean_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('mean')
 test['id_02_to_mean_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('mean')
 test['id_02_to_std_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('std')
 test['id_02_to_std_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('std')
 train['D15_to_mean_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['D15_to_mean_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['TransactionAmt_Log'] = np.log(train['TransactionAmt'])
 test['TransactionAmt_Log'] = np.log(test['TransactionAmt'])
 for feature in ['card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'id_36']:
train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
for feature in ['id_01', 'id_31', 'id_33', 'id_36']:
train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))
test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 train = id_split(train)
 test = id_split(test)
 df = pd.concat([train.drop(target_var,axis=1),test],axis = 0)
 del train, test
with timer("Whole Feature Engineering"):
START_DATE = '2017-12-01'
startdate = datetime.datetime.strptime(START_DATE, '%Y-%m-%d')
 df = df.assign(
 TransactionAmt_decimal =(( df['TransactionAmt'] - df['TransactionAmt'].astype(int)) * 1000 ).astype(int),
card1_count_full = df['card1'].map(df['card1'].value_counts(dropna=False)) ,
Transaction_day_of_week = np.floor(( df['TransactionDT'] /(3600 * 24)- 1)% 7),
Transaction_hour = np.floor(df['TransactionDT'] / 3600)% 24,
TransactionDT = df['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds = x))), 
)
 df = df.assign(
 year = df['TransactionDT'].dt.year,
month = df['TransactionDT'].dt.month,
dow = df['TransactionDT'].dt.dayofweek,
quarter = df['TransactionDT'].dt.quarter,
hour = df['TransactionDT'].dt.hour,
day = df['TransactionDT'].dt.day,
all_group_nan_sum = df.isnull().sum(axis=1)/ df.shape[1],
all_group_0_count =(df == 0 ).astype(int ).sum(axis=1)/(df.shape[1] - df.isnull().sum(axis=1)) 
)
 prefix = ['C','D','Device','M','Transaction','V','addr','card','dist','id']
for i, p in enumerate(prefix):
column_set = [x for x in df.columns.tolist() if x.startswith(prefix[i])]
df[p + "group_nan_sum"] = df[column_set].isnull().sum(axis=1)/ df[column_set].shape[1]
numeric_cols = [x for x in column_set if df[x].dtype != object]
if numeric_cols:
df[p + "group_sum"] = df[column_set].sum(axis=1)
 df[p + "group_mean"] = df[column_set].mean(axis=1)
 df[p + "group_0_count"] =(df[column_set] == 0 ).astype(int ).sum(axis=1)/(df[column_set].shape[1] - df[p + "group_nan_sum"])
 with timer("Label Encode"):
categorical_cols = []
for f in df.columns:
if df[f].dtype=='object':
categorical_cols += [f]
lbl = preprocessing.LabelEncoder()
df[f] = lbl.fit_transform(df[f].astype(str))
print("Total Shape: {} Rows, {} Columns".format(*df.shape))
return df, y, traindex, testdex, categorical_cols, sample_submission
learn.fine_tune(6, freeze_epochs=3 )<predict_on_test>
max_iter = 35
initial_iter = 5
DEBUG = None
df, y, traindex, testdex, cat_cols, sample_submission = fraud_preprocessing(debug = DEBUG )
test_dl = dls.test_dl(get_image_files(path_test))
class_score, y = learn.get_preds(dl=test_dl )<prepare_output>
df['yrmth'] = df.year.astype(str)+ df.month.map("{:02}".format)
 df['Fraud'] = np.nan
df.loc[traindex,'Fraud'] = y[traindex]
df['traintest'] = 'Test'
df.loc[df.Fraud.notnull() ,'traintest'] = 'Train'
class_score = np.argmax(class_score, axis=1)
<define_variables>
print("Are there redundant Transaction IDs?")
 print(df.index.value_counts().value_counts() )
predicted_classes = [dls.vocab[i] for i in class_score]
predicted_classes[:10]<save_to_csv>
X = df.loc[traindex,:]
feature_subset = X.columns.tolist()
test = df.loc[testdex,:]
split_size = 0.35
n_estimators = 10000
metric = 'auc'
ESR = 150
feature_subset = [x for x in X.columns.tolist() if x not in ['TransactionDT','Fraud', 'traintest', 'yrmth']]
X_train, X_valid, y_train, y_valid = train_test_split(
 X.loc[:,feature_subset], y, test_size=split_size,
random_state=seed, shuffle=False)
 del df ; gc.collect() ;
print("None Fraud: {}%, Fraud: {}%".format(*y.value_counts(normalize=True)))
 print("Randomness Score AUC: {}".format(
 metrics.roc_auc_score(y,np.array([y.value_counts(normalize=True)[0]]*y.shape[0]))))
output = pd.DataFrame({'ImageId': image_id_list, 'Label': predicted_classes})
 output.ImageId = output.ImageId.astype(int)
 output = output.sort_values(by='ImageId', ignore_index=True)
 output.to_csv('submission.csv', index=False)
 output.head()<set_options>
bds = [ {'name': 'min_data_in_leaf', 'type': 'continuous', 'domain':(2, 100)},
{'name': 'num_leaves', 'type': 'continuous', 'domain':(20, 1000)},
{'name': 'subsample_for_bin', 'type': 'continuous', 'domain':(1000, 5000)},
{'name': 'min_sum_hessian_in_leaf', 'type': 'continuous', 'domain':(0, 15)},
{'name': 'reg_alpha', 'type': 'continuous', 'domain':(0, 3)},
{'name': 'reg_lambda', 'type': 'continuous', 'domain':(0, 3)},
{'name': 'bagging_fraction', 'type': 'continuous', 'domain': (.01, 1)},
{'name': 'feature_fraction', 'type': 'continuous', 'domain': (.01, 1)}
]
sb_cat_cols = [x for x in cat_cols if x in feature_subset]
lgtrain = lgb.Dataset(X_train, y_train, categorical_feature = sb_cat_cols, free_raw_data=False)
 lgvalid = lgb.Dataset(X_valid, y_valid, categorical_feature = sb_cat_cols, free_raw_data=False)
 def lgb_score(para):
parameters = para[0]
modelstart= time.time()
params = {
'objective': 'binary',
'metric': metric,
'learning_rate': 0.05,
'device': 'gpu',
'tree_learner': 'serial',
'boost_from_average': 'true',
'num_boost_round': n_estimators,
'boosting_type': 'gbdt',
'max_depth': -1,
'num_leaves': int(parameters[1]),
'subsample_for_bin': int(parameters[2]),
'min_data_in_leaf': int(parameters[0]),
'min_sum_hessian_in_leaf': parameters[3],
'reg_alpha': parameters[4],
'reg_lambda': parameters[5],
'bagging_fraction': parameters[6],
'bagging_freq': 1,
'feature_fraction': parameters[7],
'is_unbalance': False,
'nthread': -1,
'verbose': -1,
'seed': seed
}
experiment_lgb = lgb.train(
 params = params,
train_set = lgtrain,
valid_sets=[lgtrain, lgvalid],
valid_names=['train','valid'],
verbose_eval= 0,
early_stopping_rounds= ESR 
)
 runtime =(time.time() - modelstart)/60
val_pred = experiment_lgb.predict(X_valid)
 score = experiment_lgb.best_score['valid'][metric]
loss = metrics.log_loss(y_valid, val_pred)
 params['num_boost_round'] = experiment_lgb.best_iteration
gpyopt_output.append(
 [
loss,
experiment_lgb.best_score['train'][metric],
score,
experiment_lgb.best_iteration,
params,
runtime
] 
)
 return score
%matplotlib inline
np.random.seed(2)
 sns.set(style = 'white', context= 'notebook', palette = 'deep' )<load_from_csv>
gpyopt_output = []
optimizer = BayesianOptimization(f=lgb_score,
domain=bds,
model_type='GP',
optimize_restarts = 1,
initial_design_numdata = initial_iter,
acquisition_type ='EI',
acquisition_jitter = 0.1,
exact_feval=True,
maximize=True)
 with timer("Bayesian Optimisation - {} Iterations".format(max_iter + initial_iter)) :
optimizer.run_optimization(max_iter=max_iter )
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test = pd.read_csv('.. /input/digit-recognizer/test.csv' )<prepare_x_and_y>
results = pd.DataFrame(gpyopt_output,
columns = ['logloss','train_auc','valid_auc',
'boosting_rounds','parameters', 'runtime'] 
)
 results.to_csv("gpyopt_iterations_output.csv")
 best_params = results['parameters'].iloc[np.argmax(results.valid_auc)]
results = results.reset_index().rename(columns = {'index':'iteration'})
 results['iteration'] = results['iteration'] + 1
optimizer.plot_convergence()
print("Best AUC: {}".format(optimizer.fx_opt))
print("Best Parameters")
 pprint.pprint(best_params)
 results = pd.concat([results.drop('parameters',axis=1 ).reset_index(drop=True),
json_normalize(results['parameters'] ).reset_index(drop=True)
 ], axis = 1)
 results['TPM'] = results['boosting_rounds'] / results['runtime']
results['Total Leaves'] = results['boosting_rounds'] * results['num_leaves']
Y_train = train['label']
X_train = train.drop(labels = ['label'], axis = 1)
 del train
g = sns.countplot(Y_train)
 g<feature_engineering>
bds
X_train = X_train/255.0
test = test/255.0<train_model>
results['TPM'] = results['boosting_rounds'] / results['runtime']
results['Total Leaves'] = results['boosting_rounds'] * results['num_leaves']
results.head()
X_train = X_train.values.reshape(-1,28,28,1)
 test= test.values.reshape(-1,28,28,1 )<categorify>
allmodelstart= time.time()
EPOCHS = 5
del best_params['num_boost_round']
best_params['learning_rate'] = 0.01
kf = KFold(n_splits = EPOCHS, shuffle = False)
 y_preds = np.zeros(sample_submission.shape[0])
 y_preds_fold = np.zeros([sample_submission.shape[0],EPOCHS])
 y_oof = np.zeros(X.shape[0])
 f,ax = plt.subplots(1,3,figsize = [15,6])
 sb_cat_cols = [x for x in cat_cols if x in feature_subset]
all_feature_importance_df = pd.DataFrame()
mean_fpr = np.linspace(0,1,100)
 cms, tprs, aucs, y_real, y_proba,recalls, roc_aucs,f1_scores, accuracies, precisions = [],[],[],[],[],[],[],[],[],[]
for i,(tr_idx, val_idx)in enumerate(kf.split(X, y)) :
i += 1
best_params[seed] = i
modelstart= time.time()
evals_result = {}
lgtrain = lgb.Dataset(X.iloc[tr_idx, :][feature_subset], y.iloc[tr_idx], categorical_feature = sb_cat_cols)
 lgvalid = lgb.Dataset(X.iloc[val_idx, :][feature_subset], y.iloc[val_idx], categorical_feature = sb_cat_cols)
 clf = lgb.train(
 best_params,
lgtrain,
valid_sets=[lgtrain, lgvalid],
valid_names=['train','valid'],
verbose_eval=300,
num_boost_round = n_estimators,
early_stopping_rounds=ESR,
evals_result=evals_result, 
)
 y_oof[val_idx] = clf.predict(X.iloc[val_idx, :][feature_subset])
 y_preds_fold[:,i-1] = clf.predict(test.loc[:,feature_subset])
 y_preds += y_preds_fold[:,i-1] / EPOCHS
evals_result['train_{}'.format(i)] = evals_result.pop('train')
 evals_result['valid_{}'.format(i)] = evals_result.pop('valid')
 lgb.plot_metric(evals_result, metric=metric, ax = ax[0])
 ax[0].set_title("GPU LGBM Metric Convergence over {} Folds".format(EPOCHS))
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feature_subset
fold_importance_df["importance"] = clf.feature_importance()
all_feature_importance_df = pd.concat([all_feature_importance_df, fold_importance_df], axis=0)
 print("
Model Runtime: %0.2f Minutes"%(( time.time() - modelstart)/60))
roc_aucs.append(roc_auc_score(y.iloc[val_idx].values,y_oof[val_idx]))
accuracies.append(accuracy_score(y.iloc[val_idx].values,y_oof[val_idx].round()))
recalls.append(recall_score(y.iloc[val_idx].values,y_oof[val_idx].round()))
precisions.append(precision_score(y.iloc[val_idx].values,y_oof[val_idx].round()))
f1_scores.append(f1_score(y.iloc[val_idx].values,y_oof[val_idx].round()))
fpr, tpr, t = roc_curve(y.iloc[val_idx].values,y_oof[val_idx])
 tprs.append(interp(mean_fpr, fpr, tpr))
roc_auc = auc(fpr, tpr)
 aucs.append(roc_auc)
 ax[1].plot(fpr, tpr, lw=2, alpha=0.3, label='ROC fold %d(AUC = %0.4f)' %(i,roc_auc))
precision, recall, _ = precision_recall_curve(y.iloc[val_idx].values,y_oof[val_idx])
 y_real.append(y.iloc[val_idx].values)
 y_proba.append(y_oof[val_idx])
 ax[2].plot(recall, precision, lw=2, alpha=0.3, label='P|R fold %d' %(i))
cms.append(confusion_matrix(y.iloc[val_idx].values,y_oof[val_idx].round()))
ax[1].plot([0,1],[0,1], linestyle = '--', lw = 2, color = 'grey')
 mean_tpr = np.mean(tprs, axis=0)
 mean_auc = auc(mean_fpr, mean_tpr)
 ax[1].plot(mean_fpr, mean_tpr, color='blue',
label=r'Mean ROC(AUC = %0.4f)' %(np.mean(roc_aucs)) , lw=2, alpha=1)
 ax[1].set_xlabel('False Positive Rate')
 ax[1].set_ylabel('True Positive Rate')
 ax[1].set_title('LGB ROC curve by folds')
 ax[1].legend(loc="lower right")
 ax[2].plot([0,1],[1,0],linestyle = '--',lw = 2,color = 'grey')
 y_real = np.concatenate(y_real)
 y_proba = np.concatenate(y_proba)
 precision, recall, _ = precision_recall_curve(y_real, y_proba)
 ax[2].plot(recall, precision, color='blue',
label=r'Mean P|R')
 ax[2].set_xlabel('Recall')
 ax[2].set_ylabel('Precision')
 ax[2].set_title('P|R curve by folds')
 ax[2].legend(loc="lower left")
 plt.tight_layout(pad=0)
 plt.savefig('model_eval.png')
 plt.show()
print(
 'CV roc score : {0:.4f}, std: {1:.4f}.'.format(np.mean(roc_aucs), np.std(roc_aucs)) ,
'
CV accuracy score : {0:.4f}, std: {1:.4f}.'.format(np.mean(accuracies), np.std(accuracies)) ,
'
CV recall score : {0:.4f}, std: {1:.4f}.'.format(np.mean(recalls), np.std(recalls)) ,
'
CV precision score : {0:.4f}, std: {1:.4f}.'.format(np.mean(precisions), np.std(precisions)) ,
'
CV f1 score : {0:.4f}, std: {1:.4f}.'.format(np.mean(f1_scores), np.std(f1_scores))
)
Y_train = to_categorical(Y_train,num_classes= 10 )<define_variables>
assert X[feature_subset].shape[1] == test[feature_subset].shape[1]
sample_submission['isFraud'] = y_preds
sample_submission.to_csv('{}_feats_{}fold_lgbm_gpu.csv'.format(len(feature_subset),EPOCHS))
<split><EOS> 
print("Notebook Runtime: %0.2f Hours"%(( time.time() - notebookstart)/60/60))