Kaggle competition: IEEE-CIS Fraud Detection
<SOS> metric: AUC Kaggle data source: ieee-cis-fraud-detection<categorify>
import pandas as pd
import scipy as sp
import numpy as np
from scipy import stats
import matplotlib.pyplot as plt
import seaborn as sns
import datetime
from sklearn.preprocessing import LabelEncoder
from sklearn import preprocessing
from sklearn.metrics import confusion_matrix, roc_auc_score
from sklearn.model_selection import StratifiedKFold, cross_val_score, KFold
from xgboost import XGBClassifier
import xgboost as xgb
from sklearn.model_selection import RandomizedSearchCV, GridSearchCV
import lightgbm as lgb
import os
import gc
def prediction(test_img_path, model, img_size, device):
id_list = []
pred_list = []
with torch.no_grad() :
for path in tqdm(test_img_path):
img = Image.open(path)
 _id = int(path.split('/')[-1].split('.')[0])
 transform = ImageTransform(img_size)
 img = transform(img, phase='test')
 img = img.unsqueeze(0)
 img = img.to(device)
 model.eval()
outputs = model(img)
 preds = F.softmax(outputs, dim=1)[:, 1].tolist()
id_list.append(_id)
 pred_list.append(preds[0])
 res = pd.DataFrame({
'id': id_list,
'label': pred_list
})
 res.sort_values(by='id', inplace=True)
 res.reset_index(drop=True, inplace=True)
 res.to_csv('submission.csv', index=False)
 return res<predict_on_test>
train_identity = pd.read_csv('.. /input/ieee-fraud-detection/train_identity.csv')
 train_transaction = pd.read_csv('.. /input/ieee-fraud-detection/train_transaction.csv')
 test_identity = pd.read_csv('.. /input/ieee-fraud-detection/test_identity.csv')
 test_transaction = pd.read_csv('.. /input/ieee-fraud-detection/test_transaction.csv')
 sub = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv')
 train = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')
 test = pd.merge(test_transaction, test_identity, on='TransactionID', how='left' )
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
 res = prediction(test_img_path, model, img_size, device )<set_options>
train.isFraud.value_counts()
%matplotlib inline 
<import_modules>
not_fraud=train[train.isFraud==0]
fraud=train[train.isFraud==1]
not_fraud_downsampled = resample(not_fraud,
replace = False,
n_samples = 400000,
random_state = 27)
 downsampled = pd.concat([not_fraud_downsampled, fraud])
 downsampled.isFraud.value_counts()
torch.__version__<define_variables>
train = downsampled.copy()
del not_fraud_downsampled, downsampled, train_identity, test_identity, train_transaction, test_transaction
base_dir = '.. /input/dogs-vs-cats-redux-kernels-edition'
train_dir = '.. /data/train'
test_dir = '.. /data/test'<load_pretrained>
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test )
with zipfile.ZipFile(os.path.join(base_dir, 'train.zip')) as train_zip:
train_zip.extractall('.. /data')
 with zipfile.ZipFile(os.path.join(base_dir, 'test.zip')) as test_zip:
test_zip.extractall('.. /data' )<define_variables>
train["lastest_browser"] = np.zeros(train.shape[0])
 test["lastest_browser"] = np.zeros(test.shape[0])
 def setBrowser(df):
df.loc[df["id_31"]=="samsung browser 7.0",'lastest_browser']=1
df.loc[df["id_31"]=="opera 53.0",'lastest_browser']=1
df.loc[df["id_31"]=="mobile safari 10.0",'lastest_browser']=1
df.loc[df["id_31"]=="google search application 49.0",'lastest_browser']=1
df.loc[df["id_31"]=="firefox 60.0",'lastest_browser']=1
df.loc[df["id_31"]=="edge 17.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 69.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 67.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 63.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 63.0 for ios",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 64.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 64.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 64.0 for ios",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 65.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 65.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 65.0 for ios",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 66.0",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 66.0 for android",'lastest_browser']=1
df.loc[df["id_31"]=="chrome 66.0 for ios",'lastest_browser']=1
return df
train=setBrowser(train)
 test=setBrowser(test )
train_list = glob.glob(os.path.join(train_dir, '*.jpg'))
test_list = glob.glob(os.path.join(test_dir, '*.jpg'))<count_values>
def setDevice(df):
df['DeviceInfo'] = df['DeviceInfo'].cat.add_categories('unknown_device')
 df['DeviceInfo'] = df['DeviceInfo'].fillna('unknown_device' ).str.lower()
df['device_name'] = df['DeviceInfo'].str.split('/', expand=True)[0]
df.loc[df['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
df.loc[df['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
df.loc[df['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
df.loc[df['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
df.loc[df['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
df.loc[df['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
df.loc[df['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
df.loc[df['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
df.loc[df['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
df.loc[df['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
df.loc[df['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
df['had_id'] = 1
gc.collect()
return df
train=setDevice(train)
 test=setDevice(test )
len(train_list )<split>
for col in tqdm_notebook(train.columns):
if train[col].dtype == 'object':
le = LabelEncoder()
le.fit(list(train[col].astype(str ).values)+ list(test[col].astype(str ).values))
train[col] = le.transform(list(train[col].astype(str ).values))
test[col] = le.transform(list(test[col].astype(str ).values))
train_list, val_list = train_test_split(train_list, test_size=0.1 )<normalization>
train['d2_d4_comb'] = train['D2'].astype(str)+ '_' +train['D4'].astype(str)
 test['d2_d4_comb'] = test['D2'].astype(str)+ '_' +test['D4'].astype(str )
class ImageTransform() :
def __init__(self, resize, mean, std):
self.data_transform = {
'train': transforms.Compose([
transforms.RandomResizedCrop(resize, scale=(0.5, 1.0)) ,
transforms.RandomHorizontalFlip() ,
transforms.ToTensor() ,
transforms.Normalize(mean, std)
 ]),
'val': transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(resize),
transforms.ToTensor() ,
transforms.Normalize(mean, std)
 ])
 }
def __call__(self, img, phase):
return self.data_transform[phase](img )<categorify>
for feature in ['id_02__id_20', 'id_02__D8', 'D11__DeviceInfo', 'DeviceInfo__P_emaildomain', 'P_emaildomain__C2',
'card2__dist1', 'card1__card5', 'card2__id_20', 'card5__P_emaildomain', 'addr1__card1']:
f1, f2 = feature.split('__')
 train[feature] = train[f1].astype(str)+ '_' + train[f2].astype(str)
 test[feature] = test[f1].astype(str)+ '_' + test[f2].astype(str)
 le = LabelEncoder()
le.fit(list(train[feature].astype(str ).values)+ list(test[feature].astype(str ).values))
train[feature] = le.transform(list(train[feature].astype(str ).values))
test[feature] = le.transform(list(test[feature].astype(str ).values))
class DogvsCatDataset(data.Dataset):
def __init__(self, file_list, transform=None, phase='train'):
self.file_list = file_list
self.transform = transform
self.phase = phase
def __len__(self):
return len(self.file_list)
 def __getitem__(self, idx):
img_path = self.file_list[idx]
img = Image.open(img_path)
 img_transformed = self.transform(img, self.phase)
 label = img_path.split('/')[-1].split('.')[0]
if label == 'dog':
label = 1
elif label == 'cat':
label = 0
return img_transformed, label<set_options>
def uid_fe(df):
df['uid'] = df['card1'].astype(str)+'_'+df['card2'].astype(str)
 df['uid2'] = df['uid'].astype(str)+'_'+df['card3'].astype(str)+'_'+df['card5'].astype(str)
 df['uid3'] = df['uid2'].astype(str)+'_'+df['addr1'].astype(str)+'_'+df['addr2'].astype(str)
 df['uid4'] = df['uid3'].astype(str)+'_'+df['P_emaildomain'].astype(str)
 df['uid5'] = df['uid3'].astype(str)+'_'+df['R_emaildomain'].astype(str)
 return df
train = uid_fe(train)
 test = uid_fe(test )
size = 224
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 batch_size = 32
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu" )<categorify>
def andrew_fe(df):
df['TransactionAmt_to_mean_card1'] = df['TransactionAmt'] / df.groupby(['card1'])['TransactionAmt'].transform('mean')
 df['TransactionAmt_to_mean_card4'] = df['TransactionAmt'] / df.groupby(['card4'])['TransactionAmt'].transform('mean')
 df['TransactionAmt_to_std_card1'] = df['TransactionAmt'] / df.groupby(['card1'])['TransactionAmt'].transform('std')
 df['TransactionAmt_to_std_card4'] = df['TransactionAmt'] / df.groupby(['card4'])['TransactionAmt'].transform('std')
 df['id_02_to_mean_card1'] = df['id_02'] / df.groupby(['card1'])['id_02'].transform('mean')
 df['id_02_to_mean_card4'] = df['id_02'] / df.groupby(['card4'])['id_02'].transform('mean')
 df['id_02_to_std_card1'] = df['id_02'] / df.groupby(['card1'])['id_02'].transform('std')
 df['id_02_to_std_card4'] = df['id_02'] / df.groupby(['card4'])['id_02'].transform('std')
 df['D15_to_mean_card1'] = train['D15'] / df.groupby(['card1'])['D15'].transform('mean')
 df['D15_to_mean_card4'] = train['D15'] / df.groupby(['card4'])['D15'].transform('mean')
 df['D15_to_std_card1'] = train['D15'] / df.groupby(['card1'])['D15'].transform('std')
 df['D15_to_std_card4'] = train['D15'] / df.groupby(['card4'])['D15'].transform('std')
 df['D15_to_mean_addr1'] = df['D15'] / df.groupby(['addr1'])['D15'].transform('mean')
 df['D15_to_mean_addr2'] = df['D15'] / df.groupby(['addr2'])['D15'].transform('mean')
 df['D15_to_std_addr1'] = df['D15'] / df.groupby(['addr1'])['D15'].transform('std')
 df['D15_to_std_addr2'] = df['D15'] / df.groupby(['addr2'])['D15'].transform('std')
 return df
train = andrew_fe(train)
 test = andrew_fe(test )
train_dataset = DogvsCatDataset(train_list, transform=ImageTransform(size, mean, std), phase='train')
 val_dataset = DogvsCatDataset(val_list, transform=ImageTransform(size, mean, std), phase='val')
 print('Operation Check')
 index = 0
print(train_dataset.__getitem__(index)[0].size())
 print(train_dataset.__getitem__(index)[1] )<load_pretrained>
def product_fe(df):
df["card1_count"] = df.groupby(["card1"])["ProductCD"].transform("size")
 df["card2_count"] = df.groupby(["card2"])["ProductCD"].transform("size")
 df["card5_count"] = df.groupby(["card5"])["ProductCD"].transform("size")
 df["R_email_count"] = df.groupby(["R_emaildomain"])["ProductCD"].transform("size")
 df["P_email_count"] = df.groupby(["P_emaildomain"])["ProductCD"].transform("size")
 df["P_R_emails_count"] = df.groupby(["P_emaildomain","R_emaildomain"])["ProductCD"].transform("size")
 df["addr1_count"] = df.groupby(["addr1"])["ProductCD"].transform("size")
 df["addr2_count"] = df.groupby(["addr2"])["ProductCD"].transform("size")
 df["joint_addr1_card1_count"] = df.groupby(["addr1","card1"])["ProductCD"].transform("size")
 df["joint_matches_count"] = df.groupby(["M4","M5", "M6"])["ProductCD"].transform("size")
 df["joint_d2_d4_count"] = df.groupby(['d2_d4_comb'])["ProductCD"].transform("size")
 df["joint_card1_d2_d4_count"] = df.groupby(["card1","d2_d4_comb"])["ProductCD"].transform("size")
 df["uid_pd5_count"] = df.groupby(["uid5"])["ProductCD"].transform("size")
 df["uid_pd5_count"] = df.groupby(["uid5"])["ProductCD"].transform("size")
 return df
train = product_fe(train)
 test = product_fe(test )
train_dataloader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
 val_dataloader = data.DataLoader(val_dataset, batch_size=batch_size, shuffle=False)
 dataloader_dict = {'train': train_dataloader, 'val': val_dataloader}
print('Operation Check')
 batch_iterator = iter(train_dataloader)
 inputs, label = next(batch_iterator)
 print(inputs.size())
 print(label )<choose_model_class>
def decimal_fe(df):
df['TransactionAmt_decimal'] =(( df['TransactionAmt'] - df['TransactionAmt'].astype(int)) * 1000 ).astype(int)
 df['TransactionAmt_decimal_to_mean_card1'] = df['TransactionAmt_decimal'] / train.groupby(['card1'])['TransactionAmt_decimal'].transform('mean')
 df['TransactionAmt_decimal_to_mean_card2'] = df['TransactionAmt_decimal'] / train.groupby(['card2'])['TransactionAmt_decimal'].transform('mean')
 df['TransactionAmt_decimal_to_mean_card4'] = df['TransactionAmt_decimal'] / train.groupby(['card4'])['TransactionAmt_decimal'].transform('mean')
 df['TransactionAmt_decimal_to_std_card1'] = df['TransactionAmt_decimal'] / train.groupby(['card1'])['TransactionAmt_decimal'].transform('std')
 df['TransactionAmt_decimal_to_std_card2'] = df['TransactionAmt_decimal'] / train.groupby(['card2'])['TransactionAmt_decimal'].transform('std')
 df['TransactionAmt_decimal_to_std_card4'] = df['TransactionAmt_decimal'] / train.groupby(['card4'])['TransactionAmt_decimal'].transform('std')
 return df
train = decimal_fe(train)
 test = decimal_fe(test)
use_pretrained = True
net = models.vgg16(pretrained=use_pretrained)
 print(net )<choose_model_class>
train.drop(labels=['TransactionAmt_decimal'],axis=1,inplace=True)
 test.drop(labels=['TransactionAmt_decimal'],axis=1,inplace=True )
net.classifier[6] = nn.Linear(in_features=4096, out_features=2)
 print('Done' )<find_best_params>
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us' )
params_to_update = []
update_params_name = ['classifier.6.weight', 'classifier.6.bias']
for name, param in net.named_parameters() :
if name in update_params_name:
param.requires_grad = True
params_to_update.append(param)
 print(name)
 else:
param.requires_grad = False<choose_model_class>
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d' )
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(params=params_to_update, lr=0.001, momentum=0.9 )<train_model>
def setTime(df):
df['DT'] = df['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 df['DT_M'] =(df['DT'].dt.year-2017)*12 + df['DT'].dt.month
df['DT_W'] =(df['DT'].dt.year-2017)*52 + df['DT'].dt.weekofyear
df['DT_D'] =(df['DT'].dt.year-2017)*365 + df['DT'].dt.dayofyear
df['DT_hour'] = df['DT'].dt.hour
df['DT_day_week'] = df['DT'].dt.dayofweek
df['DT_day'] = df['DT'].dt.day
df['TransactionAmt_to_mean_hour'] = df['TransactionAmt'] / df.groupby(['DT_hour'])['TransactionAmt'].transform('mean')
 df['TransactionAmt_to_mean_dow'] = df['TransactionAmt'] / df.groupby(['DT_day_week'])['TransactionAmt'].transform('mean')
 df['TransactionAmt_to_std_dow'] = df['TransactionAmt'] / df.groupby(['DT_day_week'])['TransactionAmt'].transform('std')
 df['TransactionAmt_to_std_hour'] = df['TransactionAmt'] / df.groupby(['DT_hour'])['TransactionAmt'].transform('std')
 return df
train = setTime(train)
 test= setTime(test )
def train_model(net, dataloader_dict, criterion, optimizer, num_epoch):
since = time.time()
best_model_wts = copy.deepcopy(net.state_dict())
 best_acc = 0.0
net = net.to(device)
 for epoch in range(num_epoch):
print('Epoch {}/{}'.format(epoch + 1, num_epoch))
print('-'*20)
 for phase in ['train', 'val']:
if phase == 'train':
net.train()
else:
net.eval()
epoch_loss = 0.0
epoch_corrects = 0
for inputs, labels in tqdm(dataloader_dict[phase]):
inputs = inputs.to(device)
 labels = labels.to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
outputs = net(inputs)
 _, preds = torch.max(outputs, 1)
 loss = criterion(outputs, labels)
 if phase == 'train':
loss.backward()
optimizer.step()
epoch_loss += loss.item() * inputs.size(0)
 epoch_corrects += torch.sum(preds == labels.data)
 epoch_loss = epoch_loss / len(dataloader_dict[phase].dataset)
 epoch_acc = epoch_corrects.double() / len(dataloader_dict[phase].dataset)
 print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))
if phase == 'val' and epoch_acc > best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(net.state_dict())
 time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
 time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))
net.load_state_dict(best_model_wts)
 return net<train_model>
for col in ['DT_M','DT_W','DT_D']:
temp_df = pd.concat([train[[col]], test[[col]]])
 fq_encode = temp_df[col].value_counts().to_dict()
train[col+'_total'] = train[col].map(fq_encode)
 test[col+'_total'] = test[col].map(fq_encode )
num_epoch = 2
net = train_model(net, dataloader_dict, criterion, optimizer, num_epoch )<categorify>
periods = ['DT_M','DT_W','DT_D']
i_cols = ['uid']
for period in periods:
for col in i_cols:
new_column = col + '_' + period
temp_df = pd.concat([train[[col,period]], test[[col,period]]])
 temp_df[new_column] = temp_df[col].astype(str)+ '_' +(temp_df[period] ).astype(str)
 fq_encode = temp_df[new_column].value_counts().to_dict()
train[new_column] =(train[col].astype(str)+ '_' + train[period].astype(str)).map(fq_encode)
 test[new_column] =(test[col].astype(str)+ '_' + test[period].astype(str)).map(fq_encode)
 train[new_column] /= train[period+'_total']
test[new_column] /= test[period+'_total']
id_list = []
pred_list = []
with torch.no_grad() :
for test_path in tqdm(test_list):
img = Image.open(test_path)
 _id = int(test_path.split('/')[-1].split('.')[0])
 transform = ImageTransform(size, mean, std)
 img = transform(img, phase='val')
 img = img.unsqueeze(0)
 img = img.to(device)
 net.eval()
outputs = net(img)
 preds = F.softmax(outputs, dim=1)[:, 1].tolist()
id_list.append(_id)
 pred_list.append(preds[0])
 res = pd.DataFrame({
'id': id_list,
'label': pred_list
})
 res.sort_values(by='id', inplace=True)
 res.reset_index(drop=True, inplace=True)
 res.to_csv('submission.csv', index=False )<load_from_csv>
i_cols = ['card1','card2','card3','card5',
'C1','C2','C4','C5','C6','C7','C8','C9','C10','C11','C12','C13','C14',
'D1','D2','D3','D4','D5','D6','D7','D8',
'addr1','addr2',
'dist1',
'P_emaildomain', 'R_emaildomain',
'DeviceInfo',
'id_30','id_33',
'uid','uid2','uid3','uid4','uid5'
]
for col in i_cols:
temp_df = pd.concat([train[[col]], test[[col]]])
 fq_encode = temp_df[col].value_counts(dropna=False ).to_dict()
train[col+'_fq_enc'] = train[col].map(fq_encode)
 test[col+'_fq_enc'] = test[col].map(fq_encode )
with timer("Load"):
nrow = None
train = pd.read_csv("/kaggle/input/cat-in-the-dat/train.csv", index_col = 'id', nrows = nrow)
 test = pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv", index_col = 'id')
 submission_df = pd.read_csv("/kaggle/input/cat-in-the-dat/sample_submission.csv")
 [print(x.shape)for x in [train, test, submission_df]]
traindex = train.index
testdex = test.index
y = train.target.copy()
df = pd.concat([train.drop('target',axis = 1), test], axis = 0)
 del train, test, submission_df<categorify>
i_cols = ['M1','M2','M3','M5','M6','M7','M8','M9']
for df in [train, test]:
df['M_sum'] = df[i_cols].sum(axis=1 ).astype(np.int8 )
with timer("Categorical Processing"):
categorical = df.columns
for col in categorical:
diff = list(set(df.loc[testdex, col].unique())- set(df.loc[traindex,col].unique()))
if diff:
print("Column {} has {} unseen categories in test set".format(col, len(diff)))
 df.loc[df[col].isin(diff),col] = 999
if df[col].dtypes == object:
df[col] = df[col].astype(str)
 lbl = preprocessing.LabelEncoder()
df[col] = pd.Series(lbl.fit_transform(df[col].values)).astype('category' )<groupby>
for col in ['ProductCD','M4']:
temp_dict = train.groupby([col])['isFraud'].agg(['mean'] ).reset_index().rename(
 columns={'mean': col+'_target_mean'})
 temp_dict.index = temp_dict[col].values
temp_dict = temp_dict[col+'_target_mean'].to_dict()
train[col+'_target_mean'] = train[col].map(temp_dict)
 test[col+'_target_mean'] = test[col].map(temp_dict )
categorical_index = list(range(0, len(categorical)))
 features_names = df.columns
catboost_pool = catboost.Pool(df.loc[traindex,:],
label=y,
cat_features=categorical_index)
 test_pool = catboost.Pool(data=df.loc[testdex,:],
cat_features = categorical_index)
 del df
gc.collect()<init_hyperparams>
for feature in ['card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'id_36']:
train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
def catboost_blackbox(max_depth, reg_lambda):
param = {
'learning_rate': 0.2,
'bagging_temperature': 0.1,
'l2_leaf_reg': reg_lambda,
'depth': int(max_depth),
'iterations' : top_boosting_rounds,
'task_type':'GPU',
'loss_function' : "Logloss",
'objective':'Logloss',
'eval_metric' : "AUC",
'bootstrap_type' : 'Bayesian',
'random_seed': seed,
'early_stopping_rounds' : early_stopping_rounds,
'use_best_model': False,
"verbose": False
}
modelstart= time.time()
scores = catboost.cv(catboost_pool,
param,
fold_count = 2,
stratified = True,
shuffle = True,
partition_random_seed = seed,
plot = False 
)
 runtime =(time.time() - modelstart)/60
optimise = scores.loc[scores['test-AUC-mean'].idxmax() ,'test-AUC-mean'] - scores.loc[scores['test-AUC-mean'].idxmax() ,'test-AUC-std']
optimisation_info.append([scores['test-AUC-mean'].idxmax() , optimise, runtime, param, scores['test-AUC-mean'].idxmax() ])
 return optimise<choose_model_class>
train.drop(labels=['uid'],axis=1,inplace=True)
 test.drop(labels=['uid'],axis=1,inplace=True )
parameter_bounds = {
'reg_lambda':(0.1, 10),
'max_depth':(3,16)
 }
init_points = 2
n_iter = 8
optimisation_info = []
CATBOOST_BO = BayesianOptimization(catboost_blackbox,
parameter_bounds,
random_state=seed )<train_model>
for feature in ['d2_d4_comb', 'uid2', 'uid3', 'uid4', 'uid5', 'P_emaildomain_bin', 'P_emaildomain_suffix', 'R_emaildomain_bin', 'R_emaildomain_suffix']:
le = LabelEncoder()
le.fit(list(train[feature].astype(str ).values)+ list(test[feature].astype(str ).values))
train[feature] = le.transform(list(train[feature].astype(str ).values))
test[feature] = le.transform(list(test[feature].astype(str ).values))
with timer("Bayesian Optimisation - {} Iterations".format(init_points + n_iter)) :
with warnings.catch_warnings() :
warnings.filterwarnings('ignore')
 CATBOOST_BO.maximize(init_points = init_points,
n_iter = n_iter,
acq = 'ucb',
xi = 0.0,
alpha = 1e-6 )<compute_test_metric>
X = train.sort_values('TransactionDT' ).drop(['isFraud', 'DT','TransactionDT','TransactionID'], axis=1)
 Y = train.sort_values('TransactionDT')['isFraud']
X_test = test.drop(['TransactionDT', 'DT','TransactionID'], axis=1 )
print("Best Score: {}".format(CATBOOST_BO.max['target']))<prepare_output>
del train, test
optimisation_pd = pd.DataFrame(optimisation_info, columns = ['Best Round', 'Score', 'Runtime','Param', 'Iterations'])
 optimisation_pd.head()<find_best_params>
params = {'num_leaves': 493,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.3797454081646243,
'bagging_fraction': 0.4181193142567742,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.006883242363721497,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47
}
best_param = optimisation_pd.loc[optimisation_pd['Score'].idxmax() ,'Param']
best_param['iterations'] = top_boosting_rounds*3
best_param['learning_rate'] = 0.04
best_param['early_stopping_rounds'] = early_stopping_rounds
best_param<train_model>
folds = StratifiedKFold(n_splits=5)
 evals_result = {}
trainings = list()
aucs = list()
feature_importances = pd.DataFrame()
feature_importances['feature'] = X.columns
training_start_time = time()
for fold,(trn_idx, test_idx)in enumerate(folds.split(X, Y)) :
start_time = time()
print('Training on fold {}'.format(fold + 1))
trn_data = lgb.Dataset(X.iloc[trn_idx], label=Y.iloc[trn_idx])
 val_data = lgb.Dataset(X.iloc[test_idx], label=Y.iloc[test_idx])
 clf = lgb.train(params, trn_data, 10000, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds=500, evals_result=evals_result)
 feature_importances['fold_{}'.format(fold + 1)] = clf.feature_importance()
aucs.append(clf.best_score['valid_1']['auc'])
 trainings.append(clf.best_score['training']['auc'])
 print('Fold {} finished in {}'.format(fold + 1, str(datetime.timedelta(seconds=time() - start_time))))
print('-' * 30)
 print('Training has finished.')
 print('Total training time is {}'.format(str(datetime.timedelta(seconds=time() - training_start_time))))
print('Mean AUC:', np.mean(aucs))
print('STD AUC:', np.std(aucs))
print('Training AUC:', np.mean(trainings))
print('Difference is:', np.mean(trainings)-np.mean(aucs))
print('-' * 30 )
with timer("Catboost Single Model"):
best_param['iterations'] = best_iteration
model = catboost.CatBoostClassifier(**best_param)
 model.fit(catboost_pool )<compute_test_metric>
best_iter = clf.best_iteration
cm = catboost.utils.get_confusion_matrix(model, catboost_pool)
 print(cm )<find_best_params>
clf = lgb.LGBMClassifier(**params, num_boost_round=best_iter)
 clf.fit(X, Y )
roc_curve_values = catboost.utils.get_roc_curve(model, catboost_pool, plot=True )<find_best_params>
sub['isFraud'] = clf.predict_proba(X_test)[:, 1]
<save_to_csv><EOS> 
sub.to_csv('lightgbmdownsampled400k.csv', index=False )