Kaggle competition: Titanic - Machine Learning from Disaster
sub=pd.read_csv('.. /input/sample_submission.csv')
 sub['isFraud']=pred
sub.to_csv('submission.csv',index=False )<import_modules>
%matplotlib inline
import os
import numpy as np
import pandas as pd
from sklearn import preprocessing
import xgboost as xgb<import_modules>
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv' )
print("XGBoost version:", xgb.__version__ )<load_from_csv>
train.isnull().sum()
%%time
train_transaction = pd.read_csv('.. /input/train_transaction.csv', index_col='TransactionID')
 test_transaction = pd.read_csv('.. /input/test_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('.. /input/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/test_identity.csv', index_col='TransactionID')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv', index_col='TransactionID')
 train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print(train.shape)
 print(test.shape)
 y_train = train['isFraud'].copy()
del train_transaction, train_identity, test_transaction, test_identity
X_train = train.drop('isFraud', axis=1)
 X_test = test.copy()
del train, test
X_train = X_train.fillna(-999)
 X_test = X_test.fillna(-999)
 for f in X_train.columns:
if X_train[f].dtype=='object' or X_test[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(X_train[f].values)+ list(X_test[f].values))
X_train[f] = lbl.transform(list(X_train[f].values))
X_test[f] = lbl.transform(list(X_test[f].values))<choose_model_class>
train.isnull().sum()
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=9,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
missing=-999,
random_state=2019,
tree_method='gpu_hist'
)<train_model>
def impute_age(cols):
Age = cols[0]
Pclass = cols[1]
if pd.isnull(Age):
if Pclass == 1:
return 37
elif Pclass == 2:
return 29
else:
return 24
else:
return Age
%time clf.fit(X_train, y_train )<save_to_csv>
train['Age'] = train[['Age','Pclass']].apply(impute_age,axis=1)
 test['Age'] = test[['Age','Pclass']].apply(impute_age,axis=1 )
sample_submission['isFraud'] = clf.predict_proba(X_test)[:,1]
sample_submission.to_csv('simple_xgboost.csv' )<set_options>
train.drop('Cabin',axis=1,inplace=True)
warnings.filterwarnings("ignore")
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 300)
 pd.set_option('display.max_colwidth', 5000)
 pd.options.display.float_format = '{:.3f}'.format
%matplotlib inline
plt.style.use('fivethirtyeight')
 print(os.listdir(".. /input/"))
DIR_NAME = ".. /input" 
<import_modules>
test.drop('Cabin',axis=1,inplace=True )
from sklearn import preprocessing
from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold, KFold, RepeatedKFold, GridSearchCV
from sklearn.metrics import classification_report, roc_auc_score, roc_curve, auc
import xgboost as xgb<load_from_csv>
train.isnull().sum()
def load_dir_csv(directory, csv_files=None):
if csv_files is None:
csv_files = sorted([ f for f in os.listdir(directory)if f.endswith(".csv")])
 csv_vars = [ filename[:-4] for filename in csv_files ]
gdict = globals()
for filename, var in zip(csv_files, csv_vars):
print(f"{var:32s} = pd.read_csv({directory}/{filename})")
 gdict[var] = pd.read_csv(f"{directory}/{filename}")
 print(f"{'shape ':32s} = " + str(gdict[var].shape))
display(gdict[var].head() )<load_from_csv>
train.isnull().sum()
%%time
load_dir_csv(DIR_NAME, ["train_transaction.csv", "test_transaction.csv", "train_identity.csv", "test_identity.csv"] )<merge>
train.dropna(inplace=True)
%%time
train = pd.merge(train_transaction, train_identity, on='TransactionID', how='left')
 test = pd.merge(test_transaction, test_identity, on='TransactionID', how='left' )<drop_column>
test.isnull().sum()
del train_transaction, train_identity, test_transaction, test_identity<set_options>
test.isnull().sum()
gc.collect()<count_missing_values>
test[test['Fare'].isnull() ]
def is_integer_num(n):
if isinstance(n, int):
return True
if isinstance(n, float):
return n.is_integer()
return False
def missing_values_table_specified_value(df, value=0.5):
mis_val = df.isnull().sum()
mis_val_percent = 100 * df.isnull().sum() /len(df)
 mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)
 mis_val_table_ren_columns = mis_val_table.rename(
 columns = {0 : 'Missing Values', 1 : '% of Total Values'})
 if is_integer_num(value):
mis_val_table_ren_columns = mis_val_table_ren_columns[mis_val_table_ren_columns['Missing Values'] >= value]
print('The number of columns with {} counts missing values is {}.'.format(value, len(mis_val_table_ren_columns)))
 else:
value = value * 100
mis_val_table_ren_columns = mis_val_table_ren_columns[mis_val_table_ren_columns['% of Total Values'] >= value]
print('The number of columns with {}% missing values is {}.'.format(value, len(mis_val_table_ren_columns)))
 return mis_val_table_ren_columns
def missing_values_table(data):
total = data.isnull().sum()
percent =(data.isnull().sum() /data.isnull().count() *100)
 tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 tt['Types'] = types
return(np.transpose(tt))<count_missing_values>
train.isnull().sum()
display(missing_values_table(train), missing_values_table(test))<count_values>
test.isnull().sum()
train['isFraud'].value_counts()<count_values>
test.isnull().sum()
quantitative = [f for f in train.columns if train.dtypes[f] != 'object']
print(quantitative)
 print('Counts: {}'.format(len(quantitative)) )<count_values>
sex = pd.get_dummies(train['Sex'],drop_first=True)
 embark = pd.get_dummies(train['Embarked'],drop_first=True )
qualitative = [f for f in train.columns if train.dtypes[f] == 'object']
print(qualitative)
 print('Counts: {}'.format(len(qualitative)) )<count_unique_values>
sex_test = pd.get_dummies(test['Sex'],drop_first=True)
 embark_test = pd.get_dummies(test['Embarked'],drop_first=True )
def visualization_object_data(df, fig_x=10, fig_y=5):
object_list = unique_data_under_n_columns_list(df, 'object', 65)
 for colomn in object_list:
print('column name:{} / unique value(axis x):{} / max value:{} / min value:{} / missing value:{} '.format(colomn, len(df[colomn].unique()), max(df[colomn].value_counts()), min(df[colomn].value_counts()), df[colomn].isnull().sum()))
fig_x = len(df[colomn].unique())
 order = df[colomn].value_counts(ascending=False ).index
if fig_x <= 8:
fig, ax = plt.subplots(1, 3, figsize=(fig_x*6,fig_y))
sns.countplot(x=colomn, ax=ax[0], data=df, order=order)
 ax[0].set_title('All', fontsize=14)
 sns.countplot(x=colomn, ax=ax[1], data=df.loc[df['isFraud'] == 1], order=order)
 ax[1].set_title('isFraud = 1', fontsize=14)
 sns.countplot(x=colomn, ax=ax[2], data=df.loc[df['isFraud'] == 0], order=order)
 ax[2].set_title('isFraud = 0', fontsize=14)
 else:
fig, ax = plt.subplots(1, 3, figsize=(32,10))
sns.countplot(y=colomn, ax=ax[0], data=df, order=order)
 ax[0].set_title('All', fontsize=14)
 sns.countplot(y=colomn, ax=ax[1], data=df.loc[df['isFraud'] == 1], order=order)
 ax[1].set_title('isFraud = 1', fontsize=14)
 sns.countplot(y=colomn, ax=ax[2], data=df.loc[df['isFraud'] == 0], order=order)
 ax[2].set_title('isFraud = 0', fontsize=14)
 plt.show()
plt.pause(0.05 )<prepare_x_and_y>
train.drop(['Sex','Embarked','Name','Ticket','PassengerId'],axis=1,inplace=True)
 test.drop(['Sex','Embarked','Name','Ticket','PassengerId'],axis=1,inplace=True)
%%time
X_train = train.drop('isFraud', axis=1)
 y_train = train['isFraud'].copy()
X_test = test.copy()<set_options>
train = pd.concat([train,sex,embark],axis=1)
 test = pd.concat([test,sex_test,embark_test],axis=1 )
del train, test
gc.collect()<categorify>
predictors = train.drop(['Survived'],axis=1 )
%%time
for f in qualitative:
lbl = preprocessing.LabelEncoder()
lbl.fit(list(X_train[f].values)+ list(X_test[f].values))
X_train[f] = lbl.transform(list(X_train[f].values))
X_test[f] = lbl.transform(list(X_test[f].values))<feature_engineering>
target = train['Survived']
%%time
X_test = reduce_mem_usage(X_test )<set_options>
X = predictors.values
Y = target.values
validation_size = 0.20
seed = 42
X_train,X_test,y_train,y_test = train_test_split(X,Y,test_size=validation_size,random_state=seed )
gc.collect()<load_from_csv>
models = []
models.append(( 'LR',LogisticRegression(solver='lbfgs',multi_class='auto',max_iter=10000)))
 models.append(( 'LDA',LinearDiscriminantAnalysis()))
models.append(( 'KNN',KNeighborsClassifier()))
models.append(( 'CART',DecisionTreeClassifier()))
models.append(( 'NB',GaussianNB()))
models.append(( 'SVM',SVC(gamma='scale')))
 results = []
names = []
for name, model in models:
kfold = KFold(n_splits = 10, random_state=seed)
 cv_results = cross_val_score(model,X_train,y_train,cv=kfold, scoring="accuracy")
 results.append(cv_results)
 names.append(name)
 print(name, cv_results.mean() *100.0, "(",cv_results.std() *100.0,")" )
sub = pd.read_csv(DIR_NAME + '/sample_submission.csv' )<prepare_x_and_y>
num_folds=10
seed=42
scoring='accuracy'
NFOLDS = 5
kf = KFold(n_splits = NFOLDS, shuffle = True)
 y_preds = np.zeros(sub.shape[0])
 y_oof = np.zeros(X_train.shape[0] )<train_model>
simplefilter(action='ignore', category=FutureWarning )
%%time
for tr_idx, val_idx in kf.split(X_train, y_train):
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=9,
learning_rate=0.05,
subsample=0.9,
colsample_bytree=0.9,
tree_method='gpu_hist' 
)
 X_tr, X_vl = X_train.iloc[tr_idx, :], X_train.iloc[val_idx, :]
y_tr, y_vl = y_train.iloc[tr_idx], y_train.iloc[val_idx]
clf.fit(X_tr, y_tr)
 y_pred_train = clf.predict_proba(X_vl)[:,1]
y_oof[val_idx] = y_pred_train
print('ROC AUC {}'.format(roc_auc_score(y_vl, y_pred_train)))
 y_preds += clf.predict_proba(X_test)[:,1] / NFOLDS<save_to_csv>
scaler = StandardScaler().fit(X_train)
 rescaledX = scaler.transform(X_train)
 c_values = [0.1, 0.3, 0.5, 0.7, 0.9, 1.0, 1.3, 1.5, 1.7, 2.0]
kernel_values = ['linear', 'poly', 'rbf', 'sigmoid']
param_grid = dict(C=c_values, kernel=kernel_values)
 model = SVC()
kfold = KFold(n_splits=num_folds, random_state=seed)
 grid = GridSearchCV(estimator= model, param_grid=param_grid, scoring=scoring,cv=kfold)
 grid_result = grid.fit(rescaledX,y_train)
 print("Best: %f using %s"%(grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds= grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
print("%f(%f)with: %r"%(mean,stdev,param))
sub['isFraud'] = y_preds
sub.to_csv('submission.csv', index=False )<install_modules>
scaler = StandardScaler().fit(X)
 rescaledX = scaler.transform(X)
 model = SVC(C=2.0,kernel='rbf')
 model.fit(rescaledX,Y)
 transformed_test = scaler.transform(test )
!pip install -q 'pandas==0.25' --force-reinstall
!pip install GPyOpt<import_modules>
predictions = model.predict(transformed_test )
print("GPyOpt version:", GPyOpt.__version__)
 print("LGBM version:", xgb.__version__)
 print("Pandas version:", pd.__version__ )<set_options>
from xgboost import XGBClassifier
notebookstart = time.time()
warnings.filterwarnings("ignore")
 seed = 24
np.random.seed(seed)
 pd.set_option('display.max_columns', 500)
 pd.options.display.max_rows = 999
pd.set_option('max_colwidth', 500 )<define_variables>
new_model = XGBClassifier(n_estimators = 1000, learning_rate=0.05 )
print("Define DF Schema.. ")
 target_var = 'isFraud'
schema = {
"TransactionDT": "int32",
"TransactionAmt": "float32",
"ProductCD": "object",
"card1": "int16",
"card2": "float32",
"card3": "float32",
"card4": "object",
"card5": "float32",
"card6": "object",
"addr1": "float32",
"addr2": "float32",
"dist1": "float32",
"dist2": "float32",
"P_emaildomain": "object",
"R_emaildomain": "object",
"C1": "float32",
"C2": "float32",
"C3": "float32",
"C4": "float32",
"C5": "float32",
"C6": "float32",
"C7": "float32",
"C8": "float32",
"C9": "float32",
"C10": "float32",
"C11": "float32",
"C12": "float32",
"C13": "float32",
"C14": "float32",
"D1": "float32",
"D2": "float32",
"D3": "float32",
"D4": "float32",
"D5": "float32",
"D6": "float32",
"D7": "float32",
"D8": "float32",
"D9": "float32",
"D10": "float32",
"D11": "float32",
"D12": "float32",
"D13": "float32",
"D14": "float32",
"D15": "float32",
"M1": "object",
"M2": "object",
"M3": "object",
"M4": "object",
"M5": "object",
"M6": "object",
"M7": "object",
"M8": "object",
"M9": "object",
"V1": "float32",
"V2": "float32",
"V3": "float32",
"V4": "float32",
"V5": "float32",
"V6": "float32",
"V7": "float32",
"V8": "float32",
"V9": "float32",
"V10": "float32",
"V11": "float32",
"V12": "float32",
"V13": "float32",
"V14": "float32",
"V15": "float32",
"V16": "float32",
"V17": "float32",
"V18": "float32",
"V19": "float32",
"V20": "float32",
"V21": "float32",
"V22": "float32",
"V23": "float32",
"V24": "float32",
"V25": "float32",
"V26": "float32",
"V27": "float32",
"V28": "float32",
"V29": "float32",
"V30": "float32",
"V31": "float32",
"V32": "float32",
"V33": "float32",
"V34": "float32",
"V35": "float32",
"V36": "float32",
"V37": "float32",
"V38": "float32",
"V39": "float32",
"V40": "float32",
"V41": "float32",
"V42": "float32",
"V43": "float32",
"V44": "float32",
"V45": "float32",
"V46": "float32",
"V47": "float32",
"V48": "float32",
"V49": "float32",
"V50": "float32",
"V51": "float32",
"V52": "float32",
"V53": "float32",
"V54": "float32",
"V55": "float32",
"V56": "float32",
"V57": "float32",
"V58": "float32",
"V59": "float32",
"V60": "float32",
"V61": "float32",
"V62": "float32",
"V63": "float32",
"V64": "float32",
"V65": "float32",
"V66": "float32",
"V67": "float32",
"V68": "float32",
"V69": "float32",
"V70": "float32",
"V71": "float32",
"V72": "float32",
"V73": "float32",
"V74": "float32",
"V75": "float32",
"V76": "float32",
"V77": "float32",
"V78": "float32",
"V79": "float32",
"V80": "float32",
"V81": "float32",
"V82": "float32",
"V83": "float32",
"V84": "float32",
"V85": "float32",
"V86": "float32",
"V87": "float32",
"V88": "float32",
"V89": "float32",
"V90": "float32",
"V91": "float32",
"V92": "float32",
"V93": "float32",
"V94": "float32",
"V95": "float32",
"V96": "float32",
"V97": "float32",
"V98": "float32",
"V99": "float32",
"V100": "float32",
"V101": "float32",
"V102": "float32",
"V103": "float32",
"V104": "float32",
"V105": "float32",
"V106": "float32",
"V107": "float32",
"V108": "float32",
"V109": "float32",
"V110": "float32",
"V111": "float32",
"V112": "float32",
"V113": "float32",
"V114": "float32",
"V115": "float32",
"V116": "float32",
"V117": "float32",
"V118": "float32",
"V119": "float32",
"V120": "float32",
"V121": "float32",
"V122": "float32",
"V123": "float32",
"V124": "float32",
"V125": "float32",
"V126": "float32",
"V127": "float32",
"V128": "float32",
"V129": "float32",
"V130": "float32",
"V131": "float32",
"V132": "float32",
"V133": "float32",
"V134": "float32",
"V135": "float32",
"V136": "float32",
"V137": "float32",
"V138": "float32",
"V139": "float32",
"V140": "float32",
"V141": "float32",
"V142": "float32",
"V143": "float32",
"V144": "float32",
"V145": "float32",
"V146": "float32",
"V147": "float32",
"V148": "float32",
"V149": "float32",
"V150": "float32",
"V151": "float32",
"V152": "float32",
"V153": "float32",
"V154": "float32",
"V155": "float32",
"V156": "float32",
"V157": "float32",
"V158": "float32",
"V159": "float32",
"V160": "float32",
"V161": "float32",
"V162": "float32",
"V163": "float32",
"V164": "float32",
"V165": "float32",
"V166": "float32",
"V167": "float32",
"V168": "float32",
"V169": "float32",
"V170": "float32",
"V171": "float32",
"V172": "float32",
"V173": "float32",
"V174": "float32",
"V175": "float32",
"V176": "float32",
"V177": "float32",
"V178": "float32",
"V179": "float32",
"V180": "float32",
"V181": "float32",
"V182": "float32",
"V183": "float32",
"V184": "float32",
"V185": "float32",
"V186": "float32",
"V187": "float32",
"V188": "float32",
"V189": "float32",
"V190": "float32",
"V191": "float32",
"V192": "float32",
"V193": "float32",
"V194": "float32",
"V195": "float32",
"V196": "float32",
"V197": "float32",
"V198": "float32",
"V199": "float32",
"V200": "float32",
"V201": "float32",
"V202": "float32",
"V203": "float32",
"V204": "float32",
"V205": "float32",
"V206": "float32",
"V207": "float32",
"V208": "float32",
"V209": "float32",
"V210": "float32",
"V211": "float32",
"V212": "float32",
"V213": "float32",
"V214": "float32",
"V215": "float32",
"V216": "float32",
"V217": "float32",
"V218": "float32",
"V219": "float32",
"V220": "float32",
"V221": "float32",
"V222": "float32",
"V223": "float32",
"V224": "float32",
"V225": "float32",
"V226": "float32",
"V227": "float32",
"V228": "float32",
"V229": "float32",
"V230": "float32",
"V231": "float32",
"V232": "float32",
"V233": "float32",
"V234": "float32",
"V235": "float32",
"V236": "float32",
"V237": "float32",
"V238": "float32",
"V239": "float32",
"V240": "float32",
"V241": "float32",
"V242": "float32",
"V243": "float32",
"V244": "float32",
"V245": "float32",
"V246": "float32",
"V247": "float32",
"V248": "float32",
"V249": "float32",
"V250": "float32",
"V251": "float32",
"V252": "float32",
"V253": "float32",
"V254": "float32",
"V255": "float32",
"V256": "float32",
"V257": "float32",
"V258": "float32",
"V259": "float32",
"V260": "float32",
"V261": "float32",
"V262": "float32",
"V263": "float32",
"V264": "float32",
"V265": "float32",
"V266": "float32",
"V267": "float32",
"V268": "float32",
"V269": "float32",
"V270": "float32",
"V271": "float32",
"V272": "float32",
"V273": "float32",
"V274": "float32",
"V275": "float32",
"V276": "float32",
"V277": "float32",
"V278": "float32",
"V279": "float32",
"V280": "float32",
"V281": "float32",
"V282": "float32",
"V283": "float32",
"V284": "float32",
"V285": "float32",
"V286": "float32",
"V287": "float32",
"V288": "float32",
"V289": "float32",
"V290": "float32",
"V291": "float32",
"V292": "float32",
"V293": "float32",
"V294": "float32",
"V295": "float32",
"V296": "float32",
"V297": "float32",
"V298": "float32",
"V299": "float32",
"V300": "float32",
"V301": "float32",
"V302": "float32",
"V303": "float32",
"V304": "float32",
"V305": "float32",
"V306": "float32",
"V307": "float32",
"V308": "float32",
"V309": "float32",
"V310": "float32",
"V311": "float32",
"V312": "float32",
"V313": "float32",
"V314": "float32",
"V315": "float32",
"V316": "float32",
"V317": "float32",
"V318": "float32",
"V319": "float32",
"V320": "float32",
"V321": "float32",
"V322": "float32",
"V323": "float32",
"V324": "float32",
"V325": "float32",
"V326": "float32",
"V327": "float32",
"V328": "float32",
"V329": "float32",
"V330": "float32",
"V331": "float32",
"V332": "float32",
"V333": "float32",
"V334": "float32",
"V335": "float32",
"V336": "float32",
"V337": "float32",
"V338": "float32",
"V339": "float32",
"id_01": "float32",
"id_02": "float32",
"id_03": "float32",
"id_04": "float32",
"id_05": "float32",
"id_06": "float32",
"id_07": "float32",
"id_08": "float32",
"id_09": "float32",
"id_10": "float32",
"id_11": "float32",
"id_12": "object",
"id_13": "float32",
"id_14": "float32",
"id_15": "object",
"id_16": "object",
"id_17": "float32",
"id_18": "float32",
"id_19": "float32",
"id_20": "float32",
"id_21": "float32",
"id_22": "float32",
"id_23": "object",
"id_24": "float32",
"id_25": "float32",
"id_26": "float32",
"id_27": "object",
"id_28": "object",
"id_29": "object",
"id_30": "object",
"id_31": "object",
"id_32": "float32",
"id_33": "object",
"id_34": "object",
"id_35": "object",
"id_36": "object",
"id_37": "object",
"id_38": "object",
"DeviceType": "object",
"DeviceInfo": "object",
"is_fraud":			 "int8"
}
emails = {'gmail': 'google',
'att.net': 'att',
'twc.com': 'spectrum',
'scranton.edu': 'other',
'optonline.net': 'other',
'hotmail.co.uk': 'microsoft',
'comcast.net': 'other',
'yahoo.com.mx': 'yahoo',
'yahoo.fr': 'yahoo',
'yahoo.es': 'yahoo',
'charter.net': 'spectrum',
'live.com': 'microsoft',
'aim.com': 'aol',
'hotmail.de': 'microsoft',
'centurylink.net': 'centurylink',
'gmail.com': 'google',
'me.com': 'apple',
'earthlink.net': 'other',
'gmx.de': 'other',
'web.de': 'other',
'cfl.rr.com': 'other',
'hotmail.com': 'microsoft',
'protonmail.com': 'other',
'hotmail.fr': 'microsoft',
'windstream.net': 'other',
'outlook.es': 'microsoft',
'yahoo.co.jp': 'yahoo',
'yahoo.de': 'yahoo',
'servicios-ta.com': 'other',
'netzero.net': 'other',
'suddenlink.net': 'other',
'roadrunner.com': 'other',
'sc.rr.com': 'other',
'live.fr': 'microsoft',
'verizon.net': 'yahoo',
'msn.com': 'microsoft',
'q.com': 'centurylink',
'prodigy.net.mx': 'att',
'frontier.com': 'yahoo',
'anonymous.com': 'other',
'rocketmail.com': 'yahoo',
'sbcglobal.net': 'att',
'frontiernet.net': 'yahoo',
'ymail.com': 'yahoo',
'outlook.com': 'microsoft',
'mail.com': 'other',
'bellsouth.net': 'other',
'embarqmail.com': 'centurylink',
'cableone.net': 'other',
'hotmail.es': 'microsoft',
'mac.com': 'apple',
'yahoo.co.uk': 'yahoo',
'netzero.com': 'other',
'yahoo.com': 'yahoo',
'live.com.mx': 'microsoft',
'ptd.net': 'other',
'cox.net': 'other',
'aol.com': 'aol',
'juno.com': 'other',
'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']<feature_engineering>
new_model.fit(X_train,y_train,
early_stopping_rounds = 5,
eval_set = [(X_test,y_test)],
verbose = False )
@contextmanager
def timer(name):

 t0 = time.time()
yield
print('
[{}] done in {} Minutes
'.format(name, round(( time.time() - t0)/60,2)))
 def id_split(dataframe):
dataframe['device_name'] = dataframe['DeviceInfo'].str.split('/', expand=True)[0]
dataframe['device_version'] = dataframe['DeviceInfo'].str.split('/', expand=True)[1]
dataframe['OS_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[0]
dataframe['version_id_30'] = dataframe['id_30'].str.split(' ', expand=True)[1]
dataframe['browser_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[0]
dataframe['version_id_31'] = dataframe['id_31'].str.split(' ', expand=True)[1]
dataframe['screen_width'] = dataframe['id_33'].str.split('x', expand=True)[0]
dataframe['screen_height'] = dataframe['id_33'].str.split('x', expand=True)[1]
dataframe['id_34'] = dataframe['id_34'].str.split(':', expand=True)[1]
dataframe['id_23'] = dataframe['id_23'].str.split(':', expand=True)[1]
dataframe.loc[dataframe['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
dataframe.loc[dataframe['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('moto', na=False), 'device_name'] = 'Motorola'
dataframe.loc[dataframe['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
dataframe.loc[dataframe['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
dataframe.loc[dataframe['device_name'].str.contains('HUAWEI', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
dataframe.loc[dataframe['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
dataframe.loc[dataframe['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
dataframe.loc[dataframe['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
dataframe.loc[dataframe['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
dataframe.loc[dataframe['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'
dataframe.loc[dataframe.device_name.isin(dataframe.device_name.value_counts() [dataframe.device_name.value_counts() < 200].index), 'device_name'] = "Others"
dataframe['had_id'] = 1
gc.collect()
return dataframe
def fraud_preprocessing(debug = None):
print("Starting Pre-Processing.. ")
 with timer("Load Tables"):
train_transaction = pd.read_csv('.. /input/train_transaction.csv',
index_col='TransactionID', nrows= debug, dtype = schema)
 test_transaction = pd.read_csv('.. /input/test_transaction.csv',
index_col='TransactionID', nrows= debug, dtype = schema)
 train_identity = pd.read_csv('.. /input/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/test_identity.csv', index_col='TransactionID')
 sample_submission = pd.read_csv('.. /input/sample_submission.csv',
index_col='TransactionID',
nrows= debug)
 with timer("Merge Tables"):
train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 print("Train Shape: {} Rows, {} Columns".format(*train.shape))
print("Test Shape: {} Rows, {} Columns".format(*test.shape))
y = train[target_var].copy()
del train_transaction, train_identity, test_transaction, test_identity
traindex = train.index
testdex = test.index
with timer("Train/Test Split Feature Engineering"):
train['TransactionAmt_to_mean_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_mean_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_std_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('std')
 train['TransactionAmt_to_std_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_mean_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_mean_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_std_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_std_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('std')
 train['id_02_to_mean_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('mean')
 train['id_02_to_mean_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('mean')
 train['id_02_to_std_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('std')
 train['id_02_to_std_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('std')
 test['id_02_to_mean_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('mean')
 test['id_02_to_mean_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('mean')
 test['id_02_to_std_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('std')
 test['id_02_to_std_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('std')
 train['D15_to_mean_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['D15_to_mean_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['TransactionAmt_Log'] = np.log(train['TransactionAmt'])
 test['TransactionAmt_Log'] = np.log(test['TransactionAmt'])
 for feature in ['card1', 'card2', 'card3', 'card4', 'card5', 'card6', 'id_36']:
train[feature + '_count_full'] = train[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
test[feature + '_count_full'] = test[feature].map(pd.concat([train[feature], test[feature]], ignore_index=True ).value_counts(dropna=False))
for feature in ['id_01', 'id_31', 'id_33', 'id_36']:
train[feature + '_count_dist'] = train[feature].map(train[feature].value_counts(dropna=False))
test[feature + '_count_dist'] = test[feature].map(test[feature].value_counts(dropna=False))
for c in ['P_emaildomain', 'R_emaildomain']:
train[c + '_bin'] = train[c].map(emails)
 test[c + '_bin'] = test[c].map(emails)
 train[c + '_suffix'] = train[c].map(lambda x: str(x ).split('.')[-1])
 test[c + '_suffix'] = test[c].map(lambda x: str(x ).split('.')[-1])
 train[c + '_suffix'] = train[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 test[c + '_suffix'] = test[c + '_suffix'].map(lambda x: x if str(x)not in us_emails else 'us')
 train = id_split(train)
 test = id_split(test)
 df = pd.concat([train.drop(target_var,axis=1),test],axis = 0)
 del train, test
with timer("Whole Feature Engineering"):
START_DATE = '2017-12-01'
startdate = datetime.datetime.strptime(START_DATE, '%Y-%m-%d')
 df = df.assign(
 TransactionAmt_decimal =(( df['TransactionAmt'] - df['TransactionAmt'].astype(int)) * 1000 ).astype(int),
card1_count_full = df['card1'].map(df['card1'].value_counts(dropna=False)) ,
Transaction_day_of_week = np.floor(( df['TransactionDT'] /(3600 * 24)- 1)% 7),
Transaction_hour = np.floor(df['TransactionDT'] / 3600)% 24,
TransactionDT = df['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds = x))), 
)
 df = df.assign(
 year = df['TransactionDT'].dt.year,
month = df['TransactionDT'].dt.month,
dow = df['TransactionDT'].dt.dayofweek,
quarter = df['TransactionDT'].dt.quarter,
hour = df['TransactionDT'].dt.hour,
day = df['TransactionDT'].dt.day,
all_group_nan_sum = df.isnull().sum(axis=1)/ df.shape[1],
all_group_0_count =(df == 0 ).astype(int ).sum(axis=1)/(df.shape[1] - df.isnull().sum(axis=1)) 
)
 prefix = ['C','D','Device','M','Transaction','V','addr','card','dist','id']
for i, p in enumerate(prefix):
column_set = [x for x in df.columns.tolist() if x.startswith(prefix[i])]
df[p + "group_nan_sum"] = df[column_set].isnull().sum(axis=1)/ df[column_set].shape[1]
numeric_cols = [x for x in column_set if df[x].dtype != object]
if numeric_cols:
df[p + "group_sum"] = df[column_set].sum(axis=1)
 df[p + "group_mean"] = df[column_set].mean(axis=1)
 df[p + "group_0_count"] =(df[column_set] == 0 ).astype(int ).sum(axis=1)/(df[column_set].shape[1] - df[p + "group_nan_sum"])
 with timer("Label Encode"):
categorical_cols = []
for f in df.columns:
if df[f].dtype=='object':
categorical_cols += [f]
lbl = preprocessing.LabelEncoder()
df[f] = lbl.fit_transform(df[f].astype(str))
print("Total Shape: {} Rows, {} Columns".format(*df.shape))
return df, y, traindex, testdex, categorical_cols, sample_submission<prepare_x_and_y>
xgb_predictions = new_model.predict(X_test )
DEBUG = None
df, y, traindex, testdex, cat_cols, sample_submission = fraud_preprocessing(debug = DEBUG)
 df.fillna(-999, inplace=True)
 X = df.loc[traindex,:]
feature_subset = [x for x in X.columns.tolist() if x not in ['TransactionDT','Fraud', 'traintest', 'yrmth']]
test = df.loc[testdex,:]
del df; gc.collect() ;<compute_test_metric>
accuracy_score(xgb_predictions,y_test )
print("None Fraud: {}%, Fraud: {}%".format(*y.value_counts(normalize=True)))
 print("Randomness Score AUC: {}".format(
 metrics.roc_auc_score(y,np.array([y.value_counts(normalize=True)[0]]*y.shape[0]))))<init_hyperparams>
test_predictions = new_model.predict(test.values )
def XGB_score(para):
parameters = para[0]
params = {
'objective': 'binary:logistic',
'eval_metric': metric,
'learning_rate': 0.1,
'tree_method': 'gpu_hist',
'importance_type': 'weight',
'missing': -999,
'max_depth': int(parameters[0]),
'reg_alpha': parameters[1],
'reg_lambda': parameters[2],
'subsample': parameters[3],
'colsample_bytree': parameters[4],
'seed': seed
}
experiment_clf = xgb.XGBClassifier(**params,
n_estimators = n_estimators)
 modelstart= time.time()
experiment_clf.fit(X_train,
y_train,
eval_set=[(X_valid,y_valid)],
early_stopping_rounds=100,
verbose=0)
 runtime =(time.time() - modelstart)/60
rounds = experiment_clf.best_ntree_limit
val_pred = experiment_clf.predict_proba(X_valid)[:,1]
score = experiment_clf.best_score
loss = metrics.log_loss(y_valid, val_pred)
 params['num_boost_round'] = rounds
gpyopt_output.append(
 [
loss,
score,
rounds,
params,
runtime,
experiment_clf.feature_importances_
] 
)
 return score<split>
xgb_submission= pd.read_csv('/kaggle/input/titanic/gender_submission.csv' )
split_size = 0.4
n_estimators = 10000
metric = 'auc'
ESR = 175
X_train, X_valid, y_train, y_valid = train_test_split(
 X[feature_subset], y, test_size=split_size,
random_state=seed, shuffle=True,stratify=y)
 bds = [ {'name': 'max_depth', 'type': 'discrete', 'domain':(5, 15)},
{'name': 'reg_alpha', 'type': 'continuous', 'domain': (.05, 0.8)},
{'name': 'reg_lambda', 'type': 'continuous', 'domain': (.05, 0.8)},
{'name': 'subsample', 'type': 'continuous', 'domain':(0.1, 0.80)},
{'name': 'colsample_bytree', 'type': 'continuous', 'domain':(0.1, 0.80)},
]
sb_cat_cols = [x for x in cat_cols if x in feature_subset]
max_iter = 30
initial_iter = 5
gpyopt_output = []
gpy_importance = pd.DataFrame()
optimizer = BayesianOptimization(f=XGB_score,
domain=bds,
model_type='GP',
optimize_restarts = 1,
initial_design_numdata = initial_iter,
acquisition_type ='EI',
acquisition_jitter = 0.05,
exact_feval=True,
maximize=True)
 with timer("Bayesian Optimisation - {} Iterations".format(max_iter + initial_iter)) :
optimizer.run_optimization(max_iter=max_iter)
 results = pd.DataFrame(gpyopt_output,
columns = ['logloss','valid_auc',
'boosting_rounds','parameters', 'runtime', 'imp'] 
)
 gpyimp = results.imp.apply(lambda x: pd.Series(x)).T
gpyimp.index = feature_subset
og_cols = gpyimp.columns
gpyimp['mean'] = gpyimp[og_cols].mean(axis = 1)
 gpyimp['std'] = gpyimp[og_cols].std(axis = 1)
 del gpyopt_output; results.drop('imp', axis =1, inplace=True)
 results.to_csv("gpyopt_iterations_output.csv")
 best_params = results['parameters'].iloc[np.argmax(results.valid_auc)]
optimizer.plot_convergence()
results['TPM'] = results['boosting_rounds'] / results['runtime']
plt.plot(results['TPM'], '-rx')
 plt.xlabel("Bayesian Search Iteration")
 plt.ylabel("TPM")
 plt.title("Tree per Minute through Bayesian Search")
 plt.show()
print("Best AUC: {}".format(optimizer.fx_opt))
print("Best Parameters")
 pprint.pprint(best_params)
 results = pd.concat([results.drop('parameters',axis=1 ).reset_index(drop=True),
json_normalize(results['parameters'] ).reset_index(drop=True)
 ], axis = 1)
 del X_train, X_valid, y_train, y_valid<drop_column>
xgb_submission['Survived'] = test_predictions
<prepare_x_and_y><EOS> 
xgb_submission.to_csv('gender_submission.csv',index=False )