Kaggle competition: IEEE-CIS Fraud Detection
def score(clf, random_state = 0):
kf = StratifiedKFold(y, n_folds=5, shuffle=True, random_state=random_state)
 pred = np.zeros(( y.shape[0],nclasses))
for itrain, itest in kf:
Xtr, Xte = Xtrain[itrain, :], Xtrain[itest, :]
ytr, yte = y[itrain], y[itest]
clf.fit(Xtr, ytr)
 pred[itest,:] = clf.predict_proba(Xte)
 return log_loss(yte, pred[itest, :])
 print("{:.5f}".format(log_loss(yte, pred[itest,:])) , end=' ')
 print('')
 return log_loss(y, pred )<compute_test_metric>
!pip install -Uqq fastbook
fastbook.setup_book()
score(LogisticRegression(C=0.02))<choose_model_class>
from pandas.api.types import is_string_dtype, is_numeric_dtype, is_categorical_dtype
from fastai.tabular.all import *
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import train_test_split
from IPython.display import Image, display_svg, SVG
score(LogisticRegression(C=0.02, multi_class='multinomial',solver='lbfgs'))<compute_test_metric>
from sklearn.metrics import confusion_matrix, roc_auc_score, plot_confusion_matrix, precision_score, recall_score, classification_report, plot_roc_curve, roc_curve
from sklearn.model_selection import train_test_split
clf = LogisticRegression(C=0.02, multi_class='multinomial',solver='lbfgs')
 clf.fit(Xtrain, y)
 pred = pd.DataFrame(clf.predict_proba(Xtest), index = gatest.index, columns=targetencoder.classes_)
 pred.head()<save_to_csv>
import gc
pred.to_csv('logreg_subm.csv',index=True )<set_options>
pd.options.display.max_rows = 20
pd.options.display.max_columns = 12
plt.style.use('seaborn-white')
 sns.set_style("white")
<train_model>
main_path = ".. /input/fraud-detection/"
img_size_ori = 101
img_size_target = 128
def upsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_target, img_size_target), mode='constant', preserve_range=True)
 def downsample(img):
if img_size_ori == img_size_target:
return img
return resize(img,(img_size_ori, img_size_ori), mode='constant', preserve_range=True )<load_from_csv>
import xgboost as xgb
import random as r
train_df = pd.read_csv(".. /input/tgs-salt-identification-challenge/train.csv", index_col="id", usecols=[0])
 val_ids = pd.read_csv('.. /input/validation-ids/validation_ids.csv')
 train_df = train_df.drop(val_ids.iloc[:,0].values)
 no_salt_ids = pd.read_csv(".. /input/pseudolabel-gen/no_salt_ids.csv", index_col="id", usecols=[0])
 no_salt_ids = no_salt_ids.sample(2340)
 some_salt_ids = pd.read_csv(".. /input/pseudolabel-gen/some_salt_ids.csv", index_col="id")
 some_salt_ids = some_salt_ids.sample(3660)
 depths_df = pd.read_csv(".. /input/tgs-salt-identification-challenge/depths.csv", index_col="id")
 train_df = train_df.join(depths_df)
 test_df = depths_df[~depths_df.index.isin(train_df.index)]<feature_engineering>
import joblib
train_df["images"] = [np.array(load_img(".. /input/tgs-salt-identification-challenge/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]
train_df["masks"] = [np.array(load_img(".. /input/tgs-salt-identification-challenge/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(train_df.index)]<categorify>
buildLv = False
buildSkf = True
FI=False
def rle_decode(rle_mask):

 if str(rle_mask)==str(np.nan):
return np.zeros(( 101,101))
s = rle_mask.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(101*101, dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(101,101 )<feature_engineering>
%%time
dset_str="all"
to = load_pickle(main_path+"to_"+dset_str+"c.pkl")
 to_tst = load_pickle(main_path+"to_tst_"+dset_str+"c.pkl")
train_df_temp = pd.DataFrame()
train_df_temp['id'] = no_salt_ids.index
train_df_temp = train_df_temp.set_index('id')
 train_df_temp = train_df_temp.join(depths_df)
 train_df_temp["images"] = [np.array(load_img(".. /input/tgs-salt-identification-challenge/test/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(no_salt_ids.index)]
train_df_temp["masks"] = [np.zeros(( img_size_ori,img_size_ori)) for idx in tqdm_notebook(no_salt_ids.index)]
train_df = train_df.append(train_df_temp)
 del train_df_temp<feature_engineering>
xs,y = to.train.xs,to.train.y
valid_xs,valid_y = to.valid.xs,to.valid.y
xs.shape
train_df_temp = pd.DataFrame()
train_df_temp['id'] = some_salt_ids.index
train_df_temp = train_df_temp.set_index('id')
 train_df_temp = train_df_temp.join(depths_df)
 train_df_temp["images"] = [np.array(load_img(".. /input/tgs-salt-identification-challenge/test/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(some_salt_ids.index)]
train_df_temp["masks"] = [np.fliplr(np.rot90(rle_decode(some_salt_ids.loc[idx,'rle_mask']),-1)) for idx in tqdm_notebook(some_salt_ids.index)]
train_df = train_df.append(train_df_temp)
 del train_df_temp
gc.collect()<categorify>
xs.isna().sum()
train_df["coverage"] = train_df.masks.map(np.sum)/ pow(img_size_ori, 2)
 def cov_to_class(val):
for i in range(0, 11):
if val * 10 <= i :
return i
train_df["coverage_class"] = train_df.coverage.map(cov_to_class )<prepare_x_and_y>
del to, to_tst; x=gc.collect()
ids_valid = val_ids.iloc[:,0].values
temp_df = pd.DataFrame()
temp_df["images"] = [np.array(load_img(".. /input/tgs-salt-identification-challenge/train/images/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(ids_valid)]
temp_df["masks"] = [np.array(load_img(".. /input/tgs-salt-identification-challenge/train/masks/{}.png".format(idx), grayscale=True)) / 255 for idx in tqdm_notebook(ids_valid)]
x_valid = np.array(temp_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1)
 y_valid = np.array(temp_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1)
 del temp_df
gc.collect()<prepare_x_and_y>
def m_rep(m,xs,y): return classification_report(y, m.predict(xs), labels=[1,0], digits=4, output_dict=True)
 def metrics(m, xs, y , valid_xs, valid_y):
tr_rep = m_rep(m,xs,y)
 vd_rep = m_rep(m,valid_xs,valid_y)
 tr_auc = roc_auc_score(y,m.predict_proba(xs)[:,1])
 oob_auc=0.0000
vd_auc = roc_auc_score(valid_y,m.predict_proba(valid_xs)[:,1])
 print('{:.4f} ; {:.4f} ; {:.4f}' 
.format(tr_auc, oob_auc, vd_auc))
ids_train,x_train,y_train = train_df.index.values,\
np.array(train_df.images.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1), \
np.array(train_df.masks.map(upsample ).tolist() ).reshape(-1, img_size_target, img_size_target, 1 )<compute_train_metric>
def format_test_preds(preds,test_xs,comment):
print(pd.DataFrame(preds ).describe())
 df_pred = pd.DataFrame({"TransactionID":test_xs.TransactionID.to_list() ,
"isFraud": preds})
 df_pred.to_csv(comment+"my_subs.csv",index=False)
 print(df_pred.shape)
 del df_pred; x=gc.collect()
def get_iou_vector(A, B):
batch_size = A.shape[0]
metric = 0.0
for batch in range(batch_size):
t, p = A[batch], B[batch]
true = np.sum(t)
 pred = np.sum(p)
 if true == 0:
metric +=(pred == 0)
 continue
intersection = np.sum(t * p)
 union = true + pred - intersection
iou = intersection / union
iou = np.floor(max(0,(iou - 0.45)*20)) / 10
metric += iou
metric /= batch_size
return metric
def my_iou_metric(label, pred):
return tf.py_func(get_iou_vector, [label, pred > 0.5], tf.float64 )<compute_test_metric>
def xgb_lvd_fun(i):
clf = xgb.XGBClassifier(n_estimators=1000,
max_depth=8,
learning_rate= 0.05,
subsample= 0.6,
colsample_bytree= 0.4,
random_state = r.randint(0,9999),
tree_method='gpu_hist')
 m = clf.fit(xs[cols].iloc[idxT], y.iloc[idxT], eval_set=[(xs[cols].iloc[idxV],y.iloc[idxV])],
eval_metric= "auc", verbose=100, early_stopping_rounds=100)
 tr_auc = roc_auc_score(y.iloc[idxT], m.predict_proba(xs[cols].iloc[idxT])[:,1])
 vd_auc = roc_auc_score(y.iloc[idxV],m.predict_proba(xs[cols].iloc[idxV])[:,1])
 te_pred = m.predict_proba(test_xs[cols])[:,1]
del m, clf; x=gc.collect()
return [tr_auc, vd_auc], te_pred
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
 y_pred = K.cast(y_pred, 'float32')
 y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')
 intersection = y_true_f * y_pred_f
score = 2.* K.sum(intersection)/(K.sum(y_true_f)+ K.sum(y_pred_f))
return score
def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred)
 def bce_logdice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)- K.log(1.- dice_loss(y_true, y_pred))<choose_model_class>
def xgb_pred_fun() :
skf = KFold(n_splits = 5, shuffle = False, random_state = 42)
 tr_pred = np.zeros(len(xs))
oof_pred = np.zeros(len(xs))
te_pred = np.zeros(len(test_xs))
for i,(idxT, idxV)in enumerate(skf.split(xs,y)) :
print('Fold',i)
 print(' n_rows of train =',len(idxT),'rows of holdout =',min(idxV),"to",max(idxV))
clf = xgb.XGBClassifier(n_estimators=1000,
max_depth=8,
learning_rate= 0.05,
subsample= 0.6,
colsample_bytree= 0.4,
random_state = r.randint(0,9999),
tree_method='gpu_hist')
 m = clf.fit(xs[cols].iloc[idxT], y.iloc[idxT], eval_set=[(xs[cols].iloc[idxV],y.iloc[idxV])],
eval_metric= "auc", verbose=100, early_stopping_rounds=100)
 tr_pred[idxT] += m.predict_proba(xs[cols].iloc[idxT])[:,1]/(skf.n_splits-1)
 oof_pred[idxV] += m.predict_proba(xs[cols].iloc[idxV])[:,1]
te_pred += m.predict_proba(test_xs[cols])[:,1]/skf.n_splits
del m; x=gc.collect()
print('{:.4f} ; {:.4f} ; {:.4f}' 
.format(roc_auc_score(y,tr_pred),roc_auc_score(y,oof_pred),0.000))
return te_pred
class SnapshotCallbackBuilder:
def __init__(self, nb_epochs, nb_snapshots, init_lr=0.1):
self.T = nb_epochs
self.M = nb_snapshots
self.alpha_zero = init_lr
def get_callbacks(self, model_prefix='Model'):
callback_list = [
callbacks.ModelCheckpoint("./keras.model",monitor='val_my_iou_metric',
mode = 'max', save_best_only=True, verbose=1),
swa,
callbacks.LearningRateScheduler(schedule=self._cosine_anneal_schedule)
 ]
return callback_list
def _cosine_anneal_schedule(self, t):
cos_inner = np.pi *(t %(self.T // self.M))
cos_inner /= self.T // self.M
cos_out = np.cos(cos_inner)+ 1
return float(self.alpha_zero / 2 * cos_out )<define_search_model>
def split_cols(df, col = "TransactionAmt"):
df['cents'] = df[col].mod(1)
 df[col] = df[col].floordiv(1)
 split_cols(xs), split_cols(test_xs )
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 x = BatchNormalization()(x)
 if activation == True:
x = LeakyReLU(alpha=0.1 )(x)
 return x
def residual_block(blockInput, num_filters=16):
x = LeakyReLU(alpha=0.1 )(blockInput)
 x = BatchNormalization()(x)
 blockInput = BatchNormalization()(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 return x<choose_model_class>
cols = xs.columns.to_list()
def UXception(input_shape=(None, None, 3)) :
backbone = Xception(input_shape=input_shape,weights='imagenet',include_top=False)
 input = backbone.input
start_neurons = 16
conv4 = backbone.layers[121].output
conv4 = LeakyReLU(alpha=0.1 )(conv4)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(0.1 )(pool4)
 convm = Conv2D(start_neurons * 32,(3, 3), activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 32)
 convm = residual_block(convm,start_neurons * 32)
 convm = LeakyReLU(alpha=0.1 )(convm)
 deconv4 = Conv2DTranspose(start_neurons * 16,(3, 3), strides=(2, 2), padding="same" )(convm)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(0.1 )(uconv4)
 uconv4 = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 16)
 uconv4 = residual_block(uconv4,start_neurons * 16)
 uconv4 = LeakyReLU(alpha=0.1 )(uconv4)
 deconv3 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(uconv4)
 conv3 = backbone.layers[31].output
uconv3 = concatenate([deconv3, conv3])
 uconv3 = Dropout(0.1 )(uconv3)
 uconv3 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 8)
 uconv3 = residual_block(uconv3,start_neurons * 8)
 uconv3 = LeakyReLU(alpha=0.1 )(uconv3)
 deconv2 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 conv2 = backbone.layers[21].output
conv2 = ZeroPadding2D(((1,0),(1,0)) )(conv2)
 uconv2 = concatenate([deconv2, conv2])
 uconv2 = Dropout(0.1 )(uconv2)
 uconv2 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 4)
 uconv2 = residual_block(uconv2,start_neurons * 4)
 uconv2 = LeakyReLU(alpha=0.1 )(uconv2)
 deconv1 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv2)
 conv1 = backbone.layers[11].output
conv1 = ZeroPadding2D(((3,0),(3,0)) )(conv1)
 uconv1 = concatenate([deconv1, conv1])
 uconv1 = Dropout(0.1 )(uconv1)
 uconv1 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 2)
 uconv1 = residual_block(uconv1,start_neurons * 2)
 uconv1 = LeakyReLU(alpha=0.1 )(uconv1)
 uconv0 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="same" )(uconv1)
 uconv0 = Dropout(0.1 )(uconv0)
 uconv0 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv0)
 uconv0 = residual_block(uconv0,start_neurons * 1)
 uconv0 = residual_block(uconv0,start_neurons * 1)
 uconv0 = LeakyReLU(alpha=0.1 )(uconv0)
 uconv0 = Dropout(0.1/2 )(uconv0)
 output_layer = Conv2D(1,(1,1), padding="same", activation="sigmoid" )(uconv0)
 model = Model(input, output_layer)
 model.name = 'u-xception'
return model<choose_model_class>
v = []
v += ['V1', 'V3', 'V11', 'V9', 'V5', 'V7']
v += ['V13', 'V17', 'V24', 'V14', 'V20', 'V27', 'V34', 'V26', 'V30']
v += ['V36', 'V37', 'V47', 'V40', 'V48', 'V52', 'V41', 'V45']
v += ['V54', 'V65', 'V60', 'V67', 'V56', 'V68', 'V62', 'V55', 'V70']
v += ['V76', 'V89', 'V91', 'V81', 'V82', 'V87', 'V78', 'V88']
v += ['V127', 'V121', 'V99', 'V110', 'V104', 'V130', 'V129', 'V131', 'V109', 'V136', 'V116', 'V120', 'V125', 'V113', 'V118', 'V98', 'V107', 'V117', 'V115']
v += ['V138', 'V140', 'V142', 'V147', 'V155', 'V162']
v += ['V165', 'V160', 'V166']
v += ['V203', 'V207', 'V216', 'V187', 'V176', 'V173', 'V183', 'V215']
v += ['V169', 'V195', 'V201', 'V171', 'V174', 'V175', 'V209', 'V185', 'V188', 'V210', 'V198', 'V180']
v += ['V274', 'V264', 'V265', 'V261', 'V235', 'V223', 'V258', 'V260', 'V246', 'V252', 'V241', 'V266', 'V240', 'V277', 'V228', 'V226']
v += ['V220', 'V239', 'V271', 'V221', 'V234', 'V251']
v += ['V307', 'V291', 'V285', 'V290', 'V312', 'V297', 'V305', 'V320', 'V303', 'V286', 'V309', 'V284', 'V310']
v += ['V281', 'V301', 'V282', 'V315', 'V289', 'V296', 'V314', 'V283']
v += ['V332', 'V338', 'V337', 'V336', 'V325', 'V326', 'V328', 'V335']
len(v)
K.clear_session()
model = UXception(input_shape=(img_size_target,img_size_target,3))<find_best_params>
def set_approach(a,b):
return list(set(a)-set(b))
tf_V = [bool(re.search("^V"+"[0-9]+",col)) for col in xs.columns.to_list() ]
cols = xs.columns[list(~np.array(tf_V)) ].to_list()
cols +=v
print(len(cols))
def l2_norm(grad):
norm = K.sqrt(K.sum(K.square(grad)))+ K.epsilon()
return norm
class OptimizerWrapper(optimizers.Optimizer):
def __init__(self, optimizer):
self.optimizer = optimizers.get(optimizer)
 self._optimizer_get_gradients = self.optimizer.get_gradients
def get_gradients(self, loss, params):
grads = self._optimizer_get_gradients(loss, params)
 return grads
@interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
self.optimizer.get_gradients = self.get_gradients
self.optimizer.get_updates(loss, params)
 self.optimizer.get_gradients = self._optimizer_get_gradients
return self.updates
def set_weights(self, weights):
self.optimizer.set_weights(weights)
 def get_weights(self):
return self.optimizer.get_weights()
def get_config(self):
config = {'optimizer_name': self.optimizer.__class__.__name__.lower() }
optimizer_config = {'optimizer_config': self.optimizer.get_config() }
return dict(list(optimizer_config.items())+ list(config.items()))
@property
def weights(self):
return self.optimizer.weights
@property
def updates(self):
return self.optimizer.updates
@classmethod
def from_config(cls, config):
raise NotImplementedError
@classmethod
def set_normalization_function(cls, name, func):
global _NORMS
_NORMS[name] = func
@classmethod
def get_normalization_functions(cls):
global _NORMS
return sorted(list(_NORMS.keys()))
class NormalizedOptimizer(OptimizerWrapper):
def __init__(self, optimizer, normalization='l2'):
super(NormalizedOptimizer, self ).__init__(optimizer)
 if normalization not in _NORMS:
raise ValueError('`normalization` must be one of %s.
 '
'Provided was "%s".' %(str(sorted(list(_NORMS.keys()))), normalization))
self.normalization = normalization
self.normalization_fn = _NORMS[normalization]
self.lr = K.variable(1e-3, name='lr')
 def get_gradients(self, loss, params):
grads = super(NormalizedOptimizer, self ).get_gradients(loss, params)
 grads = [grad / self.normalization_fn(grad)for grad in grads]
return grads
def get_config(self):
config = {'normalization': self.normalization}
base_config = super(NormalizedOptimizer, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
@classmethod
def from_config(cls, config):
optimizer_config = {'class_name': config['optimizer_name'],
'config': config['optimizer_config']}
optimizer = optimizers.get(optimizer_config)
 normalization = config['normalization']
return cls(optimizer, normalization=normalization)
 _NORMS = {
'l2': l2_norm,
}
get_custom_objects().update({'NormalizedOptimizer': NormalizedOptimizer} )<train_model>
cols_rem = ['TransactionID', 'TransactionDT']
class SWA(keras.callbacks.Callback):
def __init__(self, filepath, swa_epoch):
super(SWA, self ).__init__()
self.filepath = filepath
self.swa_epoch = swa_epoch
def on_train_begin(self, logs=None):
self.nb_epoch = self.params['epochs']
print('Stochastic weight averaging selected for last {} epochs.' 
.format(self.nb_epoch - self.swa_epoch))
def on_epoch_end(self, epoch, logs=None):
if epoch == self.swa_epoch:
self.swa_weights = self.model.get_weights()
elif epoch > self.swa_epoch:
for i in range(len(self.swa_weights)) :
self.swa_weights[i] =(self.swa_weights[i] * 
(epoch - self.swa_epoch)+ self.model.get_weights() [i])/(( epoch - self.swa_epoch)+ 1)
 else:
pass
def on_train_end(self, logs=None):
self.model.set_weights(self.swa_weights)
 print('Final model parameters set to stochastic weight average.')
 self.model.save_weights(self.filepath)
 print('Final stochastic averaged weights saved to file.' )<choose_model_class>
cols = set_approach(cols,cols_rem )
sgd = SGD(0.01, momentum=0.9, nesterov=True)
 sgd = NormalizedOptimizer(sgd, normalization='l2' )<choose_model_class>
msk = np.arange(len(xs)) <0.8*len(xs)
 idxT = list(np.where(msk)[0])
 idxV = list(np.where(~msk)[0] )
model.compile(loss=bce_dice_loss, optimizer=sgd, metrics=[my_iou_metric] )<concatenate>
%%time
if buildLv:
n=5
preds = np.zeros(len(test_xs))
auc_lst = []
for i in range(n):
print("iteation",i)
 a,temp = xgb_lvd_fun()
preds += temp/n
auc_lst.append(a)
 del temp; x=gc.collect()
auc_lst = np.array(auc_lst)
 print(auc_lst.mean(axis=0))
print(auc_lst.std(axis=0))
format_test_preds(preds,test_xs,"baselineLV")
x_train = np.append(x_train, [np.fliplr(x)for x in x_train[0:3200]], axis=0)
 y_train = np.append(y_train, [np.fliplr(x)for x in y_train[0:3200]], axis=0)
 x_train = np.repeat(x_train,3,axis=3)
 x_valid = np.repeat(x_valid,3,axis=3 )<train_model>
%%time
if buildSkf:
preds = xgb_pred_fun()
format_test_preds(preds,test_xs,"baseline")
 del preds; x=gc.collect()
<prepare_x_and_y><EOS> 
if FI==True:
clf = xgb.XGBClassifier(n_estimators=1000,
max_depth=8,
learning_rate= 0.05,
subsample= 0.6,
colsample_bytree= 0.4,
random_state = r.randint(0,9999),
tree_method='gpu_hist')
 m = clf.fit(xs[cols].iloc[idxT], y.iloc[idxT], eval_set=[(xs[cols].iloc[idxV],y.iloc[idxV])],
eval_metric= "auc", verbose=100, early_stopping_rounds=100)
 joblib.dump(m, 'm_lv.pkl')
 fi = xgb_fi(m, xs[cols])
 fi.to_csv("fi_lv.csv",index=False)
 print(fi.cols[fi.imp>0.0005].to_list() )