Kaggle competition: Santander Customer Transaction Prediction
<SOS> metric: AUC Kaggle data source: santander-customer-transaction-prediction<define_variables>
sns.set_style('whitegrid')
i = 0
for fileName in os.listdir("test_dir_processed/"):
i = i + 1
print(i )<train_on_grid>
train_df = pd.read_csv('.. /input/train.csv')
 print('Rows: ',train_df.shape[0],'Columns: ',train_df.shape[1])
 train_df.info()
image1 = cv2.imread('.. /input/aptos2019-blindness-detection/train_images/295fdc964f6e.png')
 image1 = cv2.cvtColor(image1, cv2.COLOR_BGR2RGB)
 plt.imshow(image1 )<define_variables>
train_df['target'].value_counts()
WORKERS = 2
CHANNEL = 3
warnings.filterwarnings("ignore")
 NUM_CLASSES = 5
SEED = 72
TRAIN_NUM = 1000<split>
test_df = pd.read_csv('.. /input/test.csv' )
def df_train_test_split_preprocess(df):
image_ids = df["id_code"].values.tolist()
labels = df["diagnosis"].values.tolist()
for i in range(len(image_ids)) :
imgname = image_ids[i]
newname = str(imgname)+ ".png"
image_ids[i] = newname
xtrain, xval, ytrain, yval = train_test_split(image_ids, labels, test_size = 0.15)
 df_train = pd.DataFrame({"id_code":xtrain, "diagnosis":ytrain})
 df_val = pd.DataFrame({"id_code":xval, "diagnosis":yval})
 df_train["diagnosis"] = df_train["diagnosis"].astype('str')
 df_val["diagnosis"] = df_val["diagnosis"].astype('str')
 print("Length of Training Data :",len(df_train))
print("Length of Validation Data :",len(df_val))
return df_train, df_val<load_from_csv>
X_test1 = test_df.drop('ID_code',axis=1 )
df = pd.read_csv(".. /input/aptos2019-blindness-detection/train.csv")
 df_train, df_val = df_train_test_split_preprocess(df )<import_modules>
X1 = train_df.drop(['ID_code','target'],axis=1 )
from sklearn.metrics import f1_score, fbeta_score, cohen_kappa_score<set_options>
def _my_fit(y, y_pred, w):
diffs = np.abs(y - y_pred)
 return 100.* np.average(diffs, weights=w)
 my_fit = make_fitness(_my_fit, greater_is_better=False )
train_aug = ImageDataGenerator(rescale=1./255,
horizontal_flip = True,
zoom_range = 0.15,
vertical_flip = True,
shear_range=0.1,
rotation_range = 90
)<create_dataframe>
function_set = ['add', 'sub', 'mul', 'div', 'log',
'sqrt', 'log', 'abs', 'neg', 'inv',
'max', 'min',
'sin', 'cos', 'tan' ]
gp = SymbolicRegressor(function_set=function_set, metric = my_fit,
verbose=1, generations = 3,
random_state=0, n_jobs=-1)
train_generator = train_aug.flow_from_dataframe(dataframe = df_train,
directory = train_dir,
x_col = "id_code",
y_col = "diagnosis",
batch_size = 16,
target_size =(IMG_SIZE, IMG_SIZE),
class_mode = "categorical" )<create_dataframe>
X1a = np.array(X1)
 sam = X1a.shape[0]
col = X1a.shape[1]
X2a = np.zeros(( sam, col))
X_test1a = np.array(X_test1)
 sam_test = X_test1a.shape[0]
col_test = X_test1a.shape[1]
X_test2a = np.zeros(( sam_test, col_test))
for i in range(col):
X = np.delete(X1a,i,1)
 y = X1a[:,i]
gp.fit(X, y)
 X2a[:,i] = gp.predict(X)
 X = np.delete(X_test1a, i, 1)
 X_test2a[:,i] = gp.predict(X)
 X2 = pd.DataFrame(X2a)
 X_test2 = pd.DataFrame(X_test2a )
validation_generator = train_aug.flow_from_dataframe(dataframe = df_val,
directory = train_dir,
x_col = "id_code",
y_col = "diagnosis",
batch_size = 16,
target_size =(IMG_SIZE, IMG_SIZE),
class_mode = "categorical" )<compute_test_metric>
X = pd.concat([X1, X2], axis=1, sort=False)
 X_test = pd.concat([X_test1, X_test2], axis=1, sort=False)
 y = train_df['target']
def cohens_kappa(y_true, y_pred):
y_true_classes = tf.argmax(y_true, 1)
 y_pred_classes = tf.argmax(y_pred, 1)
 ck_val = tf.contrib.metrics.cohen_kappa(y_true_classes, y_pred_classes, 5)[1]
print(ck_val)
 return ck_val 
<feature_engineering>
n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=42 )
test_df_orig = pd.read_csv(test_csv)
 def process_test_df(test_df):
test_ids = test_df["id_code"].values.tolist()
for i in range(len(test_ids)) :
imgname = test_ids[i]
newname = str(imgname)+ ".png"
test_ids[i] = newname
test_df["id_code"] = test_ids
return test_df
test_df = process_test_df(test_df_orig )<create_dataframe>
params = {
'num_leaves': 13,
'min_data_in_leaf': 80,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.01,
'boosting': 'gbdt',
'bagging_freq': 5,
'bagging_fraction': 0.33,
'feature_fraction': 0.05,
'metric':'auc',
'verbosity': 1,
'min_sum_hessian_in_leaf': 10.0,
'num_threads': 12,
'tree_learner': 'serial',
'boost_from_average':'false'
}
test_aug = ImageDataGenerator(rescale = 1./255)
 test_generator = test_aug.flow_from_dataframe(dataframe = test_df,
directory = test_dir_processed,
x_col = "id_code",
batch_size = 1,
target_size =(IMG_SIZE, IMG_SIZE),
shuffle = False,
class_mode = None )<choose_model_class>
prediction = np.zeros(len(X_test))
for fold_n,(train_index, valid_index)in enumerate(folds.split(X,y)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 model = lgb.train(params,train_data,num_boost_round=20000,
valid_sets = [train_data, valid_data],verbose_eval=300,early_stopping_rounds = 200)
 prediction += model.predict(X_test, num_iteration=model.best_iteration)/n_fold
class RAdam(keras.optimizers.Optimizer):

 def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., weight_decay=0., amsgrad=False,
total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):
super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.weight_decay = K.variable(weight_decay, name='weight_decay')
 self.total_steps = K.variable(total_steps, name='total_steps')
 self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')
 self.min_lr = K.variable(lr, name='min_lr')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.initial_weight_decay = weight_decay
self.initial_total_steps = total_steps
self.amsgrad = amsgrad
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = self.total_steps - warmup_steps
lr = K.switch(
 t <= warmup_steps,
lr *(t / warmup_steps),
lr *(1.0 - K.minimum(t, decay_steps)/ decay_steps), 
)
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for(i, p)in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for(i, p)in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for(i, p)in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params)) ]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
 beta_2_t = K.pow(self.beta_2, t)
 sma_inf = 2.0 /(1.0 - self.beta_2)- 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t /(1.0 - beta_2_t)
 for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 m_corr_t = m_t /(1.0 - beta_1_t)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 v_corr_t = K.sqrt(vhat_t /(1.0 - beta_2_t)+ self.epsilon)
 self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t /(1.0 - beta_2_t)+ self.epsilon)
 r_t = K.sqrt(( sma_t - 4.0)/(sma_inf - 4.0)* 
(sma_t - 2.0)/(sma_inf - 2.0)*
sma_inf / sma_t)
 p_t = K.switch(sma_t > 5, r_t * m_corr_t / v_corr_t, m_corr_t)
 if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {
'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.weight_decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad,
'total_steps': float(K.get_value(self.total_steps)) ,
'warmup_proportion': float(K.get_value(self.warmup_proportion)) ,
'min_lr': float(K.get_value(self.min_lr)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<choose_model_class>
train_pool = Pool(X,y)
 m = CatBoostClassifier(iterations=300,eval_metric="AUC", boosting_type = 'Ordered')
 m.fit(X,y,silent=True)
 y_pred1 = m.predict(X_test)
 m.best_score_
from keras.models import Model
from keras.models import Sequential
from keras.layers import Dense, GlobalAveragePooling2D, Dropout, Input
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import EarlyStopping, ReduceLROnPlateau
from sklearn.model_selection import train_test_split
import keras
from keras.engine import Layer,InputSpec 
<choose_model_class>
sub = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub["target"] = prediction
sub.to_csv("submission.csv", index=False )
class GroupNormalization(Layer):

 def __init__(self,
groups=32,
axis=-1,
epsilon=1e-5,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
beta_regularizer=None,
gamma_regularizer=None,
beta_constraint=None,
gamma_constraint=None,
**kwargs):
super(GroupNormalization, self ).__init__(**kwargs)
 self.supports_masking = True
self.groups = groups
self.axis = axis
self.epsilon = epsilon
self.center = center
self.scale = scale
self.beta_initializer = initializers.get(beta_initializer)
 self.gamma_initializer = initializers.get(gamma_initializer)
 self.beta_regularizer = regularizers.get(beta_regularizer)
 self.gamma_regularizer = regularizers.get(gamma_regularizer)
 self.beta_constraint = constraints.get(beta_constraint)
 self.gamma_constraint = constraints.get(gamma_constraint)
 def build(self, input_shape):
dim = input_shape[self.axis]
if dim is None:
raise ValueError('Axis ' + str(self.axis)+ ' of '
'input tensor should have a defined dimension '
'but the layer received an input with shape ' +
str(input_shape)+ '.')
 if dim < self.groups:
raise ValueError('Number of groups(' + str(self.groups)+ ')cannot be '
'more than the number of channels(' +
str(dim)+ ' ).')
 if dim % self.groups != 0:
raise ValueError('Number of groups(' + str(self.groups)+ ')must be a '
'multiple of the number of channels(' +
str(dim)+ ' ).')
 self.input_spec = InputSpec(ndim=len(input_shape),
axes={self.axis: dim})
 shape =(dim,)
 if self.scale:
self.gamma = self.add_weight(shape=shape,
name='gamma',
initializer=self.gamma_initializer,
regularizer=self.gamma_regularizer,
constraint=self.gamma_constraint)
 else:
self.gamma = None
if self.center:
self.beta = self.add_weight(shape=shape,
name='beta',
initializer=self.beta_initializer,
regularizer=self.beta_regularizer,
constraint=self.beta_constraint)
 else:
self.beta = None
self.built = True
def call(self, inputs, **kwargs):
input_shape = K.int_shape(inputs)
 tensor_input_shape = K.shape(inputs)
 reduction_axes = list(range(len(input_shape)))
 del reduction_axes[self.axis]
broadcast_shape = [1] * len(input_shape)
 broadcast_shape[self.axis] = input_shape[self.axis] // self.groups
broadcast_shape.insert(1, self.groups)
 reshape_group_shape = K.shape(inputs)
 group_axes = [reshape_group_shape[i] for i in range(len(input_shape)) ]
group_axes[self.axis] = input_shape[self.axis] // self.groups
group_axes.insert(1, self.groups)
 group_shape = [group_axes[0], self.groups] + group_axes[2:]
group_shape = K.stack(group_shape)
 inputs = K.reshape(inputs, group_shape)
 group_reduction_axes = list(range(len(group_axes)))
 group_reduction_axes = group_reduction_axes[2:]
mean = K.mean(inputs, axis=group_reduction_axes, keepdims=True)
 variance = K.var(inputs, axis=group_reduction_axes, keepdims=True)
 inputs =(inputs - mean)/(K.sqrt(variance + self.epsilon))
inputs = K.reshape(inputs, group_shape)
 outputs = inputs
if self.scale:
broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
 outputs = outputs * broadcast_gamma
if self.center:
broadcast_beta = K.reshape(self.beta, broadcast_shape)
 outputs = outputs + broadcast_beta
outputs = K.reshape(outputs, tensor_input_shape)
 return outputs
def get_config(self):
config = {
'groups': self.groups,
'axis': self.axis,
'epsilon': self.epsilon,
'center': self.center,
'scale': self.scale,
'beta_initializer': initializers.serialize(self.beta_initializer),
'gamma_initializer': initializers.serialize(self.gamma_initializer),
'beta_regularizer': regularizers.serialize(self.beta_regularizer),
'gamma_regularizer': regularizers.serialize(self.gamma_regularizer),
'beta_constraint': constraints.serialize(self.beta_constraint),
'gamma_constraint': constraints.serialize(self.gamma_constraint)
 }
base_config = super(GroupNormalization, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape<load_pretrained>
sub1 = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub1["target"] = y_pred1
sub1.to_csv("submission1.csv", index=False )
<define_variables><EOS> 
sub2 = pd.DataFrame({"ID_code": test_df.ID_code.values})
 sub2["target"] =(prediction + y_pred1)/2
sub2.to_csv("submission2.csv", index=False )