Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<feature_engineering>
base_dir = ".. /input/aptos2019-blindness-detection/"
train_csv = base_dir+"train.csv"
test_csv = base_dir+"test.csv"
test_dir = base_dir+"test_images/"
test_dir_processed = 'test_dir_processed'
train_dir = "train_data_cropped"
IMG_SIZE = 224
SEED = 72
for df in [x, x_test]:
for feat in features:
df['r2_'+feat] = np.round(df[feat], 2)
 df['r2_'+feat] = np.round(df[feat], 2)
 df['sum'] = df[features].sum(axis=1)
 df['min'] = df[features].min(axis=1)
 df['max'] = df[features].max(axis=1)
 df['mean'] = df[features].mean(axis=1)
 df['std'] = df[features].std(axis=1)
 df['skew'] = df[features].skew(axis=1)
 df['kurt'] = df[features].kurtosis(axis=1)
 df['med'] = df[features].median(axis=1)
 print('Train and test shape:',x.shape, x_test.shape )<train_model>
model = ExtraTreesClassifier()
model.fit(x,y )<prepare_output>
sys.path.append(os.path.abspath('.. /input/efficientnet/efficientnet-master/efficientnet-master/'))
impotrant_featues = {}
for i, j in enumerate(model.feature_importances_):
impotrant_featues[i] = j<sort_values>
import cv2
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.utils import class_weight, shuffle
variables = [k for k in sorted(impotrant_featues, key=impotrant_featues.get, reverse=True)][:50]<define_variables>
image = cv2.imread('.. /input/aptos2019-blindness-detection/train_images/295fdc964f6e.png')
 plt.imshow(image )
imp_features = []
for var in variables:
if var <200:
imp_features.append("var_"+str(var))<compute_test_metric>
i = 0
for fileName in os.listdir("train_data_cropped/"):
i = i + 1
print(i )
def dist(a,b):
return distance.euclidean(a, b)
 for i in imp_features:
mean_1 = train_df[i][train_df['target']==1].mean()
mean_0 = train_df[i][train_df['target']==0].mean()
mean_1_col = i + "_mean_1_dist"
mean_0_col = i + "_mean_0_dist"
x[mean_1_col] = x.apply(lambda p: dist(p[i], mean_1), axis=1)
 x[mean_0_col] = x.apply(lambda p: dist(p[i], mean_0), axis=1)
 x_test[mean_1_col] = x_test.apply(lambda x: dist(x[i], mean_1), axis=1)
 x_test[mean_0_col] = x_test.apply(lambda x: dist(x[i], mean_0), axis=1)
 print("done ",i )<init_hyperparams>
i = 0
for fileName in os.listdir("test_dir_processed/"):
i = i + 1
print(i )
param = {
'bagging_freq': 5,
'bagging_fraction': 0.38,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.045,
'learning_rate': 0.0095,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': 1,
'lambda_l1': 4.972,
'lambda_l2': 2.276,
'is_unbalance': True
}<init_hyperparams>
image1 = cv2.imread('.. /input/aptos2019-blindness-detection/train_images/295fdc964f6e.png')
 image1 = cv2.cvtColor(image1, cv2.COLOR_BGR2RGB)
 plt.imshow(image1 )
param = {'max_depth': 3,
'num_leaves': 6,
'min_child_samples': 200,
'scale_pos_weight': 1.0,
'subsample': 0.6,
'colsample_bytree': 0.6,
'metric': 'auc',
'nthread': 8,
'boosting_type': 'gbdt',
'objective': 'binary',
'learning_rate': 0.15,
'max_bin': 100,
'min_child_weight': 0,
'min_split_gain': 0,
'subsample_freq': 1,
'is_unbalance': True,
'bagging_freq': 5,
'bagging_fraction': 0.38,
'boost_from_average':'false',
'feature_fraction': 0.045,
'learning_rate': 0.0095,
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'tree_learner': 'serial'}<init_hyperparams>
WORKERS = 2
CHANNEL = 3
warnings.filterwarnings("ignore")
 NUM_CLASSES = 5
SEED = 72
TRAIN_NUM = 1000
param = {
'bagging_freq': 5,
'bagging_fraction': 0.335,
'boost_from_average':'false',
'boost': 'gbdt',
'feature_fraction': 0.041,
'learning_rate': 0.0083,
'max_depth': -1,
'metric':'auc',
'min_data_in_leaf': 80,
'min_sum_hessian_in_leaf': 10.0,
'num_leaves': 13,
'num_threads': 8,
'tree_learner': 'serial',
'objective': 'binary',
'verbosity': -1
}
gc.collect()<split>
def df_train_test_split_preprocess(df):
image_ids = df["id_code"].values.tolist()
labels = df["diagnosis"].values.tolist()
for i in range(len(image_ids)) :
imgname = image_ids[i]
newname = str(imgname)+ ".png"
image_ids[i] = newname
xtrain, xval, ytrain, yval = train_test_split(image_ids, labels, test_size = 0.15)
 df_train = pd.DataFrame({"id_code":xtrain, "diagnosis":ytrain})
 df_val = pd.DataFrame({"id_code":xval, "diagnosis":yval})
 df_train["diagnosis"] = df_train["diagnosis"].astype('str')
 df_val["diagnosis"] = df_val["diagnosis"].astype('str')
 print("Length of Training Data :",len(df_train))
print("Length of Validation Data :",len(df_val))
return df_train, df_val
nfold = 5
k =StratifiedKFold(n_splits=nfold, shuffle=True, random_state=49)
 oof = np.zeros(len(y))
predictions = np.zeros(len(x_test))
i =1
for train_idx, val_idx in k.split(x, y.values):
print("
fold {}".format(i))
light_train = lightgbm.Dataset(x.iloc[train_idx].values,
label = y.iloc[train_idx].values,
free_raw_data = False)
 light_val = lightgbm.Dataset(x.iloc[val_idx].values,
label = y.iloc[val_idx].values,
free_raw_data = False)
 clf = lightgbm.train(param, light_train, 9000, valid_sets=[light_val], verbose_eval=200, early_stopping_rounds=100)
 oof[val_idx] = clf.predict(x.iloc[val_idx].values, num_iteration = clf.best_iteration)
 predictions += clf.predict(x_test.values, num_iteration= clf.best_iteration)/nfold
i+=1<compute_test_metric>
df = pd.read_csv(train_csv)
 df_train, df_val = df_train_test_split_preprocess(df )
print("CV AUC: {:<0.2f}".format(metrics.roc_auc_score(y.values, oof)) )<save_to_csv>
from sklearn.metrics import f1_score, fbeta_score, cohen_kappa_score
sub_df = pd.DataFrame()
sub_df['ID_code'] = test_df['ID_code']
sub_df['target'] = predictions
sub_df.to_csv("sub1.csv", index = False )<load_from_csv>
train_aug = ImageDataGenerator(rescale=1./255,
horizontal_flip = True,
zoom_range = 0.15,
vertical_flip = True,
shear_range=0.1,
rotation_range = 90
)
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv')
 submit_data = pd.read_csv('.. /input/sample_submission.csv')
 print("Data Loaded" )<define_variables>
train_generator = train_aug.flow_from_dataframe(dataframe = df_train,
directory = train_dir,
x_col = "id_code",
y_col = "diagnosis",
batch_size = 16,
target_size =(IMG_SIZE, IMG_SIZE),
class_mode = "categorical" )
target = train_data['target']
tar = train_data['target']
features = [c for c in train_data.columns if c not in ['ID_code', 'target']]
print("Data is ready!" )<drop_column>
validation_generator = train_aug.flow_from_dataframe(dataframe = df_val,
directory = train_dir,
x_col = "id_code",
y_col = "diagnosis",
batch_size = 16,
target_size =(IMG_SIZE, IMG_SIZE),
class_mode = "categorical" )
train_data = train_data.drop(["ID_code"], axis=1)
 test_data = test_data.drop(["ID_code"], axis=1 )<filter>
def cohens_kappa(y_true, y_pred):
y_true_classes = tf.argmax(y_true, 1)
 y_pred_classes = tf.argmax(y_pred, 1)
 ck_val = tf.contrib.metrics.cohen_kappa(y_true_classes, y_pred_classes, 5)[1]
print(ck_val)
 return ck_val
target0 = train_data[target==0]
target1 = train_data[target==1]<categorify>
test_df_orig = pd.read_csv(test_csv)
 def process_test_df(test_df):
test_ids = test_df["id_code"].values.tolist()
for i in range(len(test_ids)) :
imgname = test_ids[i]
newname = str(imgname)+ ".png"
test_ids[i] = newname
test_df["id_code"] = test_ids
return test_df
test_df = process_test_df(test_df_orig )
for i in range(0,199):
ch = 'var_' + str(i)
 target0[ch] = np.random.permutation(target0[ch])
 for i in range(0,199):
ch = 'var_' + str(i)
 target1[ch] = np.random.permutation(target1[ch] )<concatenate>
test_aug = ImageDataGenerator(rescale = 1./255)
 test_generator = test_aug.flow_from_dataframe(dataframe = test_df,
directory = test_dir_processed,
x_col = "id_code",
batch_size = 1,
target_size =(IMG_SIZE, IMG_SIZE),
shuffle = False,
class_mode = None )
target0 = target0.append(target1)
 train_data = train_data.append(target0)
 train_data = train_data.sample(frac=1)
 target = train_data['target']<drop_column>
class RAdam(keras.optimizers.Optimizer):

 def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., weight_decay=0., amsgrad=False,
total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):
super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.weight_decay = K.variable(weight_decay, name='weight_decay')
 self.total_steps = K.variable(total_steps, name='total_steps')
 self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')
 self.min_lr = K.variable(lr, name='min_lr')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.initial_weight_decay = weight_decay
self.initial_total_steps = total_steps
self.amsgrad = amsgrad
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = self.total_steps - warmup_steps
lr = K.switch(
 t <= warmup_steps,
lr *(t / warmup_steps),
lr *(1.0 - K.minimum(t, decay_steps)/ decay_steps), 
)
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for(i, p)in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for(i, p)in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for(i, p)in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params)) ]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
 beta_2_t = K.pow(self.beta_2, t)
 sma_inf = 2.0 /(1.0 - self.beta_2)- 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t /(1.0 - beta_2_t)
 for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 m_corr_t = m_t /(1.0 - beta_1_t)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 v_corr_t = K.sqrt(vhat_t /(1.0 - beta_2_t)+ self.epsilon)
 self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t /(1.0 - beta_2_t)+ self.epsilon)
 r_t = K.sqrt(( sma_t - 4.0)/(sma_inf - 4.0)* 
(sma_t - 2.0)/(sma_inf - 2.0)*
sma_inf / sma_t)
 p_t = K.switch(sma_t > 5, r_t * m_corr_t / v_corr_t, m_corr_t)
 if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {
'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.weight_decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad,
'total_steps': float(K.get_value(self.total_steps)) ,
'warmup_proportion': float(K.get_value(self.warmup_proportion)) ,
'min_lr': float(K.get_value(self.min_lr)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
train_data = train_data.drop(["target"], axis=1 )<define_variables>
from keras.applications import ResNet50
from keras.models import Model
from keras.layers import Dense, GlobalAveragePooling2D, Dropout, Input
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import EarlyStopping, ReduceLROnPlateau
from sklearn.model_selection import train_test_split
import keras
from keras.engine import Layer,InputSpec
train_size = train_data.shape[0]
test_size = test_data.shape[0]<concatenate>
class GroupNormalization(Layer):

 def __init__(self,
groups=32,
axis=-1,
epsilon=1e-5,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
beta_regularizer=None,
gamma_regularizer=None,
beta_constraint=None,
gamma_constraint=None,
**kwargs):
super(GroupNormalization, self ).__init__(**kwargs)
 self.supports_masking = True
self.groups = groups
self.axis = axis
self.epsilon = epsilon
self.center = center
self.scale = scale
self.beta_initializer = initializers.get(beta_initializer)
 self.gamma_initializer = initializers.get(gamma_initializer)
 self.beta_regularizer = regularizers.get(beta_regularizer)
 self.gamma_regularizer = regularizers.get(gamma_regularizer)
 self.beta_constraint = constraints.get(beta_constraint)
 self.gamma_constraint = constraints.get(gamma_constraint)
 def build(self, input_shape):
dim = input_shape[self.axis]
if dim is None:
raise ValueError('Axis ' + str(self.axis)+ ' of '
'input tensor should have a defined dimension '
'but the layer received an input with shape ' +
str(input_shape)+ '.')
 if dim < self.groups:
raise ValueError('Number of groups(' + str(self.groups)+ ')cannot be '
'more than the number of channels(' +
str(dim)+ ' ).')
 if dim % self.groups != 0:
raise ValueError('Number of groups(' + str(self.groups)+ ')must be a '
'multiple of the number of channels(' +
str(dim)+ ' ).')
 self.input_spec = InputSpec(ndim=len(input_shape),
axes={self.axis: dim})
 shape =(dim,)
 if self.scale:
self.gamma = self.add_weight(shape=shape,
name='gamma',
initializer=self.gamma_initializer,
regularizer=self.gamma_regularizer,
constraint=self.gamma_constraint)
 else:
self.gamma = None
if self.center:
self.beta = self.add_weight(shape=shape,
name='beta',
initializer=self.beta_initializer,
regularizer=self.beta_regularizer,
constraint=self.beta_constraint)
 else:
self.beta = None
self.built = True
def call(self, inputs, **kwargs):
input_shape = K.int_shape(inputs)
 tensor_input_shape = K.shape(inputs)
 reduction_axes = list(range(len(input_shape)))
 del reduction_axes[self.axis]
broadcast_shape = [1] * len(input_shape)
 broadcast_shape[self.axis] = input_shape[self.axis] // self.groups
broadcast_shape.insert(1, self.groups)
 reshape_group_shape = K.shape(inputs)
 group_axes = [reshape_group_shape[i] for i in range(len(input_shape)) ]
group_axes[self.axis] = input_shape[self.axis] // self.groups
group_axes.insert(1, self.groups)
 group_shape = [group_axes[0], self.groups] + group_axes[2:]
group_shape = K.stack(group_shape)
 inputs = K.reshape(inputs, group_shape)
 group_reduction_axes = list(range(len(group_axes)))
 group_reduction_axes = group_reduction_axes[2:]
mean = K.mean(inputs, axis=group_reduction_axes, keepdims=True)
 variance = K.var(inputs, axis=group_reduction_axes, keepdims=True)
 inputs =(inputs - mean)/(K.sqrt(variance + self.epsilon))
inputs = K.reshape(inputs, group_shape)
 outputs = inputs
if self.scale:
broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
 outputs = outputs * broadcast_gamma
if self.center:
broadcast_beta = K.reshape(self.beta, broadcast_shape)
 outputs = outputs + broadcast_beta
outputs = K.reshape(outputs, tensor_input_shape)
 return outputs
def get_config(self):
config = {
'groups': self.groups,
'axis': self.axis,
'epsilon': self.epsilon,
'center': self.center,
'scale': self.scale,
'beta_initializer': initializers.serialize(self.beta_initializer),
'gamma_initializer': initializers.serialize(self.gamma_initializer),
'beta_regularizer': regularizers.serialize(self.beta_regularizer),
'gamma_regularizer': regularizers.serialize(self.gamma_regularizer),
'beta_constraint': constraints.serialize(self.beta_constraint),
'gamma_constraint': constraints.serialize(self.gamma_constraint)
 }
base_config = super(GroupNormalization, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
train_test = train_data
train_test = train_test.append(test_data )<create_dataframe>
input_layer = Input(shape =(IMG_SIZE,IMG_SIZE,3))
base_model = EfficientNetB5(weights = None,
include_top = False,
input_tensor = input_layer)
 base_model.load_weights('.. /input/effnet/efficientnetb5notop.h5')
 x = GlobalAveragePooling2D()(base_model.output)
 x = Dense(1024, activation='relu' )(x)
 x = Dropout(0.40 )(x)
 x = Dense(512, activation='relu' )(x)
 x = Dropout(0.3 )(x)
 out = Dense(5, activation = 'softmax' )(x)
 model = Model(inputs = input_layer, outputs = out )
%%time
train_test = pd.DataFrame(scale(train_test.values), columns=train_test.columns, index=train_test.index )<train_model>
optimizer = RAdam(lr=0.0005)
 es = EarlyStopping(monitor='cohens_kappa', mode='auto', verbose=1, patience=3,restore_best_weights=True)
 rlrop = ReduceLROnPlateau(monitor='cohens_kappa',
factor=0.2,
patience=5,
verbose=1,
mode='auto',
min_lr=1e-6)
 callback_list = [ rlrop ]
model.compile(optimizer = optimizer, loss = "categorical_crossentropy", metrics = ["accuracy",cohens_kappa] )
kmeans = KMeans(n_clusters=K)
 kmeans.fit(train_test)
 train_test['k_labels'] = kmeans.labels_<feature_engineering>
K.get_session().run(tf.local_variables_initializer() )
temp_df = train_test['k_labels']
train_test['weight1'] = temp_df.apply(lambda x: x*3)
 train_test['weight2'] = temp_df.apply(lambda x: x*7)
 train_test['weight3'] = temp_df.apply(lambda x: x*11)
 train_test['weight4'] = temp_df.apply(lambda x: x*5)
 train_test['weight5'] = temp_df.apply(lambda x: x*13)
 train_test['weight6'] = temp_df.apply(lambda x: x*17)
 train_test['weight7'] = temp_df.apply(lambda x: x*19)
 train_test['weight8'] = temp_df.apply(lambda x: x*23)
 train_test['weight9'] = temp_df.apply(lambda x: x*29)
 train_test['weight10'] = temp_df.apply(lambda x: x*31)
 train_test['weight11'] = temp_df.apply(lambda x: x*37)
 train_test['weight12'] = temp_df.apply(lambda x: x*41)
 train_test['weight13'] = temp_df.apply(lambda x: x*43)
 train_test['weight14'] = temp_df.apply(lambda x: x*47)
 train_test['weight15'] = temp_df.apply(lambda x: x*53)
 train_test['weight16'] = temp_df.apply(lambda x: x*59)
 train_test['weight17'] = temp_df.apply(lambda x: x*61)
 train_test['weight18'] = temp_df.apply(lambda x: x*67)
 train_test['weight19'] = temp_df.apply(lambda x: x*71)
 train_test['weight20'] = temp_df.apply(lambda x: x*73)
 train_test.head(2 )<define_search_space>
history = model.fit_generator(generator = train_generator,
steps_per_epoch = len(train_generator),
epochs = 18,
validation_data = validation_generator,
validation_steps = len(validation_generator),
callbacks = callback_list )
cluster_count = pd.Series()
for i in range(0,K):
print(i , " " , train_test.loc[train_test['k_labels']==i].shape[0])
 temp =(train_test.loc[train_test['k_labels']==i].shape[0])
 cluster_count = cluster_count.append(pd.Series(temp))
clusterCountVal = pd.DataFrame()
clusterCountVal = cluster_count.to_frame()<split>
predprobs = model.predict_generator(test_generator, steps=len(test_generator))
train_data = train_test[:test_size]
test_data = train_test[train_size:]<init_hyperparams>
shutil.rmtree('train_data_cropped')
 shutil.rmtree('test_dir_processed' )
params = {}
params['bagging_freq'] = 5
params['bagging_fraction'] = 0.0331
params['random_state'] = 42
params['learning_rate'] = 0.0123
params['boost_from_average'] = False
params['boosting_type'] = 'gbdt'
params['feature_fraction'] = 0.045
params['objective'] = 'binary'
params['metric'] = 'auc'
params['min_data_in_leaf'] = 80
params['num_leaves'] = 13
params['num_threads'] = 8
params['tree_learner'] = 'serial'
params['max_depth'] = -1
params['min_sum_hessian_in_leaf'] = 10.0
params['verbosity'] = 1
params['bagging_seed'] = 42
params['seed'] = 42<create_dataframe>
predictions = []
for i in predprobs:
predictions.append(np.argmax(i))
num_folds = 10
features = [c for c in test_data.columns if c not in ['ID_code', 'target']]
folds = StratifiedKFold(n_splits=num_folds,shuffle=True, random_state=42)
 oof = np.zeros(len(train_data))
getVal = np.zeros(len(train_data))
predictions = np.zeros(len(tar))
print(predictions.shape)
 feature_importance_df = pd.DataFrame()<prepare_x_and_y>
test_df_orig["diagnosis"] = predictions
%%time
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_data.values, target.values)) :
X_train, y_train = train_data.iloc[trn_idx][features], target.iloc[trn_idx]
X_valid, y_valid = train_data.iloc[val_idx][features], target.iloc[val_idx]
X_tr, y_tr = X_train.values, y_train.values
X_tr = pd.DataFrame(X_tr)
 print("Fold idx:{}".format(fold_ + 1))
trn_data = lgb.Dataset(X_tr, label=y_tr)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 clf = lgb.train(params, trn_data, 1000000, valid_sets = [trn_data, val_data], verbose_eval=5000, early_stopping_rounds = 4000)
 oof[val_idx] = clf.predict(train_data.iloc[val_idx][features], num_iteration=clf.best_iteration)
 getVal[val_idx]+= clf.predict(train_data.iloc[val_idx][features], num_iteration=clf.best_iteration)/ folds.n_splits
predictions += clf.predict(test_data[features], num_iteration=clf.best_iteration)/ folds.n_splits 
<save_to_csv>
test_ids = test_df_orig["id_code"].values.tolist()
for i in range(len(test_ids)) :
imgname = test_ids[i]
newname = imgname.split('.')[0]
test_ids[i] = newname
test_df_orig["id_code"] = test_ids
submit_data['target'] = pd.DataFrame(predictions)
 submit_data.to_csv("LGBMwithClustering.csv", index=False )<import_modules>
test_df_orig.to_csv('submission.csv',index=False )
<load_from_csv><EOS> 
subfile = pd.read_csv('submission.csv')
 subfile.head(3 )