Kaggle competition: CareerCon 2019 - Help Navigate Robots
<SOS> metric: categorizationaccuracy Kaggle data source: career-con-2019<define_variables>
%matplotlib inline
seed_everything(42 )<set_options>
warnings.filterwarnings('ignore')
 os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config )<init_hyperparams>
train = pd.read_csv('.. /input/career-con-2019/X_train.csv')
 target = pd.read_csv('.. /input/career-con-2019/y_train.csv')
 test = pd.read_csv('.. /input/career-con-2019/X_test.csv')
 sub = pd.read_csv('.. /input/career-con-2019/sample_submission.csv' )
Dropout_model = 0.385
FVC_weight = 0.2
Confidence_weight = 0.2<create_dataframe>
train['series_id'].nunique() , test['series_id'].nunique()
commits_df = pd.DataFrame(columns = ['commit_num', 'FVC_weight', 'Dropout_model', 'LB_score', 'seed'] )<feature_engineering>
target['group_id'].nunique()
n=0
commits_df.loc[n,'commit_num'] = 1
commits_df.loc[n,'Dropout_model'] = 0.4
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8111<feature_engineering>
target['surface'].value_counts().reset_index().rename(columns={'index': 'target'} )
n=1
commits_df.loc[n,'commit_num'] = 4
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8105<feature_engineering>
n=2
commits_df.loc[n,'commit_num'] = 6
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.35
commits_df.loc[n,'LB_score'] = -6.8158<feature_engineering>
n=3
commits_df.loc[n,'commit_num'] = 8
commits_df.loc[n,'Dropout_model'] = 0.35
commits_df.loc[n,'FVC_weight'] = 0.25
commits_df.loc[n,'LB_score'] = -6.8107<feature_engineering>
n=4
commits_df.loc[n,'commit_num'] = 9
commits_df.loc[n,'Dropout_model'] = 0.35
commits_df.loc[n,'FVC_weight'] = 0.3
commits_df.loc[n,'LB_score'] = -6.8125<feature_engineering>
n=5
commits_df.loc[n,'commit_num'] = 10
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8089<feature_engineering>
train_df = train.copy()
test_df = test.copy()
n=6
commits_df.loc[n,'commit_num'] = 11
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.15
commits_df.loc[n,'LB_score'] = -6.8100<feature_engineering>
def feat_diff(data):
for col in data.columns[3:]:
data[col+'_diff'] = data.groupby(['series_id'])[col].diff().fillna(0)
 return data
def feat_make(data):
data = feat_diff(data)
 X0 = data.loc[:, 'orientation_X':'orientation_W'].values.reshape(-1, 128, 4)
 X1 = data.loc[:, 'angular_velocity_X': 'angular_velocity_Z'].values.reshape(-1, 128, 3)
 X2 = data.loc[:, 'linear_acceleration_X': 'linear_acceleration_Z'].values.reshape(-1, 128, 3)
 X7 = data.loc[:, 'orientation_X_diff':'orientation_W_diff'].values.reshape(-1, 128, 4)
 X8 = data.loc[:, 'angular_velocity_X_diff': 'angular_velocity_Z_diff'].values.reshape(-1, 128, 3)
 X9 = data.loc[:, 'linear_acceleration_X_diff': 'linear_acceleration_Z_diff'].values.reshape(-1, 128, 3)
 return X0, X1, X2, X7, X8, X9
n=7
commits_df.loc[n,'commit_num'] = 13
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.175
commits_df.loc[n,'LB_score'] = -6.8096<feature_engineering>
def mfft(x):
return [ x/math.sqrt(128.0)for x in np.absolute(np.fft.fft(x)) ][1:65]
def feat_fft(X_trn):
ars=[]
for ar in X_trn:
ar= ar.T.tolist()
ms= []
for line in ar:
m= mfft(line)
 ms.append(m)
 ms= np.array(ms ).T.tolist()
ars.append(ms)
 ars= np.array(ars)
 return ars
n=8
commits_df.loc[n,'commit_num'] = 14
commits_df.loc[n,'Dropout_model'] = 0.36
commits_df.loc[n,'FVC_weight'] = 0.225
commits_df.loc[n,'LB_score'] = -6.8100<feature_engineering>
n=9
commits_df.loc[n,'commit_num'] = 15
commits_df.loc[n,'Dropout_model'] = 0.32
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092<feature_engineering>
le = preprocessing.LabelEncoder()
le.fit(target['surface'])
 y_train= le.transform(target['surface'])
 le.classes_
n=10
commits_df.loc[n,'commit_num'] = 16
commits_df.loc[n,'Dropout_model'] = 0.25
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8093<feature_engineering>
le.transform(['carpet', 'concrete', 'fine_concrete', 'hard_tiles',
'hard_tiles_large_space', 'soft_pvc', 'soft_tiles', 'tiled',
'wood'] )
n=11
commits_df.loc[n,'commit_num'] = 17
commits_df.loc[n,'Dropout_model'] = 0.25
commits_df.loc[n,'FVC_weight'] = 0.5
commits_df.loc[n,'LB_score'] = -6.8283<feature_engineering>
X_trn_q, X_trn_a, X_trn_l, X_trn_q_d, X_trn_a_d, X_trn_l_d = feat_make(train_df)
 X_tst_q, X_tst_a, X_tst_l, X_tst_q_d, X_tst_a_d, X_tst_l_d = feat_make(test_df )
n=12
commits_df.loc[n,'commit_num'] = 18
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8087<feature_engineering>
X_trn_q_f = feat_fft(X_trn_q)
 X_trn_a_f = feat_fft(X_trn_a)
 X_trn_l_f = feat_fft(X_trn_l)
 X_trn_q_d_f = feat_fft(X_trn_q_d)
 X_trn_a_d_f = feat_fft(X_trn_a_d)
 X_trn_l_d_f = feat_fft(X_trn_l_d)
 X_tst_q_f = feat_fft(X_tst_q)
 X_tst_a_f = feat_fft(X_tst_a)
 X_tst_l_f = feat_fft(X_tst_l)
 X_tst_q_d_f = feat_fft(X_tst_q_d)
 X_tst_a_d_f = feat_fft(X_tst_a_d)
 X_tst_l_d_f = feat_fft(X_tst_l_d)
n=13
commits_df.loc[n,'commit_num'] = 19
commits_df.loc[n,'Dropout_model'] = 0.39
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8090<feature_engineering>
def run_set(X_trn):
for i in run_id:
X_run = target[:][target['run_id']==i]
idx= X_run.index
X= X_trn[idx].tolist()
X=[item for sublist in X for item in sublist]
end=int(len(X)/2048)*2048
X=X[:end]
X=np.array(X ).reshape(-1, 2048, X_trn.shape[-1])
 return X
def test_run_set(X_trn):
X_=[]
for i in test_run_id:
X_run = test_target[:][test_target['run_id']==i]
idx= X_run.index
X= X_trn[idx].tolist()
X=[item for sublist in X for item in sublist]
end=int(len(X)/2048)*2048
X=X[:end]
X=np.array(X ).reshape(-1, 2048, X_trn.shape[-1])
 return X
n=14
commits_df.loc[n,'commit_num'] = 20
commits_df.loc[n,'Dropout_model'] = 0.37
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8092<feature_engineering>
X_trn_ = [X_trn_q, X_trn_a, X_trn_l, X_trn_q_d, X_trn_a_d, X_trn_l_d,
X_trn_a_f, X_trn_l_f, X_trn_a_d_f, X_trn_l_d_f]
n=15
commits_df.loc[n,'commit_num'] = 21
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.19
commits_df.loc[n,'LB_score'] = -6.8093<feature_engineering>
X_train = [run_set(x)for x in X_trn_]
n=16
commits_df.loc[n,'commit_num'] = 23
commits_df.loc[n,'Dropout_model'] = 0.38
commits_df.loc[n,'FVC_weight'] = 0.2
commits_df.loc[n,'LB_score'] = -6.8402<feature_engineering>
X_test= [X_tst_q, X_tst_a, X_tst_l, X_tst_q_d, X_tst_a_d, X_tst_l_d,
X_tst_a_f, X_tst_l_f, X_tst_a_d_f, X_tst_l_d_f]
commits_df['LB_score'] = pd.to_numeric(commits_df['LB_score'])
 commits_df['max'] = 0
commits_df.loc[commits_df['LB_score'].idxmax() , 'max'] = 1<feature_engineering>
class Head:
def build(n_steps, n_features):
inputA = Input(shape=(n_steps, n_features))
x = Conv1D(filters=100, kernel_size=10, activation='relu' )(inputA)
 x = Conv1D(filters=100, kernel_size=10, activation='relu' )(x)
 x = MaxPooling1D(pool_size=3 )(x)
 x = Conv1D(filters=160, kernel_size=10, activation='relu' )(x)
 x = Conv1D(filters=160, kernel_size=10, activation='relu' )(x)
 x = AveragePooling1D(pool_size=3 )(x)
 x = Dropout(0.5 )(x)
 x = Flatten()(x)
 x = Model(inputs= inputA, outputs= x)
 return x
def build_f(n_steps, n_features):
inputA = Input(shape=(n_steps, n_features))
x = Conv1D(filters=100, kernel_size=5, activation='relu' )(inputA)
 x = Conv1D(filters=100, kernel_size=5, activation='relu' )(x)
 x = MaxPooling1D(pool_size=3 )(x)
 x = Conv1D(filters=160, kernel_size=5, activation='relu' )(x)
 x = Conv1D(filters=160, kernel_size=5, activation='relu' )(x)
 x = AveragePooling1D(pool_size=3 )(x)
 x = Dropout(0.5 )(x)
 x = Flatten()(x)
 x = Model(inputs= inputA, outputs= x)
 return x
commits_df['seed'] = 42
commits_df.loc[commits_df['commit_num'] == 23, 'seed'] = 0<load_from_csv>
def sub_model(X_trn, y_trn):
flatA1 = Head.build(2048, 4)
 flatA2 = Head.build(2048, 3)
 flatA3 = Head.build(2048, 3)
 flatB1 = Head.build(2048, 4)
 flatB2 = Head.build(2048, 3)
 flatB3 = Head.build(2048, 3)
 fftA2 = Head.build_f(2048, 3)
 fftA3 = Head.build_f(2048, 3)
 fftB2 = Head.build_f(2048, 3)
 fftB3 = Head.build_f(2048, 3)
 x= concatenate([
flatA1.output, flatA2.output, flatA3.output,
flatB1.output, flatB2.output, flatB3.output,
fftA2.output, fftA3.output,
fftB2.output, fftB3.output,
])
 x = Dense(500, activation='relu' )(x)
 x = Dense(100, activation='relu' )(x)
 x = Dense(n_surfaces, activation='softmax' )(x)
 model = Model(inputs=[
flatA1.input, flatA2.input, flatA3.input,
flatB1.input, flatB2.input, flatB3.input,
fftA2.input, fftA3.input,
fftB2.input, fftB3.input,
],
outputs= x)
 plot_model(model, show_shapes=True, to_file='multichannel.png')
 model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.fit(x= X_trn, y= y_trn, epochs= epochs, batch_size= batch_size, verbose= verbose)
 return model
train = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/train.csv' )<concatenate>
def load_all_models(n_models):
all_models = []
for i in range(n_models):
filename = 'model_' + str(i)+ '.h5'
model = load_model(filename)
 all_models.append(model)
 print('>loaded %s' % filename)
 return all_models
def get_tab(df):
vector = [(df.Age.values[0] - 30)/ 30]
if df.Sex.values[0] == 'male':
vector.append(0)
 else:
vector.append(1)
 if df.SmokingStatus.values[0] == 'Never smoked':
vector.extend([0,0])
 elif df.SmokingStatus.values[0] == 'Ex-smoker':
vector.extend([1,1])
 elif df.SmokingStatus.values[0] == 'Currently smokes':
vector.extend([0,1])
 else:
vector.extend([1,0])
 return np.array(vector )<feature_engineering>
def define_stacked_model(members):
for i in range(len(members)) :
model = members[i]
for layer in model.layers:
layer.trainable = False
layer.name = 'ensemble_' + str(i)+ '_' + layer.name
ensemble_visible = [model.input for model in members]
ensemble_visible = [item for sublist in ensemble_visible for item in sublist]
ensemble_outputs = [model.output for model in members]
merge = concatenate(ensemble_outputs)
 hidden = Dense(45, activation='relu' )(merge)
 output = Dense(9, activation='softmax' )(hidden)
 model = Model(inputs=ensemble_visible, outputs=output)
 plot_model(model, show_shapes=True, to_file='model_graph.png')
 model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
A = {}
TAB = {}
P = []
for i, p in tqdm(enumerate(train.Patient.unique())) :
sub = train.loc[train.Patient == p, :]
fvc = sub.FVC.values
weeks = sub.Weeks.values
c = np.vstack([weeks, np.ones(len(weeks)) ] ).T
a, b = np.linalg.lstsq(c, fvc)[0]
A[p] = a
TAB[p] = get_tab(sub)
 P.append(p )<feature_engineering>
verbose, epochs, batch_size = 2, 15, 32
n_surfaces = 9
n_models= 2
folds = StratifiedShuffleSplit(n_splits= 2, test_size= 0.5, random_state=0)
 np.set_printoptions(precision=2)
 class_names= le.classes_
class IGenerator(Sequence):
BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']
def __init__(self, keys, a, tab, batch_size=32):
self.keys = [k for k in keys if k not in self.BAD_ID]
self.a = a
self.tab = tab
self.batch_size = batch_size
self.train_data = {}
for p in train.Patient.values:
self.train_data[p] = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 def __len__(self):
return 1000
def __getitem__(self, idx):
x = []
a, tab = [], []
keys = np.random.choice(self.keys, size = self.batch_size)
 for k in keys:
try:
i = np.random.choice(self.train_data[k], size=1)[0]
img = get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')
 x.append(img)
 a.append(self.a[k])
 tab.append(self.tab[k])
 except:
print(k, i)
 x,a,tab = np.array(x), np.array(a), np.array(tab)
 x = np.expand_dims(x, axis=-1)
 return [x, tab] , a<choose_model_class>
X_trn= X_train
y_trn= y_train
for fold_,(trn, val)in enumerate(folds.split(np.zeros(len(y_trn)) , y_trn)) :
print("fold {}".format(fold_))
y_oh = to_categorical(y_trn)
 xtrn= [x[trn] for x in X_trn]
for i in range(n_models):
model = sub_model(xtrn, y_oh[trn])
 filename = 'model_' + str(i)+ '.h5'
model.save(filename)
 print('>Saved %s' % filename)
 members = load_all_models(n_models)
 print('Loaded %d models' % len(members))
stacked_model = define_stacked_model(members)
 X_t = [x[trn] for x in X_trn]
X_t = [X_t for _ in range(n_models)]
X_t = [item for sublist in X_t for item in sublist]
X_v = [x[val] for x in X_trn]
X_v = [X_v for _ in range(n_models)]
X_v = [item for sublist in X_v for item in sublist]
stacked_model.fit(X_t, y_oh[trn], validation_data=(X_v, y_oh[val]), epochs=epochs, verbose=verbose)
 plt.plot(stacked_model.history.history['acc'])
 plt.plot(stacked_model.history.history['val_acc'])
 plt.title('model accuracy')
 plt.ylabel('accuracy')
 plt.xlabel('epoch')
 plt.legend(['train', 'val'], loc='upper left')
 plt.show()
plt.plot(stacked_model.history.history['loss'])
 plt.plot(stacked_model.history.history['val_loss'])
 plt.title('model loss')
 plt.ylabel('loss')
 plt.xlabel('epoch')
 plt.legend(['train', 'val'], loc='upper left')
 plt.show()
y_pred_ = stacked_model.predict(X_v, verbose=verbose)
 y_pred = y_pred_.argmax(axis=-1)
 acc = accuracy_score(y_trn[val], y_pred)
 print('Stacked Test Accuracy: %.3f' % acc)
 plot_confusion_matrix(y_trn[val], y_pred, classes= class_names,
title='Confusion matrix, without normalization')
 plot_confusion_matrix(y_trn[val], y_pred, classes= class_names,
normalize=True, title='Normalized confusion matrix')
Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D,
LeakyReLU, Concatenate 
)
 def get_efficientnet(model, shape):
models_dict = {
'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),
'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),
'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),
'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),
'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),
'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),
'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),
'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)
 }
return models_dict[model]
def build_model(shape=(512, 512, 1), model_class=None):
inp = Input(shape=shape)
 base = get_efficientnet(model_class, shape)
 x = base(inp)
 x = GlobalAveragePooling2D()(x)
 inp2 = Input(shape=(4,))
x2 = tf.keras.layers.GaussianNoise(0.2 )(inp2)
 x = Concatenate()([x, x2])
 x = Dropout(Dropout_model )(x)
 x = Dense(1 )(x)
 model = Model([inp, inp2] , x)
 weights = [w for w in os.listdir('.. /input/osic-model-weights')if model_class in w][0]
model.load_weights('.. /input/osic-model-weights/' + weights)
 return model
model_classes = ['b5']
models = [build_model(shape=(512, 512, 1), model_class=m)for m in model_classes]
print('Number of models: ' + str(len(models)) )<split>
n_folds = 2
for f in range(n_folds):
print('fold',f)
 y_oh = to_categorical(y_train)
 for i in range(n_models):
model = sub_model(X_train, y_oh)
 filename = 'model_' + str(i)+ '.h5'
model.save(filename)
 print('>Saved %s' % filename)
 members = load_all_models(n_models)
 print('Loaded %d models' % len(members))
stacked_model = define_stacked_model(members)
 X_t = X_train
X_t = [X_t for _ in range(n_models)]
X_t = [item for sublist in X_t for item in sublist]
stacked_model.fit(X_t, y_oh, epochs=5, verbose=verbose)
 X_tst = X_test
X_tst = [X_tst for _ in range(n_models)]
X_tst = [item for sublist in X_tst for item in sublist]
y_test_ = stacked_model.predict(X_tst, verbose=verbose)
 plt.plot(model.history.history['acc'])
 plt.ylabel('accuracy')
 plt.xlabel('epoch')
 plt.plot(model.history.history['loss'])
 if f==0:
y_test = np.zeros_like(y_test_)
 y_test += y_test_
y_test = y_test/ n_folds
tr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.8 )<compute_test_metric>
y_test = y_test.argmax(axis=-1 )
def score(fvc_true, fvc_pred, sigma):
sigma_clip = np.maximum(sigma, 70)
 delta = np.abs(fvc_true - fvc_pred)
 delta = np.minimum(delta, 1000)
 sq2 = np.sqrt(2)
 metric =(delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)
 return np.mean(metric )<define_variables>
y_run=[]
for i in run_id:
X_run = target[:][target['run_id']==i]
y = le.transform(X_run['surface'])[0]
y_run.append(y )
subs = []
for model in models:
metric = []
for q in tqdm(range(1, 10)) :
m = []
for p in vl_p:
x = []
tab = []
if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:
continue
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/train/{p}/{i}'))
tab.append(get_tab(train.loc[train.Patient == p, :]))
if len(x)< 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q / 10)
 percent_true = train.Percent.values[train.Patient == p]
fvc_true = train.FVC.values[train.Patient == p]
weeks_true = train.Weeks.values[train.Patient == p]
fvc = a *(weeks_true - weeks_true[0])+ fvc_true[0]
percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])
 m.append(score(fvc_true, fvc, percent))
print(np.mean(m))
metric.append(np.mean(m))
q =(np.argmin(metric)+ 1)/ 10
sub = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/sample_submission.csv')
 test = pd.read_csv('.. /input/osic-pulmonary-fibrosis-progression/test.csv')
 A_test, B_test, P_test,W, FVC= {}, {}, {},{},{}
STD, WEEK = {}, {}
for p in test.Patient.unique() :
x = []
tab = []
ldir = os.listdir(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/')
 for i in ldir:
if int(i[:-4])/ len(ldir)< 0.8 and int(i[:-4])/ len(ldir)> 0.15:
x.append(get_img(f'.. /input/osic-pulmonary-fibrosis-progression/test/{p}/{i}'))
tab.append(get_tab(test.loc[test.Patient == p, :]))
if len(x)<= 1:
continue
tab = np.array(tab)
 x = np.expand_dims(x, axis=-1)
 _a = model.predict([x, tab])
 a = np.quantile(_a, q)
 A_test[p] = a
B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]
P_test[p] = test.Percent.values[test.Patient == p]
WEEK[p] = test.Weeks.values[test.Patient == p]
for k in sub.Patient_Week.values:
p, w = k.split('_')
 w = int(w)
 fvc = A_test[p] * w + B_test[p]
sub.loc[sub.Patient_Week == k, 'FVC'] = fvc
sub.loc[sub.Patient_Week == k, 'Confidence'] =(
 P_test[p] - A_test[p] * abs(WEEK[p] - w)
)
 _sub = sub[["Patient_Week","FVC","Confidence"]].copy()
subs.append(_sub )<prepare_output>
for run_id, i in enumerate(y_test):
if test_target['run_id']==run_id:
test_target['surface']= le.inverse_transform(i )
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )<save_to_csv>
subm = pd.DataFrame()
subm['series_id'] = test_target['series_id']
subm['surface']= test_target['surface']
subm = subm.sort_values('series_id')
 subm.to_csv("submission.csv", index= False )