Kaggle competition: IEEE's Signal Processing Society - Camera Model Identification
def convert_folds_prediction_to_sub(folds_prediction_dict):
for i, fold in enumerate(folds_prediction_dict):
if i == 0:
prediction = folds_prediction_dict[fold].sort_values('qa_id' ).set_index('qa_id' ).values
else:
prediction += folds_prediction_dict[fold].sort_values('qa_id' ).set_index('qa_id' ).values
prediction /= len(folds_prediction_dict)
 return prediction<categorify>
list_train = [filepath for filepath in list_paths if "train/" in filepath]
shuffle(list_train)
 list_test = [filepath for filepath in list_paths if "test/" in filepath]
list_train = list_train
list_test = list_test
index = [os.path.basename(filepath)for filepath in list_test]
prediction = convert_folds_prediction_to_sub(fold_prediction_dict )<create_dataframe>
list_classes = list(set([os.path.dirname(filepath ).split(os.sep)[-1] for filepath in list_paths if "train" in filepath]))
raw_prediction = prediction.copy()<import_modules>
list_classes = ['Sony-NEX-7',
'Motorola-X',
'HTC-1-M7',
'Samsung-Galaxy-Note3',
'Motorola-Droid-Maxx',
'iPhone-4s',
'iPhone-6',
'LG-Nexus-5x',
'Samsung-Galaxy-S4',
'Motorola-Nexus-6']
class OptimizedRounder(object):

 def __init__(self):
self.coef_ = 0
def _spearmanr_loss(self, coef, X, y, labels):

 X_p = pd.cut(X, [-np.inf] + list(np.sort(coef)) +
[np.inf], labels=labels)
 return -spearmanr(y, X_p ).correlation
def fit(self, X, y, initial_coef):

 labels = self.labels
loss_partial = partial(self._spearmanr_loss, X=X, y=y, labels=labels)
 self.coef_ = sp.optimize.minimize(
 loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):

 labels = self.labels
return pd.cut(X, [-np.inf] + list(np.sort(coef)) +
[np.inf], labels=labels)
 def coefficients(self):

 return self.coef_['x']
def set_labels(self, labels):
self.labels = labels<load_pretrained>
def get_class_from_path(filepath):
return os.path.dirname(filepath ).split(os.sep)[-1]
def read_and_resize(filepath):
im_array = np.array(Image.open(( filepath)) , dtype="uint8")
 pil_im = Image.fromarray(im_array)
 new_array = np.array(pil_im.resize(( 256, 256)))
 return new_array/255
def label_transform(labels):
labels = pd.get_dummies(pd.Series(labels))
label_index = labels.columns.values
return labels, label_index
with open('/kaggle/input/sub-e121-e129-bert1-roberta2-xlnet1/optRs.pkl', 'rb')as fin:
optRs = pickle.load(fin )<define_variables>
X_train = np.array([read_and_resize(filepath)for filepath in list_train])
 X_test = np.array([read_and_resize(filepath)for filepath in list_test] )
res_prediction = []
for i in tqdm(list(range(30))):
y_pred = prediction[:, i]
if i not in [2,4,5,6,7,8,9,11,12,13,14,15,16,19,23,25,]:
res_prediction.append(y_pred)
 continue
optR = optRs[i]
res = optR.predict(y_pred, optR.coefficients() ).astype(float)
 res_prediction.append(res)
 prediction = np.asarray(res_prediction ).T<load_from_csv>
labels = [get_class_from_path(filepath)for filepath in list_train]
y, label_index = label_transform(labels)
 y = np.array(y )
sub_df = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv')
 sub_df.head()<prepare_output>
input_shape =(256, 256, 3)
 nclass = len(label_index)
 def get_model() :
nclass = len(label_index)
 inp = Input(shape=input_shape)
 norm_inp = BatchNormalization()(inp)
 img_1 = Convolution2D(16, kernel_size=3, activation=activations.relu, padding="same" )(norm_inp)
 img_1 = Convolution2D(16, kernel_size=3, activation=activations.relu, padding="same" )(img_1)
 img_1 = MaxPooling2D(pool_size=(3, 3))(img_1)
 img_1 = Dropout(rate=0.2 )(img_1)
 img_1 = Convolution2D(32, kernel_size=3, activation=activations.relu, padding="same" )(img_1)
 img_1 = Convolution2D(32, kernel_size=3, activation=activations.relu, padding="same" )(img_1)
 img_1 = MaxPooling2D(pool_size=(3, 3))(img_1)
 img_1 = Dropout(rate=0.2 )(img_1)
 img_1 = Convolution2D(64, kernel_size=2, activation=activations.relu, padding="same" )(img_1)
 img_1 = Convolution2D(20, kernel_size=2, activation=activations.relu, padding="same" )(img_1)
 img_1 = GlobalMaxPool2D()(img_1)
 img_1 = Dropout(rate=0.2 )(img_1)
 dense_1 = Dense(20, activation=activations.relu )(img_1)
 dense_1 = Dense(nclass, activation=activations.softmax )(dense_1)
 model = models.Model(inputs=inp, outputs=dense_1)
 opt = optimizers.Adam()
model.compile(optimizer=opt, loss=losses.categorical_crossentropy, metrics=['acc'])
 model.summary()
return model
sub_df.iloc[:, 1:] = prediction
sub_df.head()<feature_engineering>
model = get_model()
file_path="weights.best.hdf5"
checkpoint = ModelCheckpoint(file_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
 early = EarlyStopping(monitor="val_acc", mode="max", patience=1)
 callbacks_list = [checkpoint, early]
history = model.fit(X_train, y, validation_split=0.1, epochs=3, shuffle=True, verbose=2,
callbacks=callbacks_list)
 model.load_weights(file_path)
sub_df.loc[(sub_df['question_fact_seeking'] < np.min(optRs[4].coefficients())).values, 'question_opinion_seeking'] = 1 .<define_variables>
predicts = model.predict(X_test)
 predicts = np.argmax(predicts, axis=1)
 predicts = [label_index[p] for p in predicts]
<count_unique_values><EOS> 
df = pd.DataFrame(columns=['fname', 'camera'])
 df['fname'] = index
df['camera'] = predicts
df.to_csv("sub.csv", index=False )