Kaggle competition: Kannada MNIST
<SOS> metric: CategorizationAccuracy Kaggle data source: kannada-mnist<prepare_output>
get_ipython().run_line_magic('matplotlib', 'inline')
 sns.set_style("whitegrid")
N = len(subs)
 sub = subs[0].copy()
sub["FVC"] = 0
sub["Confidence"] = 0
for i in range(N):
sub["FVC"] += subs[0]["FVC"] *(1/N)
 sub["Confidence"] += subs[0]["Confidence"] *(1/N )<save_to_csv>
train = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')
 test = pd.read_csv("/kaggle/input/Kannada-MNIST/test.csv")
 validation = pd.read_csv("/kaggle/input/Kannada-MNIST/Dig-MNIST.csv")
 sample = pd.read_csv("/kaggle/input/Kannada-MNIST/sample_submission.csv" )
sub[["Patient_Week","FVC","Confidence"]].to_csv("submission_img.csv", index=False )<prepare_output>
Y_train = to_categorical(train.iloc[:,0])
 X_train = train.iloc[:, 1:].values
X_valid = validation.iloc[:, 1:].values
y_valid = to_categorical(validation.iloc[:,0])
 Y_test = to_categorical(test.iloc[:, 0])
 X_test = test.iloc[:, 1:].values
img_sub = sub[["Patient_Week","FVC","Confidence"]].copy()<load_from_csv>
X_train = X_train/255
X_valid = X_valid/255
X_test = X_test/255
ROOT = ".. /input/osic-pulmonary-fibrosis-progression"
BATCH_SIZE=128
tr = pd.read_csv(f"{ROOT}/train.csv")
 tr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])
 chunk = pd.read_csv(f"{ROOT}/test.csv")
 print("add infos")
 sub = pd.read_csv(f"{ROOT}/sample_submission.csv")
 sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])
 sub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))
sub = sub[['Patient','Weeks','Confidence','Patient_Week']]
sub = sub.merge(chunk.drop('Weeks', axis=1), on="Patient" )<concatenate>
train_datagen = ImageDataGenerator(
 rotation_range=12,
width_shift_range=0.25,
height_shift_range=0.25,
shear_range=12,
zoom_range=0.25 
)
 valid_datagen = ImageDataGenerator(
 rotation_range=12,
width_shift_range=0.25,
height_shift_range=0.25,
shear_range=12,
zoom_range=0.25)
 valid_datagen_simple = ImageDataGenerator()
tr['WHERE'] = 'train'
chunk['WHERE'] = 'val'
sub['WHERE'] = 'test'
data = tr.append([chunk, sub] )<count_unique_values>
print(tr.shape, chunk.shape, sub.shape, data.shape)
 print(tr.Patient.nunique() , chunk.Patient.nunique() , sub.Patient.nunique() ,
data.Patient.nunique())
<feature_engineering>
model = keras.Sequential([
keras.layers.Conv2D(64, kernel_size=3, activation='relu', input_shape=(28,28,1)) ,
keras.layers.BatchNormalization() ,
keras.layers.Conv2D(32, kernel_size=5, activation='relu'),
keras.layers.MaxPooling2D(pool_size=(2, 2)) ,
keras.layers.Dropout(0.25),
keras.layers.Conv2D(128, kernel_size=3, activation='relu'),
keras.layers.BatchNormalization() ,
keras.layers.Conv2D(128, kernel_size=3, activation='relu'),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
keras.layers.Conv2D(256, kernel_size=3, activation='relu'),
keras.layers.BatchNormalization() ,
keras.layers.Conv2D(512, kernel_size=3, activation='relu'),
keras.layers.BatchNormalization() ,
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.5),
keras.layers.Conv2D(1024, kernel_size = 3, strides=2, padding='same', activation='relu'),
keras.layers.BatchNormalization() ,
keras.layers.MaxPooling2D(pool_size=(2, 2)) ,
keras.layers.Dropout(0.5),
keras.layers.Flatten() ,
keras.layers.Dense(128, activation='relu'),
keras.layers.Dense(256, activation='relu'),
keras.layers.BatchNormalization() ,
keras.layers.Dense(1024,activation='relu'),
keras.layers.BatchNormalization() ,
keras.layers.Dropout(0.2),
keras.layers.Dense(10, activation='softmax')
 ])
 model.summary()
data['min_week'] = data['Weeks']
data.loc[data.WHERE=='test','min_week'] = np.nan
data['min_week'] = data.groupby('Patient')['min_week'].transform('min' )<categorify>
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'] )
base = data.loc[data.Weeks == data.min_week]
base = base[['Patient','FVC']].copy()
base.columns = ['Patient','min_FVC']
base['nb'] = 1
base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')
 base = base[base.nb==1]
base.drop('nb', axis=1, inplace=True )<merge>
history = model.fit_generator(train_datagen.flow(X_train, Y_train, batch_size = 1800),
epochs=70,validation_data =(valid_datagen.flow(X_valid, y_valid, batch_size = 30)) ,
verbose=1
)
data = data.merge(base, on='Patient', how='left')
 data['base_week'] = data['Weeks'] - data['min_week']
del base<data_type_conversions>
valid_loss,valid_acc = model.evaluate(X_valid, y_valid, verbose=0)
 print('Validation Accuracy: %.2f' %(valid_acc*100))
print('
Validation Loss: %.2f' %(valid_loss))
train_loss, train_acc = model.evaluate(X_train, Y_train, verbose = 1)
 print('
Training accuracy: %.2f
' %(train_acc*100))
COLS = ['Sex','SmokingStatus']
FE = []
for col in COLS:
for mod in data[col].unique() :
FE.append(mod)
 data[mod] =(data[col] == mod ).astype(int )<feature_engineering>
label_hot = model.predict(X_test)
 label = np.argmax(label_hot,1)
 id_ = np.arange(0,label.shape[0] )
data['age'] =(data['Age'] - data['Age'].min())/(data['Age'].max() - data['Age'].min())
 data['BASE'] =(data['min_FVC'] - data['min_FVC'].min())/(data['min_FVC'].max() - data['min_FVC'].min())
 data['week'] =(data['base_week'] - data['base_week'].min())/(data['base_week'].max() - data['base_week'].min())
 data['percent'] =(data['Percent'] - data['Percent'].min())/(data['Percent'].max() - data['Percent'].min())
 FE += ['age','percent','week','BASE']<filter>
sample = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv' )
<compute_test_metric><EOS> 
submit = pd.DataFrame({'id':id_,'label':label})
 print(submit.head(10))
submit.to_csv('submission.csv',index=False )