Kaggle competition: Kannada MNIST
df_test = pd.read_csv('.. /input/santander-product-recommendation/test_ver2.csv', usecols = demographic_cols )<filter>
kannada_test_data = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')
 kannada_train_data =pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')
 kannada_dig_data = pd.read_csv('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv')
 print(kannada_test_data.head())
 print(kannada_train_data.head())
 print(kannada_dig_data.head())
 print(kannada_test_data.shape, kannada_train_data.shape, kannada_dig_data.shape)
df_train = df_train[df_train['ind_nuevo'] == 0]
df_train = df_train[df_train['antiguedad'] != -999999]
df_train = df_train[df_train['indrel'] == 1]
df_train = df_train[df_train['indresi'] == 'S']
df_train = df_train[df_train['indfall'] == 'N']
df_train = df_train[df_train['tipodom'] == 1]
df_train = df_train[df_train['ind_empleado'] == 'N']
df_train = df_train[df_train['pais_residencia'] == 'ES']
df_train = df_train[df_train['indrel_1mes'] == 1]
df_train = df_train[df_train['tiprel_1mes'] ==('A' or 'I')]
df_train = df_train[df_train['indext'] == 'N']<count_missing_values>
kannada_train_data['label'].value_counts().sort_index()
df_train.isnull().sum()<count_values>
kannada_dig_data['label'].value_counts().sort_index()
df_train.tiprel_1mes.value_counts()<drop_column>
X = kannada_train_data.drop(['label'], axis=1 ).values
Y = kannada_train_data['label']
X_test = kannada_test_data.drop(['id'], axis=1 ).values
X_test_1 = kannada_dig_data.drop(['label'], axis=1 ).values
Y_test_1 = kannada_dig_data['label']
print(X.shape, Y.shape, X_test.shape, X_test_1.shape, Y_test_1.shape )
drop_column = ['ind_nuevo','indrel','indresi','indfall','tipodom','ind_empleado','pais_residencia','indrel_1mes','indext','conyuemp','fecha_alta','tiprel_1mes']
df_train.drop(drop_column, axis=1, inplace = True)
 df_test.drop(drop_column, axis=1, inplace = True )<data_type_conversions>
X_train, X_devtest, Y_train, Y_devtest = train_test_split(X, Y, test_size = 0.2)
 print(X_train.shape, Y_train.shape)
 print(X_devtest.shape, Y_devtest.shape )
df_train["renta"] = pd.to_numeric(df_train["renta"], errors="coerce")
 df_test["renta"] = pd.to_numeric(df_test["renta"], errors="coerce" )<feature_engineering>
n_x = 28
train_img = X_train.reshape(( -1,n_x,n_x, 1)).astype('float32')/255
dev_img = X_devtest.reshape(( -1,n_x,n_x,1)).astype('float32')/255
test_img = X_test.reshape(( -1,n_x,n_x,1)).astype('float32')/255
dig_img = X_test_1.reshape(( -1,n_x,n_x,1)).astype('float32')/255
print(train_img.shape, dev_img.shape, test_img.shape, dig_img.shape)
 train_labels = to_categorical(Y_train)
 dev_labels = to_categorical(Y_devtest)
 dig_labels = to_categorical(Y_test_1)
 print(train_labels.shape, dev_labels.shape, dig_labels.shape)
 print(Y_test_1[8],dig_labels[8])
 plt.figure(figsize=(1,1))
plt.imshow(X_test_1[8].reshape(( 28,28)) , cmap=plt.cm.binary)
 plt.show()
for cod in unique_prov:
df_test.loc[df_test['cod_prov']==cod,['renta']] = df_test.loc[df_test['cod_prov']==cod,['renta']].fillna({'renta':grouped[cod]} ).values
df_train.loc[df_train['cod_prov']==cod,['renta']] = df_train.loc[df_train['cod_prov']==cod,['renta']].fillna({'renta':grouped[cod]} ).values<correct_missing_values>
data_augment = ImageDataGenerator(rotation_range =10, zoom_range =0.1, width_shift_range =0.1, height_shift_range = 0.1 )
df_test.renta.fillna(df_test["renta"].median() , inplace=True )<count_missing_values>
model = models.Sequential()
model.add(layers.Conv2D(16, kernel_size=3, padding ='same', activation = 'relu', input_shape=(28,28,1)))
 model.add(layers.Conv2D(32, kernel_size =5, padding= 'same', activation = 'relu'))
model.add(layers.MaxPooling2D(pool_size=(2,2)))
 model.add(layers.Dropout(rate=0.4))
model.add(layers.Conv2D(64, kernel_size = 5, activation= 'relu'))
model.add(layers.MaxPooling2D(pool_size=(2,2)))
 model.add(layers.Dropout(rate=0.4))
model.add(layers.Conv2D(128,kernel_size=3,activation = 'relu'))
model.add(layers.Flatten())
 model.add(layers.Dense(256, activation = 'relu'))
model.add(layers.Dropout(rate=0.4))
model.add(layers.Dense(10,activation = 'softmax'))
model.summary()
df_train.isnull().sum()<count_missing_values>
model.compile(optimizer ='adam', loss='categorical_crossentropy',metrics = ['accuracy'] )
df_test.isnull().sum()<categorify>
epochs = 30
batch_size = 64
history = model.fit_generator(data_augment.flow(train_img, train_labels, batch_size = batch_size),epochs=epochs, steps_per_epoch=train_img.shape[0]//batch_size,validation_data=(dev_img, dev_labels))
replace_mapping = {
"sexo": {"V":0,'H':1},
}
df_train.replace(replace_mapping, inplace= True )<define_variables>
pred_dig = model.predict(dig_img)
 pred_dig_labels = []
for i in range(len(dig_img)) :
pred_dig_label = np.argmax(pred_dig[i])
 pred_dig_labels.append(pred_dig_label )
dummy_col = ['sexo','canal_entrada','cod_prov','segmento']<count_values>
result = pd.DataFrame(Y_test_1)
 result['Y_pred'] = pred_dig_labels
result['correct'] = result['label'] - result['Y_pred']
error = result[result['correct'] != 0]
error_list = error.index
print('Number of error is :', len(error))
print('The incidents are : ', error_list )
limit = int(0.05 * len(df_train.index))
for col in dummy_col:
if len(df_train[col].unique())> 6:
trainlist = df_train[col].value_counts()
print(trainlist)
 use_col = []
for i,item in enumerate(trainlist):
if item > limit:
use_col.append(df_train[col].value_counts().index[i])
 print(use_col)
 for item in df_train[col].unique() :
if item not in use_col:
row_index = df_train[col] == item
print(row_index)
 df_train.loc[row_index,col] = np.nan
print(df_train.loc[row_index,col])
 for item in df_test[col].unique() :
if item not in use_col:
row_index = df_test[col] == item
df_test.loc[row_index,col] = np.nan<categorify>
predictions = model.predict(test_img)
 print(predictions.shape )
df_train = pd.get_dummies(df_train, prefix=dummy_col, columns = dummy_col)
 df_test = pd.get_dummies(df_test, prefix=dummy_col, columns = dummy_col )<categorify>
predicted_labels = []
for i in range(len(predictions)) :
predicted_label = np.argmax(predictions[i])
 predicted_labels.append(predicted_label )
df_train["age"] = pd.to_numeric(df_train["age"], errors="coerce")
 df_test["age"] = pd.to_numeric(df_test["age"], errors="coerce")
 age_group = [[0,19],[19,26],[26,36],[36,41],[41,47],[47,55],[55,60],[60,70],[70,80],[80,170]]
def create_age_group(df):
df['age_group'] = np.nan
for i,age in enumerate(age_group):
row_index =(df['age'] >= age[0])&(df['age'] < age[1])
 df.loc[row_index,'age_group'] = i
create_age_group(df_train)
 create_age_group(df_test)
 df_train.drop('age', axis=1, inplace = True)
 df_test.drop('age', axis=1, inplace = True )<prepare_x_and_y>
submission = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv' )
models = {}
model_preds = {}
id_preds1 = defaultdict(list)
 ids = df_test['ncodpers'].values
for c in product_col:
if c != 'ncodpers':
print(c)
 y_train = df_train[c]
x_train = df_train.drop(product_col + ['ncodpers',"fecha_dato"], 1)
 clf = LogisticRegression()
clf.fit(x_train, y_train)
 x_train2 = df_test.drop(['ncodpers',"fecha_dato"], 1)
 p_train = clf.predict_proba(x_train2)[:,1]
p_train2 = clf.predict_proba(x_train)[:,1]
models[c] = clf
model_preds[c] = p_train
for id, p in zip(ids, p_train):
id_preds1[id].append(p)
 print(roc_auc_score(y_train, p_train2)) 
<train_model>
submission['label'] = predicted_labels
submission.to_csv('submission.csv', index=False)
 print(submission )