Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<feature_engineering>
%matplotlib inline
test['EmpAge_RATIO'] = test['DAYS_EMPLOYED'] / test['AGE']
test['CredInc_RATIO'] = test['AMT_CREDIT'] / test['AMT_INCOME_TOTAL']
test['AnnInc_RATIO'] = test['AMT_ANNUITY'] / test['AMT_INCOME_TOTAL']
test['AnnCred_RATIO'] = test['AMT_ANNUITY'] /(test['AMT_CREDIT'] + 1)
 test['CredGoods_RATIO'] = test['AMT_CREDIT'] /(test['AMT_GOODS_PRICE'] + 1)
 test['AVG_EXT_INCOME'] = test['AMT_INCOME_TOTAL'] * test['AVG_EXT']
test['AVG_EXT_GOODS'] = test['AMT_GOODS_PRICE'] * test['AVG_EXT']<define_variables>
mnist_train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 mnist_test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )
dels = ['APARTMENTS_MODE', 'BASEMENTAREA_MODE', 'YEARS_BEGINEXPLUATATION_MODE',
'YEARS_BUILD_MODE', 'COMMONAREA_MODE', 'ELEVATORS_MODE', 'ENTRANCES_MODE',
'FLOORSMAX_MODE', 'FLOORSMIN_MODE', 'LANDAREA_MODE', 'LIVINGAPARTMENTS_MODE',
'LIVINGAREA_MODE', 'NONLIVINGAPARTMENTS_MODE', 'NONLIVINGAREA_MODE',
'APARTMENTS_MEDI', 'BASEMENTAREA_MEDI', 'YEARS_BEGINEXPLUATATION_MEDI',
'YEARS_BUILD_MEDI', 'COMMONAREA_MEDI', 'ELEVATORS_MEDI', 'ENTRANCES_MEDI',
'FLOORSMAX_MEDI', 'FLOORSMIN_MEDI', 'LANDAREA_MEDI', 'LIVINGAPARTMENTS_MEDI',
'LIVINGAREA_MEDI', 'NONLIVINGAPARTMENTS_MEDI', 'NONLIVINGAREA_MEDI',
'FONDKAPREMONT_MODE', 'HOUSETYPE_MODE', 'TOTALAREA_MODE',
'WALLSMATERIAL_MODE', 'EMERGENCYSTATE_MODE', 'FLAG_DOCUMENT_2',
'FLAG_DOCUMENT_3', 'FLAG_DOCUMENT_4', 'FLAG_DOCUMENT_5',
'FLAG_DOCUMENT_6', 'FLAG_DOCUMENT_7', 'FLAG_DOCUMENT_8',
'FLAG_DOCUMENT_9', 'FLAG_DOCUMENT_10', 'FLAG_DOCUMENT_11',
'FLAG_DOCUMENT_12', 'FLAG_DOCUMENT_13', 'FLAG_DOCUMENT_14',
'FLAG_DOCUMENT_15', 'FLAG_DOCUMENT_16', 'FLAG_DOCUMENT_17',
'FLAG_DOCUMENT_18', 'FLAG_DOCUMENT_19', 'FLAG_DOCUMENT_20',
'FLAG_DOCUMENT_21', 'DAYS_BIRTH', 'LIVINGAPARTMENTS_AVG',
'LIVINGAREA_AVG', 'CNT_FAM_MEMBERS', 'OBS_30_CNT_SOCIAL_CIRCLE',
'OBS_60_CNT_SOCIAL_CIRCLE', 'ELEVATORS_AVG', 'AVG_EXT']
test = test.drop(test[dels], axis =1)
 gc.collect()<categorify>
mnist_train.isna().any().any()
test = test.replace([np.inf, -np.inf], np.nan )<predict_on_test>
mnist_train_data = mnist_train.loc[:, "pixel0":]
mnist_train_label = mnist_train.loc[:, "label"]
mnist_train_data = mnist_train_data/255.0
mnist_test = mnist_test/255.0
test_pred = LGBM_model.predict_proba(X_test)
 print(test_pred.shape)
 print(test_pred[:5] )<load_from_csv>
standardized_scalar = StandardScaler()
standardized_data = standardized_scalar.fit_transform(mnist_train_data)
 standardized_data.shape
submission = pd.read_csv('.. /input/home-credit-default-risk/sample_submission.csv')
 submission.head(10 )<prepare_x_and_y>
cov_matrix = np.matmul(standardized_data.T, standardized_data)
 cov_matrix.shape
submission.TARGET = test_pred[:,1]
submission.head(10 )<save_to_csv>
lambdas, vectors = eigh(cov_matrix, eigvals=(782, 783))
vectors.shape
submission.to_csv('default_submission_wk06.csv', index=False, header = True )<load_from_csv>
new_coordinates = np.matmul(vectors, standardized_data.T)
 print(new_coordinates.shape)
 new_coordinates = np.vstack(( new_coordinates, mnist_train_label)).T
MainDir = ".. /input/.. /input/home-credit-default-risk"
test = pd.read_csv(f'{MainDir}/application_test.csv' )<load_pretrained>
df_new = pd.DataFrame(new_coordinates, columns=["f1", "f2", "labels"])
 df_new.head()
preprocessor = joblib.load('.. /input/defaultdata08/default_preprocessor_08.joblib')
 model = joblib.load('.. /input/defaultdata08/default_model_08.joblib')
 print(type(model))<load_from_csv>
pca = decomposition.PCA()
pca.n_components = 2
pca_data = pca.fit_transform(standardized_data)
 pca_data.shape
bureau = pd.read_csv(f'{MainDir}/bureau.csv')
 print(bureau.shape, "- shape of bureau table")
 bureau_balance = pd.read_csv(f'{MainDir}/bureau_balance.csv')
 bb_status = pd.crosstab(bureau_balance.SK_ID_BUREAU, bureau_balance.STATUS)
 bb_status.columns = ['BB_'+column for column in bb_status.columns]
bureau = bureau.merge(bb_status, left_on = 'SK_ID_BUREAU', right_on = 'SK_ID_BUREAU')
 bureau = bureau.drop(['SK_ID_BUREAU'], axis = 1)
 print(bureau.shape, "- shape of bureau table after merge")
 bureau.columns = ['BU_'+column if column !='SK_ID_CURR' else column for column in bureau.columns]
bureau_num = bureau.groupby(by=['SK_ID_CURR'] ).mean().reset_index()
print(bureau_num.shape, "- shape of numeric features(incl index)")
 bureau_cat = pd.get_dummies(bureau.select_dtypes('object'))
bureau_cat['SK_ID_CURR'] = bureau['SK_ID_CURR']
bureau_cat = bureau_cat.groupby(by = ['SK_ID_CURR'] ).mean().reset_index()
print(bureau_cat.shape, "- shape of categorical features(incl index)")
 bureau_count = bureau.groupby(by = ['SK_ID_CURR'])['BU_CREDIT_ACTIVE'].count().reset_index()
bureau_count.rename(columns={'BU_CREDIT_ACTIVE':'COUNT_of_BUREAU'})
 bureau_count.head(5)
 test = test.merge(bureau_num, on='SK_ID_CURR', how='left')
 test = test.merge(bureau_cat, on='SK_ID_CURR', how='left')
 test = test.merge(bureau_count, on='SK_ID_CURR', how='left')
 print(test.shape, "- shape of training data after merges")
 list = ['bureau', 'bureau_num', 'bureau_cat', 'bureau_balance']
del list
gc.collect()
previous = pd.read_csv(f'{MainDir}/previous_application.csv')
 print(previous.shape, "- shape of previous_application")
 pos = pd.read_csv(f'{MainDir}/POS_CASH_balance.csv')
 pos.columns = ['PO_'+column if column !='SK_ID_PREV' else column for column in pos.columns]
pos_num = pos.groupby(by=['SK_ID_PREV'] ).mean().reset_index()
print(pos_num.shape, "- shape of numeric features(incl index)")
 pos_cat = pd.get_dummies(pos.select_dtypes('object'))
pos_cat['SK_ID_PREV'] = pos['SK_ID_PREV']
pos_cat = pos_cat.groupby(by = ['SK_ID_PREV'] ).mean().reset_index()
print(pos_cat.shape, "- shape of categorical features(incl index)")
 previous = previous.merge(pos_num, on='SK_ID_PREV', how='left')
 previous = previous.merge(pos_cat, on='SK_ID_PREV', how='left')
 print(previous.shape, "- shape of previous data after merges")
 list = ['pos', 'pos_num', 'pos_cat']
del list
gc.collect()
inst = pd.read_csv(f'{MainDir}/installments_payments.csv')
 inst.columns = ['IP_'+column if column !='SK_ID_PREV' else column for column in inst.columns]
inst_num = inst.groupby(by=['SK_ID_PREV'] ).mean().reset_index()
print(inst_num.shape, "- shape of numeric features(incl index)")
 previous = previous.merge(inst_num, left_on='SK_ID_PREV', right_on = 'SK_ID_PREV', how='left')
 print(previous.shape, "- shape of previous data after merges")
 list = ['inst', 'inst_num']
del list
gc.collect()
ccb = pd.read_csv(f'{MainDir}/credit_card_balance.csv')
 ccb.columns = ['CC_'+column if column !='SK_ID_PREV' else column for column in ccb.columns]
ccb_num = ccb.groupby(by=['SK_ID_PREV'] ).mean().reset_index()
print(ccb_num.shape, "- shape of numeric features(incl index)")
 ccb_cat = pd.get_dummies(ccb.select_dtypes('object'))
ccb_cat['SK_ID_PREV'] = ccb['SK_ID_PREV']
ccb_cat = ccb_cat.groupby(by = ['SK_ID_PREV'] ).mean().reset_index()
print(ccb_cat.shape, "- shape of categorical features(incl index)")
 previous = previous.merge(ccb_num, on='SK_ID_PREV', how='left')
 previous = previous.merge(ccb_cat, on='SK_ID_PREV', how='left')
 print(previous.shape, "- shape of previous data after merges")
 list = ['ccb', 'ccb_num', 'ccb_cat']
del list
gc.collect()
previous.columns = ['PR_'+column if column !='SK_ID_CURR' else column for column in previous.columns]
previous['PR_DAYS_LAST_DUE'].replace({365243: np.nan}, inplace = True)
 previous['PR_DAYS_TERMINATION'].replace({365243: np.nan}, inplace = True)
 previous['PR_DAYS_FIRST_DRAWING'].replace({365243: np.nan}, inplace = True)
 previous_num = previous.groupby(by=['SK_ID_CURR'] ).mean().reset_index()
print(previous_num.shape, "- shape of numeric features(incl index)")
 previous_cat = pd.get_dummies(previous.select_dtypes('object'))
previous_cat['SK_ID_CURR'] = previous['SK_ID_CURR']
previous_cat = bureau_cat.groupby(by = ['SK_ID_CURR'] ).mean().reset_index()
print(previous_cat.shape, "- shape of categorical features(incl index)")
 test = test.merge(previous_num, on='SK_ID_CURR', how='left')
 test = test.merge(previous_cat, on='SK_ID_CURR', how='left')
 print(test.shape, "- shape of training data after merges")
 list = ['previous', 'previous_num', 'previous_cat']
del list
gc.collect()
test['DAYS_EMPLOYED'].replace({365243: np.nan}, inplace = True)
 test['CI_ratio'] = test['AMT_CREDIT'] / test['AMT_INCOME_TOTAL']
test['AI_ratio'] = test['AMT_ANNUITY'] / test['AMT_INCOME_TOTAL']
test['AC_ratio'] = test['AMT_CREDIT'] / test['AMT_ANNUITY']
test['CG_ratio'] = test['AMT_CREDIT'] / test['AMT_GOODS_PRICE']
test['log_INCOME'] = np.log(test['AMT_INCOME_TOTAL'])
 test['log_ANNUITY'] = np.log(test['AMT_ANNUITY'])
 test['log_CREDIT'] = np.log(test['AMT_CREDIT'])
 test['log_GOODS'] = np.log(test['AMT_GOODS_PRICE'])
 test['MissingBureau'] = test.iloc[:, 41:44].isnull().sum(axis=1 ).astype("category")
 test['FLAG_CG_ratio'] = test['AMT_CREDIT'] > test['AMT_GOODS_PRICE']
test['DAYS_ID_4200'] = test['DAYS_ID_PUBLISH'] < -4200
test['AVG_EXT'] = test.iloc[:, 41:44].sum(axis=1)/(3- test.iloc[:,41:44].isnull().sum(axis=1))
test['AVG_EXT'].replace(np.nan, 0.2, inplace = True)
 test.EXT_SOURCE_1.fillna(test.AVG_EXT, inplace=True)
 test.EXT_SOURCE_2.fillna(test.AVG_EXT, inplace=True)
 test.EXT_SOURCE_3.fillna(test.AVG_EXT, inplace=True)
 test.drop(['AVG_EXT'], axis = 1)
 test.drop(['ORGANIZATION_TYPE'], axis = 1)
 test['OD_ratio'] = test['BU_AMT_CREDIT_SUM_OVERDUE'] / test['BU_AMT_CREDIT_SUM_DEBT']
test['OD_ratio'].replace([np.nan, np.inf, -np.inf], 0, inplace = True)
 test['Credit_ratio'] = test['BU_AMT_CREDIT_SUM'] / test['BU_AMT_CREDIT_SUM_LIMIT']
test['Credit_ratio'].replace([np.nan, np.inf, -np.inf], 0, inplace = True)
 test['Debt_ratio'] = test['BU_AMT_CREDIT_SUM_DEBT'] / test['BU_AMT_CREDIT_SUM']
test['Debt_ratio'].replace([np.nan, np.inf, -np.inf], 0, inplace = True)
 test['PR_term'] = test['PR_IP_AMT_PAYMENT'] / test['PR_IP_AMT_INSTALMENT']
test['PR_term'].replace([np.nan, np.inf, -np.inf], 0, inplace = True)
 X_test = preprocessor.transform(test)
 print(X_test.shape )<predict_on_test>
pca_data = np.vstack(( pca_data.T, mnist_train_label)).T
test_pred = model.predict_proba(X_test)
 print(test_pred.shape)
 print(test_pred[:5] )<save_to_csv>
df_PCA = pd.DataFrame(new_coordinates, columns=["f1", "f2", "labels"])
 df_PCA.head()
submission = pd.read_csv('.. /input/home-credit-default-risk/sample_submission.csv')
 submission.head(10)
 submission.TARGET = test_pred[:,1]
submission.head(10)
 submission.to_csv('default_submission_08.csv', index=False, header = True )<import_modules>
mnist_train_data = np.array(mnist_train_data)
 mnist_train_label = np.array(mnist_train_label )
import os
import gc
import numpy as np
import pandas as pd
from scipy.stats import kurtosis
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import MinMaxScaler
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
import seaborn as sns
import warnings
from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold
import xgboost as xgb
from xgboost import XGBClassifier<define_variables>
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Lambda, Flatten, BatchNormalization
from tensorflow.keras.layers import Conv2D, MaxPool2D, AvgPool2D
from tensorflow.keras.optimizers import Adadelta
from keras.utils.np_utils import to_categorical
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import ReduceLROnPlateau
from tensorflow.keras.callbacks import LearningRateScheduler
DATA_DIRECTORY = ".. /input/home-credit-loan-better-data-processing"<load_from_csv>
nclasses = mnist_train_label.max() - mnist_train_label.min() + 1
mnist_train_label = to_categorical(mnist_train_label, num_classes = nclasses)
 print("Shape of ytrain after encoding: ", mnist_train_label.shape )
train = pd.read_csv(os.path.join(DATA_DIRECTORY, 'train.csv'))
test = pd.read_csv(os.path.join(DATA_DIRECTORY, 'test.csv'))
labels = pd.read_csv(os.path.join(DATA_DIRECTORY, 'labels.csv')) 
<categorify>
def build_model(input_shape=(28, 28, 1)) :
model = Sequential()
model.add(Conv2D(32, kernel_size = 3, activation='relu', input_shape = input_shape))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(128, kernel_size = 4, activation='relu'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
return model
def compile_model(model, optimizer='adam', loss='categorical_crossentropy'):
model.compile(optimizer=optimizer, loss=loss, metrics=["accuracy"])
 def train_model(model, train, test, epochs, split):
history = model.fit(train, test, shuffle=True, epochs=epochs, validation_split=split)
 return history
train = train.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
test = test.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))
labels = labels.rename(columns = lambda x:re.sub('[^A-Za-z0-9_]+', '', x))<categorify>
cnn_model = build_model(( 28, 28, 1))
compile_model(cnn_model, 'adam', 'categorical_crossentropy')
 model_history = train_model(cnn_model, mnist_train_data, mnist_train_label, 80, 0.2 )
train=np.nan_to_num(train)
 test=np.nan_to_num(test)
 labels=np.nan_to_num(labels )<create_dataframe>
predictions = cnn_model.predict(mnist_test_arr )
train = pd.DataFrame(train)
 test = pd.DataFrame(test)
 labels=pd.DataFrame(labels )<split>
predictions_test = []
for i in predictions:
predictions_test.append(np.argmax(i))
<predict_on_test><EOS> 
submission = pd.DataFrame({
"ImageId": mnist_test.index+1,
"Label": predictions_test
})
 submission.to_csv('my_submission.csv', index=False )