Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<define_variables>
%matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
def detect_outliers(df,n,features):

 outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers<define_variables>
ds_train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 ds_test = pd.read_csv('.. /input/digit-recognizer/test.csv')
Outliers_to_drop = detect_outliers(train_df,3,train_df.columns.values[2:])
 print("异常值：",round(len(Outliers_to_drop)/train_df.shape[0]*100,2),"%")
 train_df.loc[Outliers_to_drop]<count_duplicates>
val_size = 0.1
X_all = ds_train.loc[:, ds_train.columns != 'label'].to_numpy()
y_all = ds_train['label'].to_numpy()
test = ds_test.to_numpy()
train_df.duplicated().value_counts()<filter>
X_all = X_all / 255.0
X_all = X_all.reshape(-1,28,28,1)
 test = test / 255.0
test = test.reshape(-1,28,28,1)
 y_all = to_categorical(y_all, num_classes = 10)
 X_train, X_val, y_train, y_val = model_selection.train_test_split(X_all, y_all, test_size=val_size, random_state=1 )
len(train_df.loc[train_df['age'] == 0] )<define_variables>
optimizer = RMSprop(lr=0.001, rho=0.9, decay=0.0 )
late_pay_cols = ['NumberOfTime30-59DaysPastDueNotWorse','NumberOfTimes90DaysLate','NumberOfTime60-89DaysPastDueNotWorse']<count_values>
def get_simple_model() :
model = Sequential()
model.add(Flatten(input_shape=(28,28,1)))
 model.add(Dense(64, activation = 'sigmoid'))
model.add(Dropout(0.2))
model.add(Dense(24, activation = 'sigmoid'))
model.add(Dropout(0.2))
model.add(Dense(10, activation = 'softmax'))
model.compile(optimizer = optimizer, loss = "categorical_crossentropy", metrics=["accuracy"])
 return model
train_df["NumberOfTime30-59DaysPastDueNotWorse"].value_counts().sort_index()<count_values>
def get_cnn_model() :
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same', activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))
model.compile(optimizer = optimizer, loss = "categorical_crossentropy", metrics=["accuracy"])
 return model
train_df["NumberOfTimes90DaysLate"].value_counts().sort_index()<count_values>
def test_submission(model):
results = model.predict(test)
 results = np.argmax(results, axis=1)
 results = pd.Series(results, name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )
train_df["NumberOfTime60-89DaysPastDueNotWorse"].value_counts().sort_index()<feature_engineering>
cnn_model_2, _ = train_model(get_cnn_model() , epochs = 30)
 test_submission(cnn_model_2 )