Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<compute_test_metric>
print(os.listdir(".. /input"))
sns.set(style='white', context='notebook', palette='deep')
def rmlse(y_true, y_pred):
return tf.sqrt(tf.reduce_mean(tf.square(tf.math.log(y_pred + 1)- tf.math.log(y_true + 1))))<save_to_csv>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
def submit(model, X, ids, file_path):
SalePrice = model.predict(X)
 submission = pd.DataFrame({"Id": ids, "SalePrice": SalePrice.reshape(-1)})
 submission.to_csv(file_path, index=False )<sort_values>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
train.corr() ["SalePrice"].sort_values(key = lambda x: abs(x), ascending=False )<feature_engineering>
X_train.head()
for data in [train, test]:
null_counts = data.isnull().sum()
null_counts[null_counts > 0]
null_columns = list(pd.DataFrame(null_counts[null_counts > 0] ).index)
 for column in null_columns:
if data[column].dtype == object:
data[column] = data[[column]].replace(np.NAN, "Unknown")
 else:
data[column] = data[column].replace(np.NAN, data[column].mean() )<categorify>
X_train.isnull().any().sum()
train_test = pd.get_dummies(pd.concat([train, test]))<drop_column>
test.isnull().any().sum()
train_features = train_test.iloc[0: len(train)]
test_features = train_test.iloc[len(train):]
_ = train_features.pop("Id")
 _ = test_features.pop("SalePrice")
 test_ids = test_features.pop("Id" )<split>
X_train = X_train / 255.0
test = test / 255.0
train_features, val_features = train_test_split(train_features, test_size=0.2, random_state=np.random.randint(1000))<sort_values>
X_train = X_train.values.reshape(-1,28,28,1)
 test = test.values.reshape(-1,28,28,1 )
thresold = 0.05
correlated_scores = train_features.corr() ["SalePrice"]
correlated_scores = correlated_scores[correlated_scores.abs() >= thresold]
correlated_columns = list(correlated_scores.index)
 correlated_columns.remove("SalePrice")
 print(correlated_columns )<drop_column>
y_train = to_categorical(y_train, num_classes=10 )
train_targets = train_features.pop("SalePrice")
 val_targets = val_features.pop("SalePrice" )<define_variables>
random_seed = 2
categorical_columns = set(train.dtypes[train.dtypes==object].index )<feature_engineering>
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1, random_state=random_seed )
scale_strategies = ["none", "standard_scale", "standard_scale_exclude_categorcial_features"]
scale_strategy = scale_strategies[2]
if scale_strategy == scale_strategies[1]:
train_features =(train_features - mean_value)/ std_value
val_features =(val_features - mean_value)/ std_value
test_features =(test_features - mean_value)/ std_value
if scale_strategy == scale_strategies[2]:
for column in train_features.columns:
is_categorical_feature = False
components = column.split("_")
 if len(components)== 2 and components[0] in categorical_columns:
is_categorical_feature = True
if is_categorical_feature == False:
for features in [train_features, val_features, test_features]:
features.loc[:, column] =(features.loc[:, column] - mean_value[column])/ std_value[column]<drop_column>
model = Sequential()
model.add(Conv2D(filters=40, kernel_size=(5,5), padding='Same', activation='relu', input_shape=(28, 28, 1)))
 model.add(Conv2D(filters=40, kernel_size=(5,5), padding='Same', activation='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Conv2D(filters=60, kernel_size=(3,3), padding='Same', activation='relu'))
model.add(Conv2D(filters=60, kernel_size=(3,3), padding='Same', activation='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Flatten())
 model.add(Dense(300, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))
use_correlated_columns = True
if use_correlated_columns:
train_features = train_features[correlated_columns]
val_features = val_features[correlated_columns]
test_features = test_features[correlated_columns]<train_model>
optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False )
begin = time.time()
parameters = {
"depth": [4, 5, 6, 7, 8, 9],
"learning_rate": [0.01, 0.05, 0.07, 0.08, 0.09, 0.10, 0.11, 0.12, 0.13, 0.14, 0.15],
"iterations": [500, 10000],
}
def train_with_catboost(hyperparameters, X_train, X_val, y_train, y_val):
keys = hyperparameters.keys()
best_index = {key:0 for key in keys}
best_cat = None
best_score = 10e8
for(index, key)in enumerate(keys):
print("Find best parameter for %s" %(key))
items = hyperparameters[key]
best_parameter = None
temp_best = 10e8
for(key_index, item)in enumerate(items):
iterations = hyperparameters["iterations"][best_index["iterations"]] if key != "iterations" else item
learning_rate = hyperparameters["learning_rate"][best_index["learning_rate"]] if key != "learning_rate" else item
depth = hyperparameters["depth"][best_index["depth"]] if key != "depth" else item
print("Training with iterations: %d learning_rate: %.2f depth:%d"%(iterations, learning_rate, depth))
cat = catboost.CatBoostRegressor(
 iterations = iterations,
learning_rate = learning_rate,
depth = depth,
verbose=500 
)
 cat.fit(X_train, y_train, verbose=False)
 result = evaluate(cat, X_val, y_val)
 score = result["rmlse"]
if score < temp_best:
temp_best = score
best_index[key] = key_index
best_parameter = item
if score < best_score:
best_score = score
best_cat = cat
print("Best Parameter for %s: "%(key), best_parameter)
 best_parameters = {
"iterations": hyperparameters["iterations"][best_index["iterations"]],
"learning_rate": hyperparameters["learning_rate"][best_index["learning_rate"]],
"depth": hyperparameters["depth"][best_index["depth"]]
}
return best_cat, best_score, best_parameters
best_cat, best_score, best_parameters = train_with_catboost(parameters, train_features, val_features, train_targets, val_targets)
 print("Best RMLSE: ", best_score)
 print("Best Parameters: ", best_parameters)
 elapsed = time.time() - begin
print("Elapsed time: ", elapsed)
 submit(best_cat, test_features, test_ids, "submission_cat.csv" )<train_model>
model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'] )
X = pd.concat([train_features, val_features])
 y = pd.concat([train_targets, val_targets])
 fold = 1
models = []
for train_indices, valid_indices in KFold(n_splits=5, shuffle=True ).split(X):
print("Training with Fold %d" %(fold))
X_train = X.iloc[train_indices]
X_val = X.iloc[valid_indices]
y_train = y.iloc[train_indices]
y_val = y.iloc[valid_indices]
cat = catboost.CatBoostRegressor(
 iterations = best_parameters["iterations"],
learning_rate = best_parameters["learning_rate"],
depth = best_parameters["depth"] 
)
 cat.fit(X_train, y_train, verbose=False)
 models.append(cat)
 evaluate(cat, X_val, y_val)
 submit(cat, test_features, test_ids, "submission_cat_fold%d.csv"%(fold))
fold += 1<save_to_csv>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', patience=5, factor=0.5, verbose=1, min_lr=0.00001 )
SalePrice = np.mean([model.predict(test_features)for model in models], axis=0)
 submission = pd.DataFrame({"Id": test_ids, "SalePrice": SalePrice})
 submission.to_csv("submission.csv", index=False )<train_on_grid>
epochs = 1
batch_size = 84
cat = catboost.CatBoostRegressor(
 iterations = best_parameters["iterations"],
learning_rate = best_parameters["learning_rate"],
depth = best_parameters["depth"] 
)
 cat.fit(X, y, verbose=False)
 evaluate(cat, X, y)
 submit(cat, test_features, test_ids, "submission_cat_all_dataset.csv" )<set_options>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
warnings.filterwarnings('ignore' )<load_from_csv>
history = model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size),
epochs=epochs, verbose=2, callbacks=[learning_rate_reduction],
validation_data=(X_val, y_val), steps_per_epoch=X_train.shape[0] // batch_size )
train = pd.read_csv('.. /input/tabular-playground-series-feb-2021/train.csv')
 test = pd.read_csv('.. /input/tabular-playground-series-feb-2021/test.csv' )<prepare_x_and_y>
result = model.predict(test)
 result = np.argmax(result, axis=1)
 result = pd.Series(result, name='Label')
X_train = train.drop(['id', 'target'], axis=1)
 y_train = train.target
X_test = test.drop(['id'], axis=1 )<categorify>
my_sumission = pd.concat([pd.Series(range(1, 28001), name='ImageId'), result], axis=1)
 my_sumission.to_csv("submission.csv", index=False )