Kaggle competition: Digit Recognizer
lgbm = lightgbm.LGBMRegressor(random_state=RANDOM_SEED, n_jobs=-1, metric= 'rmse' )<compute_train_metric>
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 print(train_df.shape, test_df.shape )
model = [lgbm]
for mod in model:
score = cross_val_score(mod, X, y, cv=3, scoring="neg_root_mean_squared_error", n_jobs=-1)
 print("LGBM RMSE Mean Score: ", np.mean(score))<compute_train_metric>
train_df['label'].value_counts(sort=False )
model = [lgbm]
for mod in model:
score = cross_val_score(mod, X, y, cv=10, scoring="neg_root_mean_squared_error", n_jobs=-1)
 print("LGBM RMSE Mean Score: ", np.mean(score))<choose_model_class>
train_X = train_df.drop(['label'], axis=1 ).values
train_Y = train_df['label'].values
test_X = test_df.values
print(train_X.shape, train_Y.shape, test_X.shape )
xgbr = XGBRegressor(random_state=RANDOM_SEED )<compute_train_metric>
n_x = 28
train_X_digit = train_X.reshape(( -1, n_x, n_x, 1))
test_X_digit = test_X.reshape(( -1, n_x, n_x, 1))
print(train_X_digit.shape, test_X_digit.shape)
 train_X_digit = train_X_digit / 255.
 test_X_digit = test_X_digit / 255.
 onehot_labels = to_categorical(train_Y)
 print(onehot_labels.shape)
 print(train_Y[181], onehot_labels[181])
 plt.figure(figsize=(1,1))
plt.imshow(train_X[181].reshape(( 28,28)) ,cmap=plt.cm.binary)
 plt.show()
model = [xgbr]
for mod in model:
score = cross_val_score(mod, X, y, cv=3, scoring="neg_root_mean_squared_error", n_jobs=-1)
 print("XGB RMSE Mean Score: ", np.mean(score))<create_dataframe>
data_augment = ImageDataGenerator(rotation_range=10, zoom_range=0.1,
width_shift_range=0.1, height_shift_range=0.1 )
dtrain = lightgbm.Dataset(data=X, label=y)
 def hyp_lgbm(num_leaves, feature_fraction, bagging_fraction, max_depth, min_split_gain, min_child_weight, learning_rate):
params = {'application':'regression','num_iterations': 5000,
'early_stopping_round':100, 'metric':'rmse'}
params["num_leaves"] = int(round(num_leaves))
params['feature_fraction'] = max(min(feature_fraction, 1), 0)
 params['bagging_fraction'] = max(min(bagging_fraction, 1), 0)
 params['max_depth'] = int(round(max_depth))
params['min_split_gain'] = min_split_gain
params['min_child_weight'] = min_child_weight
params['learning_rate'] = learning_rate
cv_result = lightgbm.cv(params, dtrain, nfold=3,
seed=RANDOM_SEED, stratified=False,
verbose_eval =None, metrics=['rmse'])
 return -np.min(cv_result['rmse-mean'] )<define_search_space>
model = models.Sequential()
model.add(layers.Conv2D(32, kernel_size=5, padding='same', activation='relu', input_shape=(28, 28, 1)))
 model.add(layers.MaxPooling2D(pool_size=(2,2)))
 model.add(layers.Dropout(rate=0.4))
model.add(layers.Conv2D(64, kernel_size=5, activation='relu'))
model.add(layers.MaxPooling2D(pool_size=(2,2)))
 model.add(layers.Dropout(rate=0.4))
model.add(layers.Conv2D(128, kernel_size=3, activation='relu'))
model.add(layers.Flatten())
 model.add(layers.Dense(256, activation='relu'))
model.add(layers.Dropout(rate=0.4))
model.add(layers.Dense(10, activation='softmax'))
model.summary()
pds = {
'num_leaves':(5, 50),
'feature_fraction':(0.2, 1),
'bagging_fraction':(0.2, 1),
'max_depth':(2, 20),
'min_split_gain':(0.001, 0.1),
'min_child_weight':(10, 50),
'learning_rate':(0.01, 0.5),
}<init_hyperparams>
model.compile(optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'] )
def cat_hyp(depth, bagging_temperature, l2_leaf_reg, learning_rate):
params = {"iterations": 100,
"loss_function": "RMSE",
"verbose": False}
params["depth"] = int(round(depth))
params["bagging_temperature"] = bagging_temperature
params["learning_rate"] = learning_rate
params["l2_leaf_reg"] = l2_leaf_reg
cat_feat = []
cv_dataset = cgb.Pool(data=X,
label=y,
cat_features=cat_feat)
 scores = cgb.cv(cv_dataset,
params,
fold_count=3)
 return -np.min(scores['test-RMSE-mean'] )<define_search_space>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy',patience=3,factor=0.5,min_lr=0.00001,
verbose=1 )
pds = {'depth':(4, 10),
'bagging_temperature':(0.1,10),
'l2_leaf_reg':(0.1, 10),
'learning_rate':(0.1, 0.2)
 }<train_on_grid>
X_dev = train_X_digit[:5000]
rem_X_train = train_X_digit[5000:]
print(X_dev.shape, rem_X_train.shape)
 Y_dev = onehot_labels[:5000]
rem_Y_train = onehot_labels[5000:]
print(Y_dev.shape, rem_Y_train.shape )
dtrain = xgb.DMatrix(X, y, feature_names=X.columns.values)
 def hyp_xgb(max_depth, subsample, colsample_bytree,min_child_weight, gamma, learning_rate):
params = {
'objective': 'reg:squarederror',
'eval_metric':'rmse',
'nthread':-1
}
params['max_depth'] = int(round(max_depth))
params['subsample'] = max(min(subsample, 1), 0)
 params['colsample_bytree'] = max(min(colsample_bytree, 1), 0)
 params['min_child_weight'] = int(min_child_weight)
 params['gamma'] = max(gamma, 0)
 params['learning_rate'] = learning_rate
scores = xgb.cv(params, dtrain, num_boost_round=500,verbose_eval=False,
early_stopping_rounds=10, nfold=3)
 return -scores['test-rmse-mean'].iloc[-1]<define_search_space>
epochs = 30
batch_size = 128
history = model.fit_generator(data_augment.flow(rem_X_train, rem_Y_train, batch_size=batch_size),
epochs=epochs, steps_per_epoch=rem_X_train.shape[0]//batch_size,
validation_data=(X_dev, Y_dev), callbacks=[learning_rate_reduction] )
pds ={
'min_child_weight':(3, 20),
'gamma':(0, 5),
'subsample':(0.7, 1),
'colsample_bytree':(0.1, 1),
'max_depth':(3, 10),
'learning_rate':(0.01, 0.5)
 }<import_modules>
pred_dev = model.predict(X_dev)
 pred_dev_labels = np.argmax(pred_dev, axis=1 )
from sklearn.ensemble import StackingRegressor
from sklearn.linear_model import LinearRegression<init_hyperparams>
result = pd.DataFrame(train_Y[:5000], columns=['Y_dev'])
 result['Y_pred'] = pred_dev_labels
result['correct'] = result['Y_dev'] - result['Y_pred']
errors = result[result['correct'] != 0]
error_list = errors.index
print('Number of errors is ', len(errors))
print('The indices are ', error_list )
param_lgbm = {
'bagging_fraction': 0.973905385549851,
'feature_fraction': 0.2945585590881137,
'learning_rate': 0.03750332268701348,
'max_depth': int(7.66),
'min_child_weight': int(41.36),
'min_split_gain': 0.04033836353603582,
'num_leaves': int(46.42),
'application':'regression',
'num_iterations': 5000,
'metric': 'rmse'
}
param_cat = {
'bagging_temperature': 0.31768713094131684,
'depth': int(8.03),
'l2_leaf_reg': 1.3525686450404295,
'learning_rate': 0.2,
'iterations': 100,
'loss_function': 'RMSE',
'verbose': False
}
param_xgb = {
'colsample_bytree': 0.8119098377889549,
'gamma': 2.244423418642122,
'learning_rate': 0.015800631696721114,
'max_depth': int(9.846),
'min_child_weight': int(15.664),
'subsample': 0.82345,
'objective': 'reg:squarederror',
'eval_metric':'rmse',
'num_boost_roun' : 500
}<import_modules>
predictions = model.predict(test_X_digit)
 print(predictions.shape )
from sklearn.ensemble import RandomForestRegressor, AdaBoostRegressor, GradientBoostingRegressor
from sklearn.neural_network import MLPRegressor
from sklearn import svm
import lightgbm<define_search_model>
predicted_labels = np.argmax(predictions, axis=1 )
<train_model><EOS> 
result = pd.read_csv('.. /input/sample_submission.csv')
 result['Label'] = predicted_labels
result.to_csv('submission.csv', index=False )