Kaggle competition: Digit Recognizer
k = 10
test[target] = 0
scores = []
kf = KFold(k)
 for i,(train_idx, test_idx)in enumerate(kf.split(data)) :
X_train = data.iloc[train_idx][preds]
y_train = data.iloc[train_idx][target]
X_test = data.iloc[test_idx][preds]
y_test = data.iloc[test_idx][target]
best_params['learning_rate'] = 0.005
best_params['n_estimators'] = 100000
estimator = LGBMRegressor(**best_params)
 estimator.fit(X_train,
y_train,
eval_set=(X_test, y_test),
eval_metric='rmse',
early_stopping_rounds=1000,
verbose=1000)
 y_pred = estimator.predict(X_test)
 rmse = mean_squared_error(y_test, y_pred, squared=False)
 scores.append(rmse)
 test[target] += estimator.predict(test[preds])/ k
test[target].to_csv('submission.csv' )<compute_test_metric>
def l1_reg(weight_matrix):
return 0.01 * K.sum(K.abs(weight_matrix))
tf.keras.backend.clear_session()
print(f"Expected score: {np.mean(scores)}" )<load_from_csv>
NB_EPOCH = 40
BATCH_SIZE = 32
VERBOSE = 1
NB_CLASSES = 10
OPTIMIZER = Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0)
 N_HIDDEN = 128
VALIDATION_SPLIT = 0.05
DROPOUT = 0.3
RESHAPED =(28, 28, 1 )
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv")
 submission_df = pd.read_csv('/kaggle/input/titanic/gender_submission.csv' )<concatenate>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
df = pd.concat([train, test], axis=0)
 df = df.set_index('PassengerId')
 df.info()<drop_column>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
df = df.drop(['Name', 'Ticket', 'Cabin'], axis=1)
<drop_column>
x_train = train.drop(labels=["label"], axis=1)
 x_train = x_train / 255.0
x_train = x_train.values.reshape(-1, 28, 28, 1)
 y_train = train["label"]
y_train = np.array(to_categorical(y_train, num_classes=10))
x_test = test / 255.0
x_test = x_test.values.reshape(-1, 28, 28, 1)
 print(x_train.shape, y_train.shape, x_test.shape )
df = df.drop('Fare', axis=1 )<categorify>
( x_train_mnist, y_train_mnist),(x_val_mnist, y_val_mnist)= mnist.load_data()
x_train_mnist = np.concatenate(( x_train_mnist, x_val_mnist))
y_train_mnist = np.concatenate(( y_train_mnist, y_val_mnist))
x_train_mnist = x_train_mnist.reshape(( x_train_mnist.shape[0], 28, 28, 1))
X_train_mnist = x_train_mnist / 255
y_train_mnist = np.array(to_categorical(y_train_mnist.reshape(( y_train_mnist.shape[0], 1)) ,num_classes=10))
print(x_train_mnist.shape, y_train_mnist.shape )
def fillna_using_knn_imputer(df):
imputer = KNNImputer(n_neighbors=5,
weights='uniform',
metric='nan_euclidean')
 df['Sex'] = df['Sex'].factorize() [0]
df['Embarked'] = df['Embarked'].factorize() [0]
df['Kind'] = df['Kind'].factorize() [0]
X_train = df[~df['Survived'].isnull() ].drop('Survived', axis=1)
 X_test = df[df['Survived'].isnull() ].drop('Survived', axis=1)
 X_train_trans = pd.DataFrame(imputer.fit_transform(X_train), columns=X_train.columns, index=X_train.index)
 X_test_trans = pd.DataFrame(imputer.transform(X_test), columns=X_test.columns, index=X_test.index)
 dff = pd.concat([X_train_trans, X_test_trans], axis=0)
 dff['Survived'] = df['Survived']
dff = dff.sort_index()
return dff
df = fillna_using_knn_imputer(df)
 df.info()<normalization>
x_train = np.concatenate(( x_train, x_train_mnist))
y_train = np.concatenate(( y_train, y_train_mnist))
def encode_cols(df, cols=['Pclass', 'Embarked']):
for col in cols:
dumm = pd.get_dummies(data=df[col], prefix=col)
 df = pd.concat([df, dumm], axis=1)
 df = df.drop(col, axis=1)
 return df
def scale_all_features(df):
X = df.drop('Survived', axis=1)
 scaler = MinMaxScaler()
X = pd.DataFrame(scaler.fit_transform(X), columns=X.columns, index=X.index)
 df = pd.concat([X, df['Survived']], axis=1)
 return df, scaler
df = encode_cols(df)
 df, _ = scale_all_features(df)
 df.info()<split>
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.10,
width_shift_range=0.1,
height_shift_range=0.1 )
def split_to_train_test_X_y(df):
X = df[~df['Survived'].isna() ].drop('Survived', axis=1)
 y = df[~df['Survived'].isna() ]['Survived']
X_test = df[df['Survived'].isna() ].drop('Survived', axis=1)
 return X, y, X_test
X, y, X_test = split_to_train_test_X_y(df )<init_hyperparams>
np.random.seed(0 )
class ML_Classifier_Switcher(object):
def pick_model(self, model_name):

 self.param_grid = None
method_name = str(model_name)
 method = getattr(self, method_name, lambda: "Invalid ML Model")
 return method()
def SVM(self):
self.param_grid = {'kernel': ['rbf', 'sigmoid', 'linear'],
'C': np.logspace(-2, 2, 10),
'gamma': np.logspace(-5, 1, 14)}
return SVC()
def XGR(self):
self.param_grid = {'gamma': np.logspace(-5, 1, 7),
'subsample': [0.5, 0.75, 1.0],
'colsample_bytree': [0.5, 0.75, 1.0],
'eta': [0.1, 0.5, 0.9],
'max_depth': [3, 5]}
return XGBClassifier(random_state=42, nthread=7, use_label_encoder=False,
eval_metric='error', tree_method = "hist")
 def RF(self):
self.param_grid = {
'n_estimators': [50, 100, 200, 300],
'max_features': ['auto'],
'criterion': ['entropy'],
'max_depth': [5, 10],
'min_samples_split': [5],
'min_samples_leaf': [1]
}
return RandomForestClassifier(random_state=42, n_jobs=-1)
 def LR(self):
self.param_grid = {'solver': ['newton-cg', 'lbfgs', 'liblinear'],
'penalty': ['l2'],
'C': [100, 10, 1.0, 0.1, 0.01]}
return LogisticRegression(n_jobs=None, random_state=42)
 def KNN(self):
self.param_grid = {
'n_neighbors': list(range(1, 5)) ,
'weights': ['uniform', 'distance'],
'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute'],
'leaf_size': list(range(1, 10)) ,
'p': [1, 2]
}
return KNeighborsClassifier()<compute_train_metric>
input_shape =(28,28,1)
 model = Sequential()
model.add(Conv2D(filters = 96, kernel_size =(3, 3), padding='same', activation='relu', input_shape=input_shape))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 96,kernel_size =(3, 3), padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(filters = 192, kernel_size =(3,3), activation='relu', padding='same'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 256, kernel_size =(3,3), activation='relu', padding='same'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Conv2D(filters = 256, kernel_size =(3,3), activation='relu', padding='same'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 256, kernel_size =(3,3), activation='relu', padding='same'))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Dense(units=10, activation='softmax'))
model.compile(optimizer=OPTIMIZER, loss="categorical_crossentropy", metrics=["accuracy"] )
def cross_validate(X, y, model_name='RF', cv=5, scoring='accuracy',
gridsearch=True):

 switcher = ML_Classifier_Switcher()
model = switcher.pick_model(model_name)
 if gridsearch:
gr = GridSearchCV(model, switcher.param_grid,
scoring=scoring, cv=cv, n_jobs=-1)
 gr.fit(X, y)
 model = gr.best_estimator_
cvr = cross_val_score(model, X, y, cv=cv, scoring=scoring)
 cvr = pd.DataFrame(cvr ).T
cvr.index = [model_name]
cvr.columns = ['fold_{}'.format(x+1)for x in cvr.columns]
cvr['mean'] = cvr.mean(axis=1)
 cvr['std'] = cvr.std(axis=1)
 return cvr, model<compute_train_metric>
learning_rate = ReduceLROnPlateau(monitor='val_acc', patience=2, verbose=1, factor=0.5, min_lr=1.0e-5 )
def cross_validate_models(X, y, cv=5, scoring='accuracy'):
models = ['LR', 'KNN', 'SVM', 'RF']
cvrs = []
bests = []
for model_name in models:
print('Optimizing {} model'.format(model_name))
cvr, best = cross_validate(X, y, model_name=model_name, cv=cv, scoring=scoring,
gridsearch=True)
 cvrs.append(cvr)
 bests.append(best)
 cvr = pd.concat(cvrs, axis=0)
 return cvr, bests<compute_train_metric>
EarlyStopping_cb = keras.callbacks.EarlyStopping(monitor='val_acc', patience=5, verbose=1, mode='auto' )
cvr, bests = cross_validate_models(X,y)
 print(cvr )<save_to_csv>
csv_logger = CSVLogger('model.log' )
def predict_on_test(X_test, model, submission_df, target='Survived'):
preds_test = model.predict(X_test)
 submission_df.loc[:, target] = [int(x)for x in preds_test]
submission_df.to_csv('submission.csv', index=False)
 return
predict_on_test(X_test, bests[-1], submission_df )<load_from_csv>
x_train2, x_val2, y_train2, y_val2 = train_test_split(x_train, y_train, test_size=VALIDATION_SPLIT, random_state=0 )
train_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_data = pd.read_csv('/kaggle/input/titanic/test.csv' )<filter>
def fit_model() :
model_file = './log_dir/model_file' + '.hdf5'
checkpointer = ModelCheckpoint(filepath=model_file, verbose=1, monitor='val_acc', save_best_only=True)
 history = model.fit_generator(datagen.flow(x_train2, y_train2, batch_size=BATCH_SIZE),
validation_data=(x_val2, y_val2),
epochs = NB_EPOCH,
verbose = 2,
steps_per_epoch=x_train2.shape[0] // BATCH_SIZE ,
callbacks=[learning_rate, checkpointer, csv_logger, EarlyStopping_cb])
 return(history )
women = train_data.loc[train_data.Sex == 'female']['Survived']
print('Women survived',sum(women)/len(women))
men = train_data.loc[train_data.Sex == 'male']['Survived']
print('Men survived',sum(men)/len(men))<categorify>
history = fit_model()
train_data['female'] = pd.get_dummies(train_data['Sex'])['female']
test_data['female'] = pd.get_dummies(test_data['Sex'])['female']<feature_engineering>
results = np.zeros(( x_test.shape[0], NB_CLASSES))
model_file = './log_dir/model_file' + '.hdf5'
model.load_weights(model_file)
 results = results + model.predict(x_test)
 results = np.argmax(results, axis=1)
 results = pd.Series(results, name="Label")
 submission = pd.concat([pd.Series(range(1, 28001), name="ImageId"), results], axis=1)
 submission.to_csv("submission.csv", index=False )
<filter><EOS> 
!wc -l./submission.csv