Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<init_hyperparams>
!pip install neural-structured-learning
!pip install tensorflow-gpu==2.1.0
if allow_tuning:
params_xgb = {'n_estimators': [1000],
'learning_rate': [0.1],
'max_depth': [5],
'min_child_weight': [1],
'gamma': [0],
'subsample': [0.8],
'colsample_bytree': [0.8],
'n_jobs': [-1],
'objective': ['binary:logistic'],
'use_label_encoder': [False],
'eval_metric': ['logloss'],
'scale_pos_weight': [1]}
search_xgb, params_xgb = xgb_gridsearch(params_xgb,
['learning_rate'],
[[0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.15, 0.2]],
X_train, y_train)
 search_xgb, params_xgb = xgb_gridsearch(params_xgb,
['max_depth', 'min_child_weight'],
[range(3, 10), range(1, 6)],
X_train, y_train)
 search_xgb, params_xgb = xgb_gridsearch(params_xgb,
['gamma'],
[[0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 2]],
X_train, y_train)
 search_xgb, params_xgb = xgb_gridsearch(params_xgb,
['subsample', 'colsample_bytree'],
[[i/100.0 for i in range(75,90,5)], [i/100.0 for i in range(75,90,5)]],
X_train, y_train)
 search_xgb, params_xgb = xgb_gridsearch(params_xgb,
['reg_alpha'],
[[1e-5, 1e-2, 0.1, 1, 100]],
X_train, y_train)
 params_xgb['n_estimators'] = [5000]
search_xgb, params_xgb = xgb_gridsearch(params_xgb,
['learning_rate'],
[[0.001, 0.005, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.15, 0.2]],
X_train, y_train, last=True)
 x_train, x_test = train_test_split(X_train, test_size=.2, random_state=42)
 y_train_tmp, y_test_tmp = train_test_split(y_train, test_size=.2, random_state=42)
 model_xgb = XGBClassifier(**params_xgb)
 model_xgb = model_xgb.fit(x_train, y_train_tmp, eval_set=[(x_test, y_test_tmp)], eval_metric=['logloss'], early_stopping_rounds=15, verbose=0)
 search_xgb.best_estimator_.n_estimators = model_xgb.best_iteration<choose_model_class>
train = pd.read_csv('/kaggle/input/train.csv')
 train.head()
if allow_tuning:
model_knn = search_knn.best_estimator_
model_logistic = search_logistic.best_estimator_
model_svc = search_svc.best_estimator_
model_rf = search_rf.best_estimator_
model_xgb = search_xgb.best_estimator_
else:
model_knn = knn(algorithm='auto',
n_neighbors=9,
p=1,
weights='uniform')
 model_logistic = LogisticRegression(C=0.08858667904100823,
max_iter=2000,
penalty='l2',
solver='liblinear')
 model_svc = SVC(C=70,
gamma=0.0106,
kernel='rbf',
probability=True)
 model_rf = RandomForestClassifier(bootstrap=True,
criterion='entropy',
max_depth=50, max_features=6,
min_samples_leaf=1,
min_samples_split=10,
n_estimators=100,
random_state=734)
 model_xgb = XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.8,
enable_categorical=False, eval_metric='logloss', gamma=0.8,gpu_id=-1, importance_type=None, interaction_constraints='',
learning_rate=0.15, max_delta_step=0, max_depth=5,
min_child_weight=1, missing=np.nan, monotone_constraints='() ',
n_estimators=15, n_jobs=-1, num_parallel_tree=1, predictor='auto',
random_state=0, reg_alpha=1e-05, reg_lambda=1, scale_pos_weight=1,
subsample=0.8, tree_method='exact', use_label_encoder=False,
validate_parameters=1, verbosity=0)
 models = {
'knn': model_knn,
'logistic': model_logistic,
'svc': model_svc,
'rf': model_rf,
'xgb': model_xgb
}<choose_model_class>
def get_labels(train):
return np.array(train['label'])
 def get_features(train):
features = train.drop(['label'], axis=1)
 features_normalized = features / 255
return features_normalized
def select_models(start, cnt, goal, estimators, voting):
if cnt == goal:
estimators_copy = copy.deepcopy(estimators)
 voting_name = f'{voting}_' + '_'.join([i[0] for i in list(estimators_copy)])
 models[voting_name] = VotingClassifier(estimators=estimators_copy, voting=voting)
 return
for i in range(start, 5):
estimators.append(list(models.items())[i])
 select_models(i + 1, cnt + 1, goal, estimators, voting)
 estimators.pop()<define_search_space>
def get_generator_dict(in_gen, should_augment=True):
if should_augment:
image_gen = ImageDataGenerator(fill_mode='reflect',
data_format='channels_last',
brightness_range=[0.5, 1.5])
 else:
image_gen = ImageDataGenerator(fill_mode='reflect',
data_format='channels_last',
brightness_range=[1, 1])
 for items in in_gen:
if len(items)== 2:
in_x, in_y = items
else:
in_x, in_y, weights = items
g_x = image_gen.flow(255 * in_x, in_y, batch_size=in_x.shape[0])
 x, y = next(g_x)
 yield {'feature': x / 255.0, 'label': y}
select_models(0, 0, 2, [], 'hard')
 select_models(0, 0, 3, [], 'hard')
 select_models(0, 0, 4, [], 'hard')
 select_models(0, 0, 5, [], 'hard')
 select_models(0, 0, 2, [], 'soft')
 select_models(0, 0, 3, [], 'soft')
 select_models(0, 0, 4, [], 'soft')
 select_models(0, 0, 5, [], 'soft' )<create_dataframe>
def get_model(shape, n_class):
model = Sequential()
model.add(Conv2D(64,(4,4), input_shape=shape))
model.add(MaxPool2D(( 2,2)))
 model.add(BatchNormalization())
 model.add(tf.keras.layers.Activation("selu")) ,
model.add(Conv2D(128,(3,3)))
 model.add(MaxPool2D(( 2,2)))
 model.add(BatchNormalization())
 model.add(tf.keras.layers.Activation("selu")) ,
model.add(Conv2D(256,(2,2)))
 model.add(MaxPool2D(( 2,2)))
 model.add(Dropout(0.2))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(tf.keras.layers.Activation("selu"))
model.add(Dense(64, kernel_initializer="he_normal", use_bias=False))
model.add(Dropout(0.2))
model.add(BatchNormalization())
 model.add(tf.keras.layers.Activation("selu"))
model.add(Dense(32, kernel_initializer="he_normal", use_bias=False))
model.add(BatchNormalization())
 model.add(Dense(n_class, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
result_by_model = pd.DataFrame({'model name': models.keys() , 'model': models.values() , 'score': 0} )<compute_train_metric>
def get_adv_model(shape, n_class):
model = Sequential([
Conv2D(64,(4,4), input_shape=shape, dynamic=True),
MaxPool2D(( 2,2)) ,
BatchNormalization() ,
tf.keras.layers.Activation("selu"),
Conv2D(128,(3,3)) ,
MaxPool2D(( 2,2)) ,
BatchNormalization() ,
tf.keras.layers.Activation("selu"),
Conv2D(256,(2,2)) ,
MaxPool2D(( 2,2)) ,
Dropout(0.2),
BatchNormalization() ,
Flatten() ,
tf.keras.layers.Activation("selu"),
Dense(64, kernel_initializer="he_normal", use_bias=False),
Dropout(0.2),
BatchNormalization() ,
tf.keras.layers.Activation("selu"),
Dense(32, kernel_initializer="he_normal", use_bias=False),
BatchNormalization() ,
Dense(n_class, activation='softmax')])
 adv_config = nsl.configs.make_adv_reg_config(multiplier=0.2, adv_step_size=0.05)
 adv_model = nsl.keras.AdversarialRegularization(model, adv_config=adv_config)
 adv_model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'], run_eagerly=True)
 return adv_model
for name, model in models.items() :
result_by_model.loc[result_by_model['model name'] == name, 'score'] = cross_val_score(model, X_train,y_train,cv=5 ).mean()<sort_values>
def get_sample_model(shape, n_class):
model = tf.keras.Sequential([
tf.keras.Input(shape, name='feature'),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(128, activation=tf.nn.relu),
tf.keras.layers.Dense(n_class, activation=tf.nn.softmax)
 ])
 adv_config = nsl.configs.make_adv_reg_config(multiplier=0.2, adv_step_size=0.05)
 adv_model = nsl.keras.AdversarialRegularization(model, adv_config=adv_config)
 adv_model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
 return adv_model
result_by_model.sort_values('score', ascending=False ).reset_index(drop=True )<save_to_csv>
features = get_features(train)
 labels = get_labels(train)
model_name = 'rf'
models[model_name].fit(X_train, y_train)
 y_pred = models[model_name].predict(X_test ).astype('int')
 submission = pd.DataFrame({'PassengerId': test.PassengerId,
'Survived': y_pred})
 submission.to_csv('submission.csv', index = False )<import_modules>
model = get_adv_model(x[0].shape, 10)
 print(model )
import numpy as np
import pandas as pd
from sklearn import ensemble
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_validate,GridSearchCV
import lightgbm as lgbm
from collections import Counter<load_from_csv>
train_x = x[:36000]
test_x = x[36000:]
labels_train = labels[:36000]
labels_test = labels[36000:]
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv" )<prepare_x_and_y>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(train_x )
y = train['Survived']
X = train.drop(['Survived'], axis=1)
 X_test = test<define_search_space>
earlyStop = tf.keras.callbacks.EarlyStopping(monitor='val_acc', patience=10)
 reduceLROnPlateau = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )
params={"n_estimators":np.arange(50,200,10),
"max_depth":np.arange(1,11,1),
"learning_rate":[0.1,0.01,0.001]
}<train_on_grid>
model.fit(datagen.flow(train_x, labels_train, batch_size=32),
steps_per_epoch=len(train_x)// 32,
epochs=5)
xgb_est=lgbm.LGBMClassifier(
 random_state=42,
objective='binary',
eval_metric="auc" 
)
 gr_xgb_est=GridSearchCV(xgb_est,param_grid=params,cv=5,n_jobs=-1,verbose=10)
 gr_xgb_est.fit(X,y )<find_best_params>
model.evaluate({'feature': test_x, 'label': np.array(labels_test)} )
gr_xgb_est.best_estimator_.get_params()<predict_on_test>
test = pd.read_csv('/kaggle/input/test.csv')
 test.head()
pred_test = gr_xgb_est.predict(X_test )<save_to_csv>
test_normalized = test/ 255
x_test = np.array(test_normalized ).reshape(-1,28,28,1 )
submission = pd.read_csv('.. /input/titanic/gender_submission.csv')
 submission['Survived'] =(pred_test > 0.5 ).astype(int)
 submission.to_csv('sub.csv', index=False)
 submission.head()<set_options>
lo = [0] * len(x_test)
 predictions = model.predict({'feature':x_test, 'label':np.array(lo)} )
seed = 50
os.environ['PYTHONHASHSEED'] = str(seed)
 random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.enabled = False<set_options>
response = []
for i, prediction in enumerate(predictions):
response.append([i + 1, np.argmax(prediction)])
 resposta = pd.DataFrame(response, columns=['ImageId', 'Label'])
 resposta.to_csv('output.csv', index=False)