Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<train_model>
%matplotlib inline
keras_version = keras.__version__
tf_version = K.tensorflow_backend.tf.VERSION
print("keras version:", keras_version)
 print(K.backend() , "version:", tf_version )
lr=LogisticRegression()
lr.fit(X_train,y_train)
 lr.score(X_val,y_val )<predict_on_test>
rawdata = np.loadtxt('.. /input/train.csv', dtype=int, delimiter=',', skiprows=1 )
test_data['cuisine']=lr.predict(test_data.drop(['id','ingredients'],axis=1))<prepare_output>
y_oh = to_categorical(y, num_classes)
 X_scaled = X / 127.5 - 1
X_scaled = np.expand_dims(X_scaled, -1)
 num_val = int(y.shape[0] * 0.1)
 validation_mask = np.zeros(y.shape[0], np.bool)
 np.random.seed(1)
 for c in range(num_classes):
idxs = np.random.choice(np.flatnonzero(y == c), num_val // 10, replace=False)
 validation_mask[idxs] = 1
np.random.seed(None)
 X_train = X_scaled[~validation_mask]
X_val = X_scaled[validation_mask]
print("Train/val pixel shapes:", X_train.shape, X_val.shape)
 y_train = y_oh[~validation_mask]
y_val = y_oh[validation_mask]
print("Train/val label shapes:", y_train.shape, y_val.shape)
 print("Validation Set Class Distribution:", np.bincount(y[validation_mask]))
Submission=test_data[['id','cuisine']]
Submission.set_index('id',inplace=True )<save_to_csv>
def conv2D_bn_relu(x, filters, kernel_size, strides, padding='valid', kernel_initializer='glorot_uniform', name=None):

 x = layers.Conv2D(filters=filters,
kernel_size=kernel_size,
strides=strides,
padding=padding,
kernel_initializer=kernel_initializer,
name=name,
use_bias=False )(x)
 x = layers.BatchNormalization(scale=False )(x)
 return layers.Activation('relu' )(x)
 def inception_module_A(x, filters=None, kernel_initializer='glorot_uniform'):

 if filters is None:
filters = int(x.shape[-1])
 branch_filters = filters // 4
b1 = conv2D_bn_relu(x,
filters=(branch_filters // 3)* 2,
kernel_size=1,
strides=1,
kernel_initializer=kernel_initializer)
 b1 = conv2D_bn_relu(b1,
filters=branch_filters,
kernel_size=3,
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b2 = conv2D_bn_relu(x,
filters=(branch_filters // 3)* 2,
kernel_size=1,
strides=1,
kernel_initializer=kernel_initializer)
 b2 = conv2D_bn_relu(b2,
filters=branch_filters,
kernel_size=3,
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b2 = conv2D_bn_relu(b2,
filters=branch_filters,
kernel_size=3,
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b3 = conv2D_bn_relu(x,
filters=branch_filters,
kernel_size=1,
strides=1,
kernel_initializer=kernel_initializer)
 pool = layers.AveragePooling2D(pool_size=(3, 3), strides=1, padding='same' )(x)
 pool = conv2D_bn_relu(pool,
filters=branch_filters,
kernel_size=1,
strides=1,
kernel_initializer=kernel_initializer)
 return layers.concatenate([b1, b2, b3, pool])
 def inception_module_C(x, filters=None, kernel_initializer='glorot_uniform'):

 if filters is None:
filters = int(x.shape[-1])
 branch_filters = filters // 6
b1 = conv2D_bn_relu(x,
filters=(branch_filters // 2)* 3,
kernel_size=1,
strides=1,
kernel_initializer=kernel_initializer)
 b1a = conv2D_bn_relu(b1,
filters=branch_filters,
kernel_size=(1, 3),
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b1b = conv2D_bn_relu(b1,
filters=branch_filters,
kernel_size=(3, 1),
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b2 = conv2D_bn_relu(x,
filters=(branch_filters // 2)* 3,
kernel_size=1,
strides=1,
kernel_initializer=kernel_initializer)
 b2 = conv2D_bn_relu(b2,
filters=(branch_filters // 4)* 7,
kernel_size=(1, 3),
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b2 = conv2D_bn_relu(b2,
filters=branch_filters * 2,
kernel_size=(3, 1),
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b2a = conv2D_bn_relu(b2,
filters=branch_filters,
kernel_size=(1, 3),
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b2b = conv2D_bn_relu(b2,
branch_filters,
kernel_size=(3, 1),
strides=1,
padding='same',
kernel_initializer=kernel_initializer)
 b3 = conv2D_bn_relu(x,
filters=branch_filters,
kernel_size=1,
strides=1,
kernel_initializer=kernel_initializer)
 pool = layers.AveragePooling2D(pool_size=(3, 3), strides=1, padding='same' )(x)
 pool = conv2D_bn_relu(pool,
filters=branch_filters,
kernel_size=1,
strides=1,
kernel_initializer=kernel_initializer)
 return layers.concatenate([b1a, b1b, b2a, b2b, b3, pool] )
Submission.to_csv('Submission.csv' )<import_modules>
K.clear_session()
stem_width = 64
inputs = layers.Input(shape=X_scaled.shape[1:])
 x = conv2D_bn_relu(inputs,
filters=stem_width,
kernel_size=5,
strides=2,
padding='valid',
name='conv_1')
 x = inception_module_A(x, filters=int(1.5*stem_width))
x = layers.SpatialDropout2D(0.2 )(x)
 x = inception_module_A(x, filters=int(1.5*stem_width))
x = layers.MaxPooling2D(pool_size=(3, 3), strides=2, padding='same' )(x)
 x = layers.SpatialDropout2D(0.2 )(x)
 x = inception_module_C(x, filters=int(2.25*stem_width))
x = layers.GlobalAveragePooling2D()(x)
 x = layers.Dropout(0.3 )(x)
 x = layers.Dense(num_classes, name='logits' )(x)
 x = layers.Activation('softmax', name='softmax' )(x)
 model = Model(inputs=inputs, outputs=x)
 model.summary()
import xgboost as xgb
import numpy as np
import pandas as pd
import random
import optuna
from sklearn.model_selection import KFold, train_test_split
from sklearn.metrics import accuracy_score
from sklearn.metrics import mean_squared_error<load_from_csv>
epsilon = 0.001
y_train_smooth = y_train *(1 - epsilon)+ epsilon / 10
print(y_train_smooth )
train = pd.read_csv(".. /input/tabular-playground-series-feb-2021/train.csv")
 test = pd.read_csv(".. /input/tabular-playground-series-feb-2021/test.csv" )<categorify>
def elastic_transform(image, alpha_range, sigma, random_state=None):

 if random_state is None:
random_state = np.random.RandomState(None)
 if np.isscalar(alpha_range):
alpha = alpha_range
else:
alpha = np.random.uniform(low=alpha_range[0], high=alpha_range[1])
 shape = image.shape
dx = gaussian_filter(( random_state.rand(*shape)* 2 - 1), sigma)* alpha
dy = gaussian_filter(( random_state.rand(*shape)* 2 - 1), sigma)* alpha
x, y, z = np.meshgrid(np.arange(shape[0]), np.arange(shape[1]), np.arange(shape[2]), indexing='ij')
 indices = np.reshape(x+dx,(-1, 1)) , np.reshape(y+dy,(-1, 1)) , np.reshape(z,(-1, 1))
return map_coordinates(image, indices, order=1, mode='reflect' ).reshape(shape )
df=train
for c in df.columns:
if df[c].dtype=='object':
lbl = LabelEncoder()
df[c]=df[c].fillna('N')
 lbl.fit(list(df[c].values))
df[c] = lbl.transform(df[c].values)
 train=df<categorify>
class CosineAnneal(keras.callbacks.Callback):

 def __init__(self, max_lr, min_lr, T, T_mul=1, decay_rate=1.0):
self.max_lr = max_lr
self.min_lr = min_lr
self.decay_rate = decay_rate
self.T = T
self.T_cur = 0
self.T_mul = T_mul
self.step = 0
def on_batch_begin(self, batch, logs=None):
if self.T <= self.T_cur:
self.max_lr *= self.decay_rate
self.min_lr *= self.decay_rate
self.T *= self.T_mul
self.T_cur = 0
self.step = 0
lr = self.min_lr + 0.5 *(self.max_lr - self.min_lr)*(1 + np.cos(self.T_cur * np.pi / self.T))
K.set_value(self.model.optimizer.lr, lr)
 self.step += 1
self.T_cur = self.step / self.params['steps']
def on_epoch_end(self, epoch, logs=None):
logs = logs or {}
logs['lr'] = K.get_value(self.model.optimizer.lr )
df=test
for c in df.columns:
if df[c].dtype=='object':
lbl = LabelEncoder()
df[c]=df[c].fillna('N')
 lbl.fit(list(df[c].values))
df[c] = lbl.transform(df[c].values)
 test=df<drop_column>
model.compile(loss='categorical_crossentropy',
optimizer=optimizers.Adamax(lr=0.01, beta_1=0.49, beta_2=0.999),
metrics=['accuracy'] )
target = train['target']
data = train.drop(['target','id'],axis=1 )<split>
batch_size = 32
epochs = 100
time_id = time.strftime("%Y-%m-%d_%H-%M-%S")
 best_model_filename = 'mnist-inception-best-' + time_id + '.hdf5'
pre_train_weights = model.get_layer('conv_1' ).get_weights() [0]
pre_train_weights = pre_train_weights.transpose(3, 2, 0, 1)
 annealer = CosineAnneal(max_lr=0.014, min_lr=0.003, T=5, T_mul=1, decay_rate=0.99)
 chkpt = keras.callbacks.ModelCheckpoint(best_model_filename, monitor='val_acc',
save_best_only=True, verbose=False)
 datagen = ImageDataGenerator(
 width_shift_range=2,
height_shift_range=2,
preprocessing_function=lambda x: elastic_transform(x, alpha_range=[8, 10], sigma=3)
)
 history = model.fit_generator(
 datagen.flow(X_train, y_train_smooth, batch_size=batch_size, shuffle=True),
epochs=epochs,
steps_per_epoch=(len(y_train)- 1)// batch_size + 1,
validation_data=(X_val, y_val),
callbacks=[annealer, chkpt]
)
def objective(trial,data=data,target=target):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.2,random_state=42)
 param = {
'objective': trial.suggest_categorical('objective',['reg:tweedie']),
'tree_method': trial.suggest_categorical('tree_method',['hist']),
'lambda': trial.suggest_loguniform('lambda',1e-3,10.0),
'alpha': trial.suggest_loguniform('alpha',1e-3,10.0),
'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0]),
'subsample': trial.suggest_categorical('subsample', [0.4,0.5,0.6,0.7,0.8,1.0]),
'learning_rate': trial.suggest_categorical('learning_rate', [0.008,0.01,0.012,0.014,0.016,0.018,0.02]),
'n_estimators': trial.suggest_categorical('n_estimators', [1000,2000,4000,8000]),
'max_depth': trial.suggest_categorical('max_depth', [5,7,9,11,13,15,17,20]),
'random_state': trial.suggest_categorical('random_state', [24,48,2020]),
'min_child_weight': trial.suggest_int('min_child_weight', 1,300),
'use_label_encoder': trial.suggest_categorical('use_label_encoder',[False])
 }
model = xgb.XGBRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse<train_model>
model = keras.models.load_model(best_model_filename)
 evaluate_model(model, X_val, y_val, log=history.history, pre_train_weights=pre_train_weights )
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=16)
 print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )<create_dataframe>
test_data = np.loadtxt('.. /input/test.csv', dtype=int, delimiter=',', skiprows=1 )
study.trials_dataframe()<find_best_params>
test_scores = model.predict(test_images)
 test_predictions = np.argmax(test_scores, axis=1 )
Best_trial=study.best_trial.params
print(Best_trial )<load_from_csv>
header = 'ImageId,Label'
submission = np.stack(( range(1, 28001), test_predictions), axis=1)
 np.savetxt('submission-' + time_id + '.csv', submission, fmt='%i', delimiter=',', header=header, comments='' )