Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<install_modules>
import numpy as np
import sklearn as sk
import tensorflow as tf
import matplotlib.pyplot as plt
import pandas as pd
import sklearn.model_selection as ms
import sklearn.preprocessing as p
import math
!pip install scorecardpy<import_modules>
tf.version.VERSION
import numpy as np
import pandas as pd
from scipy.special import logit
import lightgbm as lgb
import scorecardpy as sc<load_from_csv>
mnist = pd.read_csv('.. /input/digit-recognizer/train.csv' )
train = pd.read_csv(".. /input/santander-customer-transaction-prediction/train.csv")
 test = pd.read_csv(".. /input/santander-customer-transaction-prediction/test.csv")
 train = train.drop('ID_code', axis = 1)
 train.head()<drop_column>
height = 28
width = 28
channels = 1
test_id = test.ID_code
test = test.drop('ID_code', axis = 1)
 test.head()<count_missing_values>
n_outputs = 10
print(f"The number of missing values in the training set is: {np.sum(np.sum(pd.isnull(train)))}")
 print(f"The number of missing values in the test set is: {np.sum(np.sum(pd.isnull(test)))}" )<sort_values>
mnist.loc[:3].apply(show_digit_and_print_label, axis=1)
correlations = train.drop("target", axis = 1 ).corr().abs().unstack().sort_values(kind = "quicksort" ).reset_index()
correlations = correlations[correlations['level_0'] != correlations['level_1']]
correlations.head(10 )<compute_test_metric>
X_data = mnist.drop(columns='label')
 y_data = mnist['label']
variables = train.drop("target", axis = 1 ).columns.values.tolist()
corr_pre_res = np.zeros(len(variables))
i = 0
for var in variables:
corr_pre_res[i] = np.corrcoef(train[var], train["target"])[0, 1]
i += 1<create_dataframe>
y_data = tf.keras.utils.to_categorical(y_data, num_classes = n_outputs)
 y_data.shape
corr_pre_res = abs(pd.DataFrame(corr_pre_res))
corr_pre_res.columns = ['corr_pre_res']
corr_pre_res.sort_values(by = 'corr_pre_res' )<groupby>
X_train, X_val, y_train, y_val = ms.train_test_split(X_data, y_data, test_size=0.15 )
features = [x for x in train.columns if x.startswith("var")]
hist_df = pd.DataFrame()
for var in features:
var_stats = train[var].append(test[var] ).value_counts()
hist_df[var] = pd.Series(test[var] ).map(var_stats)
 hist_df[var] = hist_df[var] > 1
ind = hist_df.sum(axis = 1)!= 200
var_stats = {var: train[var].append(test[ind][var] ).value_counts() for var in features}
pred = 0
for var in features:
model = lgb.LGBMClassifier(**{'learning_rate': 0.05,
'max_bin': 165,
'max_depth': 5,
'min_child_samples': 150,
'min_child_weight': 0.1,
'min_split_gain': 0.0018,
'n_estimators': 41,
'num_leaves': 6,
'reg_alpha': 2.0,
'reg_lambda': 2.54,
'objective': 'binary',
'n_jobs': -1})
 model = model.fit(np.hstack([train[var].values.reshape(-1, 1),
train[var].map(var_stats[var] ).values.reshape(-1, 1)]), train["target"].values)
 pred += logit(model.predict_proba(np.hstack([test[var].values.reshape(-1, 1),
test[var].map(var_stats[var] ).values.reshape(-1, 1)])) [:, 1])
 pd.DataFrame({"ID_code": test_id, "target": pred} ).to_csv("submission.csv", index = False )<define_variables>
scaler = p.StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_train = X_train.reshape(-1, height, width, channels)
 X_val = scaler.transform(X_val)
 X_val = X_val.reshape(-1, height, width, channels )
input_path = '/kaggle/input/severstal-steel-defect-detection/'
base = '/kaggle/input/severstal-inference-base'
requirements_dir = base + '/requirements/'<install_modules>
batch_size = 250
!pip -q config set global.disable-pip-version-check true
!pip -q install {requirements_dir}Keras_Applications-1.0.8-py3-none-any.whl
!pip -q install {requirements_dir}efficientnet-1.1.1-py3-none-any.whl<set_options>
train_data_gen = image_gen.flow(X_train, y=y_train, batch_size=batch_size )
!cp -r {base}/tpu_segmentation./
!cp -r {base}/*.py./
!rm -r tpu_segmentation *.py
AUTO = tf.data.experimental.AUTOTUNE
strategy = tf.distribute.get_strategy()
start_notebook = time()
print('Notebook started at: ', current_time_str())
 print('Tensorflow version: ', tf.__version__ )<define_variables>
class CosineAnnealingLearningRateCallback(tf.keras.callbacks.Callback):
def __init__(self, n_epochs, n_cycles, lrate_max, n_epochs_for_saving, verbose=0):
self.epochs = n_epochs
self.cycles = n_cycles
self.lr_max = lrate_max
self.n_epochs_for_saving = n_epochs_for_saving
self.best_val_acc_per_cycle = float('-inf')
 def is_save_range(self, epoch, epochs_per_cycle, n_epochs_for_saving):
epoch += 1
f, d = math.modf(epoch / epochs_per_cycle)
 next_end = epochs_per_cycle *(d +(1 if f > 0 else 0))
need_to_save = epoch >(next_end - n_epochs_for_saving)
 return need_to_save
def cosine_annealing(self, epoch, n_epochs, n_cycles, lrate_max):
epochs_per_cycle = math.floor(n_epochs/n_cycles)
 cos_inner =(math.pi *(epoch % epochs_per_cycle)) /(epochs_per_cycle)
 return lrate_max/2 *(math.cos(cos_inner)+ 1)
 def on_epoch_begin(self, epoch, logs=None):
lr = self.cosine_annealing(epoch, self.epochs, self.cycles, self.lr_max)
 tf.keras.backend.set_value(self.model.optimizer.lr, lr)
 def on_epoch_end(self, epoch, logs={}):
epochs_per_cycle = math.floor(self.epochs / self.cycles)
 if epoch % epochs_per_cycle == 0:
self.best_val_acc_per_cycle = float('-inf')
 isr = self.is_save_range(epoch, epochs_per_cycle, self.n_epochs_for_saving)
 last_val_acc = logs['val_accuracy']
if epoch != 0 and isr and last_val_acc > self.best_val_acc_per_cycle:
self.best_val_acc_per_cycle = last_val_acc
filename = f'snapshot_model_{epoch // epochs_per_cycle}.h5'
self.model.save(filename)
 print(f'saved snapshot {filename}, epoch: {epoch}, val_accuracy: {last_val_acc:.5f}')
 n_epochs = 300
n_cycles = n_epochs / 50
n_epochs_for_saving = 20
calrc = CosineAnnealingLearningRateCallback(n_epochs, n_cycles, 0.01, n_epochs_for_saving )
IMAGE_SIZE =(256, 1600)
 target_size =(128, 800)
 input_shape =(*target_size, 3)
 N_CLASSES = 4<define_variables>
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Conv2D(32, 3, 1, padding='same', activation='relu', input_shape=(height, width, channels)))
 model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Conv2D(32, 3, 1, padding='same', activation='relu', input_shape=(height, width, channels)))
 model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dropout(0.20))
model.add(tf.keras.layers.Conv2D(64, 3, 1, padding='same', activation='relu'))
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Conv2D(64, 3, 1, padding='same', activation='relu'))
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dropout(0.20))
model.add(tf.keras.layers.Conv2D(128, 3, 1, padding='same', activation='relu'))
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dropout(0.25))
model.add(tf.keras.layers.Flatten())
 model.add(tf.keras.layers.Dense(128, activation='relu'))
model.add(tf.keras.layers.BatchNormalization())
 model.add(tf.keras.layers.Dropout(0.20))
model.add(tf.keras.layers.Dense(10, activation='softmax'))
model.compile(optimizer='Adam', loss="categorical_crossentropy", metrics=["accuracy"] )
test_fnames = tf.io.gfile.glob(input_path + 'test_images/*')
 test_ids = [x.split('/')[-1].split('.')[0] for x in test_fnames]
get_test_path = lambda x: input_path + 'test_images/' + x + '.jpg'<categorify>
model.fit(train_data_gen, batch_size=batch_size, epochs = n_epochs, validation_data =(X_val, y_val), callbacks=[calrc], verbose=2 )
def normalize_and_reshape(img, target_size):
img = tf.image.resize(img, target_size)
 img = tf.cast(img, tf.float32)/ 255.0
img = tf.reshape(img, [*target_size, 3])
 return img
def get_image_and_id(file_name, target_size):
img = tf.io.read_file(file_name)
 img = tf.image.decode_jpeg(img, channels=3)
 img = normalize_and_reshape(img, target_size)
 img_id = tf.strings.split(file_name, os.path.sep)[-1]
img_id = tf.strings.split(img_id, '.')[0]
return img, img_id
def get_test_dataset(fnames, target_size, batch_size):
dataset = tf.data.Dataset.from_tensor_slices(fnames)
 dataset = dataset.map(lambda file_name: get_image_and_id(file_name, target_size), num_parallel_calls=AUTO)
 dataset = dataset.batch(batch_size=batch_size, drop_remainder=False)
 dataset = dataset.prefetch(AUTO)
 return dataset<load_pretrained>
def load_all_models(n_models):
all_models = list()
for i in range(n_models):
filename = f'snapshot_model_{str(i)}.h5'
model = tf.keras.models.load_model(filename)
 all_models.append(model)
 return all_models
def ensemble_predictions(models, testX):
yhats = [model.predict(testX)for model in models]
yhats = np.array(yhats)
 summed = np.sum(yhats, axis=0)
 result = np.argmax(summed, axis=1)
 return result
def evaluate_n_models(models, n_models, testX, testy):
subset = models[:n_models]
yhat = ensemble_predictions(subset, testX)
 return sk.metrics.accuracy_score(testy, yhat )
df = pd.read_csv(base + '/weights_meta.csv')
 df1 = df[df.source == 1]
df2 = df[df.source == 2]
bin1 = df1[df1.type == 'bin']
bin1 = get_best_weights(bin1, 1)
 seg1 = df1[df1.type != 'bin']
seg1 = get_best_weights(seg1, 1)
 bin2 = df2[df2.type == 'bin']
seg2 = df2[df2.type != 'bin']
bin_weights = list(bin2.filename)+ list(bin1.filename)
 seg_weights = list(seg2.filename)+ list(seg1.filename )<load_pretrained>
X_pred = pd.read_csv('.. /input/digit-recognizer/test.csv')
 X_pred = scaler.transform(X_pred)
 X_pred = X_pred.reshape(-1, height, width, channels )
<create_dataframe><EOS> 
y_pred = pd.DataFrame()
y_pred['ImageId'] = pd.Series(range(1,X_pred.shape[0] + 1))
y_pred['Label'] = ensemble_predictions(models, X_pred)
 y_pred.to_csv("submission.csv", index=False )