Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<filter>
DEVICE = "TPU"
SEED = 8080
FOLDS = 5
FOLD_WEIGHTS = [1./FOLDS]*FOLDS
BATCH_SIZE = 256
EPOCHS = 5000
MONITOR = "val_loss"
MONITOR_MODE = "min"
ES_PATIENCE = 5
LR_PATIENCE = 0
LR_FACTOR = 0.5
EFF_NET = 3
EFF_NET_WEIGHTS = 'noisy-student'
LABEL_SMOOTHING = 0.1
VERBOSE = 1
df_train[(df_train['floor'])== 33]<drop_column>
!pip install -q efficientnet >> /dev/null
df_train.drop(df_train.index[7457], inplace=True )<data_type_conversions>
import numpy as np
import pandas as pd
from tqdm import tqdm
import matplotlib.pyplot as plt
import tensorflow as tf
import tensorflow.keras.backend as K
import efficientnet.tfkeras as efn
from sklearn.model_selection import KFold
from keras.preprocessing.image import ImageDataGenerator
df_train['year'] = df_train['timestamp'].apply(lambda x: x[:4] ).astype(int)
 df_train['month'] = df_train['timestamp'].apply(lambda x: x[5:7] ).astype(int)
 df_test['year'] = df_test['timestamp'].apply(lambda x: x[:4] ).astype(int)
 df_test['month'] = df_test['timestamp'].apply(lambda x: x[5:7] ).astype(int )<sort_values>
if DEVICE == "TPU":
print("connecting to TPU...")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("initializing TPU...")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except _:
print("failed to initialize TPU")
 else:
DEVICE = "GPU"
if DEVICE != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if DEVICE == "GPU":
print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )
missingValues = df_train.columns[df_train.isnull().any() ].tolist()
pd.isnull(df_train[missingValues] ).sum().sort_values(ascending=False )<define_variables>
train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 train.describe()
cols_fillna_mode = ['floor',
'product_type',
'num_room',
'state',
'hospital_beds_raion',
'build_count_brick',
'build_count_monolith',
'green_part_2000']
cols_fillna_mean = ['life_sq',
'metro_min_walk',
'metro_km_walk',
'railroad_station_walk_km',
'railroad_station_walk_min',
'cafe_sum_1500_min_price_avg',
'cafe_sum_1500_max_price_avg',
'cafe_avg_price_1500',
'cafe_sum_2000_max_price_avg',
'cafe_avg_price_2000']<categorify>
X = train.drop(labels=['label'], axis=1)
 X = X.astype('float32')
 X = X / 255
X = X.values.reshape(X.shape[0],28,28,1)
 X = np.pad(X,(( 0,0),(2,2),(2,2),(0,0)) , mode='constant')
 X = np.squeeze(X, axis=-1)
 X = stacked_img = np.stack(( X,)*3, axis=-1)
 X.shape
for col in cols_fillna_mode:
df_train[col].fillna(df_train[col].mode().iloc[0],inplace=True)
 df_test[col].fillna(df_train[col].mode().iloc[0],inplace=True)
 for col in cols_fillna_mean:
df_train[col].fillna(df_train[col].mean() ,inplace=True)
 df_test[col].fillna(df_train[col].mean() ,inplace=True )<define_variables>
y = train['label'].values.astype('float32')
 y = tf.keras.utils.to_categorical(y, 10)
 y
numerical_features = df_train.dtypes[df_train.dtypes != "object"].index
categorical_features = df_train.dtypes[df_train.dtypes == "object"].index
print("Кол-во количественных признаков: ", len(numerical_features))
print("Кол-во категориальных признаков: ", len(categorical_features))<sort_values>
test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv")
 test.describe()
df_train.isna().sum().sort_values(ascending=False )<drop_column>
X_test = test.astype('float32')
 X_test = X_test / 255
X_test = X_test.values.reshape(X_test.shape[0],28,28,1)
 X_test = np.pad(X_test,(( 0,0),(2,2),(2,2),(0,0)) , mode='constant')
 X_test = np.squeeze(X_test, axis=-1)
 X_test = stacked_img = np.stack(( X_test,)*3, axis=-1)
 X_test.shape
df_train.drop(['id', 'price_doc', 'timestamp'], axis=1, inplace=True)
 id_test = df_test['id']
df_test.drop(['id', 'timestamp'], axis=1, inplace=True )<define_variables>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False
)
numerical_features = df_train.dtypes[df_train.dtypes != "object"].index
categorical_features = df_train.dtypes[df_train.dtypes == "object"].index
print("Кол-во количественных признаков: ", len(numerical_features))
print("Кол-во категориальных признаков: ", len(categorical_features))<categorify>
eff_nets = [
efn.EfficientNetB0,
efn.EfficientNetB1,
efn.EfficientNetB2,
efn.EfficientNetB3,
efn.EfficientNetB4,
efn.EfficientNetB5,
efn.EfficientNetB6,
efn.EfficientNetB7,
efn.EfficientNetL2,
]
def build_model() :
inp = tf.keras.layers.Input(shape=(X.shape[1], X.shape[2], X.shape[3]))
oup = eff_nets[EFF_NET](
 input_shape=(X.shape[1], X.shape[2], X.shape[3]),
weights=EFF_NET_WEIGHTS,
include_top=False,
)(inp)
 oup = tf.keras.layers.GlobalAveragePooling2D()(oup)
 oup = tf.keras.layers.Dense(512, activation='linear' )(oup)
 oup = tf.keras.layers.Activation('relu' )(oup)
 oup = tf.keras.layers.Dropout(0.5 )(oup)
 oup = tf.keras.layers.Dense(10, activation='linear' )(oup)
 oup = tf.keras.layers.Activation('softmax' )(oup)
 model = tf.keras.Model(inputs=[inp], outputs=[oup])
 loss = tf.keras.losses.CategoricalCrossentropy(
 from_logits=False,
label_smoothing=LABEL_SMOOTHING, 
)
 opt = tf.keras.optimizers.Nadam(learning_rate=3e-4)
 model.compile(optimizer=opt,loss=loss,metrics=['acc'])
 return model
build_model().summary()
encoder = OneHotEncoder(handle_unknown='error')
 encoder_cols_train = pd.DataFrame(encoder.fit_transform(df_train[categorical_features] ).toarray())
 encoder_cols_test = pd.DataFrame(encoder.transform(df_test[categorical_features] ).toarray() )<categorify>
%%time
oof = np.zeros(( X.shape[0], y.shape[1]))
preds = np.zeros(( X_test.shape[0], y.shape[1]))
skf = KFold(n_splits=FOLDS,shuffle=True,random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(X)) :
if DEVICE=='TPU':
if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)
 print('
print('
print('
K.clear_session()
with strategy.scope() :
model = build_model()
weights_filename='fold-%i.h5'%fold
sv = tf.keras.callbacks.ModelCheckpoint(
 weights_filename, monitor=MONITOR, verbose=VERBOSE, save_best_only=True,
save_weights_only=True, mode=MONITOR_MODE, save_freq='epoch')
 lrr = tf.keras.callbacks.ReduceLROnPlateau(
 monitor=MONITOR,
factor=LR_FACTOR,
patience=LR_PATIENCE,
verbose=VERBOSE,
mode=MONITOR_MODE 
)
 es = tf.keras.callbacks.EarlyStopping(
 monitor=MONITOR,
patience=ES_PATIENCE,
verbose=VERBOSE,
mode=MONITOR_MODE, 
)
 print('Generating train data...')
 i = 0
datagen.fit(X[idxT])
 steps = 2 *(X[idxT].shape[0] // BATCH_SIZE)
 X_train = None
y_train = None
with tqdm(total=steps)as pbar:
for arr in datagen.flow(X[idxT], y[idxT], batch_size=BATCH_SIZE):
if X_train is None:
X_train = arr[0]
y_train = arr[1]
else:
X_train = np.concatenate(( X_train, arr[0]))
y_train = np.concatenate(( y_train, arr[1]))
i += 1
pbar.update(1)
 if i >= steps:
break
print('Training...')
 history = model.fit(
 X_train,
y_train,
batch_size=BATCH_SIZE,
epochs=EPOCHS,
callbacks = [
sv,
lrr,
es,
],
validation_data=(
 X[idxV], y[idxV] 
),
verbose=VERBOSE, 
)
 print('Loading best model...')
 model.load_weights('fold-%i.h5'%fold)
 print('Predicting OOF...')
 oof[idxV,] = model.predict([X[idxV]],verbose=VERBOSE)
 print('Predicting Test...')
 preds +=(model.predict([X_test], verbose=VERBOSE)* FOLD_WEIGHTS[fold])
 acc_sum = 0
for k in idxV:
if np.argmax(oof[k])== np.argmax(y[k]):
acc_sum += 1
print('>>>> FOLD {} Accuracy = {:.4f}%'.format(fold+1,acc_sum/len(idxV)*100))
print()
encoder_cols_train.columns = encoder.get_feature_names(categorical_features)
 encoder_cols_test.columns = encoder.get_feature_names(categorical_features)
 encoder_cols_train.index = df_train.index
encoder_cols_test.index = df_test.index<drop_column>
final_predictions = pd.Series(np.argmax(preds, axis=1), name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),final_predictions], axis=1)
 submission.to_csv("submission.csv",index=False)
 submission.head()