Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<define_variables>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
batch_size = 32
epochs=50<choose_model_class>
from fastai import *
from fastai.vision import *
import os
import pandas as pd
import numpy as np
from pathlib import Path
from PIL import Image
simple_NN = keras.Sequential([
layers.Dense(100, activation='relu', input_shape=(28,28,1)) ,
layers.Dropout(0.3),
layers.Flatten() ,
layers.Dense(units=100, activation='relu'),
layers.Dense(units=10, activation='softmax')
 ])
<choose_model_class>
warnings.simplefilter('ignore' )
simple_NN.compile(
 optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'] 
)
 simple_NN.optimizer.lr=0.001 
<choose_model_class>
train_df = pd.read_csv(input_dir/"train.csv")
 train_df.head(3 )
early_stopping = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)
<choose_model_class>
test_df = pd.read_csv(input_dir/"test.csv")
 test_df.head(3 )
lrr = ReduceLROnPlateau(monitor='val_loss',patience=3,verbose=1,factor=0.5, min_lr=0.00001 )<train_model>
train_dir = Path(".. /train")
 test_dir = Path(".. /test" )
history_simple_NN = simple_NN.fit(
 x=X_train,
y=y_train,
validation_data=(X_val, y_val),
batch_size=batch_size,
epochs=epochs,
shuffle=True,
verbose=2,
callbacks=[early_stopping, lrr] 
)
<choose_model_class>
sorted(os.listdir(train_dir))
CNN = keras.Sequential([
layers.Conv2D(32, kernel_size=(3,3), activation='relu', padding='same', input_shape=(28,28,1)) ,
layers.Conv2D(64, kernel_size=(3,3), activation='relu', padding='same'),
layers.Conv2D(64, kernel_size=(3,3), activation='relu', padding='same'),
layers.Flatten() ,
layers.Dense(10, activation='softmax')
 ])
<choose_model_class>
for index, row in train_df.iterrows() :
label,digit = row[0], row[1:]
filepath = train_dir/str(label)
 filename = f"{index}.jpg"
digit = digit.values
digit = digit.reshape(28,28)
 digit = digit.astype(np.uint8)
 img = Image.fromarray(digit)
 img.save(filepath/filename )
CNN.compile(
 optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'] 
)
 CNN.optimizer.lr=0.001<train_model>
for index, digit in test_df.iterrows() :
filepath = test_dir
filename = f"{index}.jpg"
digit = digit.values
digit = digit.reshape(28,28)
 digit = digit.astype(np.uint8)
 img = Image.fromarray(digit)
 img.save(filepath/filename )
history_CNN = CNN.fit(
 x=X_train,
y=y_train,
validation_data=(X_val, y_val),
batch_size=batch_size,
epochs=epochs,
shuffle= True,
verbose=2,
callbacks=[early_stopping, lrr]
)<choose_model_class>
tfms = get_transforms(do_flip=False)
 data = ImageDataBunch.from_folder(
 path = train_dir,
test = test_dir,
valid_pct = 0.2,
bs = 32,
size = 28,
ds_tfms = tfms,
num_workers = 0
).normalize(imagenet_stats)
 print(data)
 print(data.classes)
 data.show_batch(figsize=(5,5))
CNN_2 = keras.Sequential([
layers.Conv2D(32, kernel_size=(3,3), activation='relu', padding='same', input_shape=(28,28,1)) ,
layers.MaxPooling2D(pool_size=(2, 2)) ,
layers.Conv2D(64, kernel_size=(3,3), activation='relu', padding='same'),
layers.MaxPooling2D(pool_size=(2, 2)) ,
layers.Conv2D(64, kernel_size=(3,3), activation='relu', padding='same'),
layers.MaxPooling2D(pool_size=(2, 2)) ,
layers.Flatten() ,
layers.Dropout(0.4),
layers.Dense(10, activation='softmax')
 ] )<choose_model_class>
learn = cnn_learner(data, models.resnet18, metrics=accuracy, model_dir="/tmp/models" )
CNN_2.compile(
 optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'] 
)
 CNN.optimizer.lr=0.001<train_model>
learn.fit_one_cycle(4 )
history_CNN_2 = CNN_2.fit(
 x=X_train,
y=y_train,
validation_data=(X_val, y_val),
batch_size=batch_size,
epochs=epochs,
shuffle= True,
verbose=2,
callbacks=[early_stopping, lrr]
)<train_model>
interp = ClassificationInterpretation.from_learner(learn )
datagen = ImageDataGenerator(
 featurewise_center=False,
featurewise_std_normalization=False,
rotation_range=10,
zoom_range=0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False,
validation_split=0.2 
)
 datagen.fit(X_train)
<define_variables>
learn.unfreeze()
data_size = len(X_train)
 steps_per_epoch = int(data_size / batch_size)
 print(steps_per_epoch )<train_model>
learn.fit_one_cycle(1 )
history_CNN_2_datagen = CNN_2.fit(
 datagen.flow(X_train,y_train,batch_size=batch_size),
epochs=epochs,
shuffle=True,
validation_data=(X_val,y_val),
verbose=2,
callbacks=[early_stopping, lrr],
steps_per_epoch=steps_per_epoch
)<predict_on_test>
learn.lr_find()
val_predictions = CNN_2.predict(X_val)
 y_pred = val_predictions.argmax(axis=-1 )<predict_on_test>
learn.unfreeze()
learn.fit_one_cycle(4, max_lr=slice(1e-6,1e-4))
predictions = CNN_2.predict(test_set)
 results = predictions.argmax(axis=-1 )<save_to_csv>
class_score, y = learn.get_preds(DatasetType.Test)
 class_score = np.argmax(class_score, axis=1 )
result = pd.DataFrame()
result['ImageId'] = list(range(1,28001))
result['Label'] = results
result.to_csv("output.csv", index = False)
<import_modules>
sample_submission = pd.read_csv(input_dir/"sample_submission.csv")
 display(sample_submission.head(2))
display(sample_submission.tail(2))
import pickle
import numpy as np
import pandas as pd
import tensorflow as tf<define_variables>
ImageId = []
for path in os.listdir(test_dir):
path = path[:-4]
path = int(path)
 path = path + 1
ImageId.append(path )
MIXED_PRECISION = False
XLA_ACCELERATE = True<set_options>
submission = pd.DataFrame({
"ImageId": ImageId,
"Label": class_score
})
 submission.sort_values(by=["ImageId"], inplace = True)
 submission.to_csv("submission.csv", index=False)
 submission[:10]
if MIXED_PRECISION:
policy = tf.keras.mixed_precision.experimental.Policy("mixed_float16")
 tf.keras.mixed_precision.experimental.set_policy(policy)
 tf.config.optimizer.set_jit(XLA_ACCELERATE )<choose_model_class>
learn = cnn_learner(data, models.resnet34, metrics=accuracy, model_dir="/tmp/models" )
def build_callbacks(
 checkpoint_path,
factor=0.1,
mode="auto",
monitor="val_loss",
patience_for_es=0,
patience_for_rlop=10,
save_best_only=False,
verbose=0, 
):
callbacks = []
if checkpoint_path is not None:
model_checkpoint = tf.keras.callbacks.ModelCheckpoint(
 checkpoint_path,
mode=mode,
monitor=monitor,
save_best_only=save_best_only,
verbose=verbose, 
)
 callbacks.append(model_checkpoint)
 if patience_for_es is not None:
early_stopping = tf.keras.callbacks.EarlyStopping(
 mode=mode, monitor=monitor, patience=patience_for_es, verbose=verbose 
)
 callbacks.append(early_stopping)
 if patience_for_rlop is not None:
reduce_lr_on_plateau = tf.keras.callbacks.ReduceLROnPlateau(
 factor=factor,
monitor=monitor,
mode=mode,
patience=patience_for_rlop,
verbose=verbose, 
)
 callbacks.append(reduce_lr_on_plateau)
 return callbacks<choose_model_class>
learn.lr_find()
def build_model(
 input_dim,
output_dim,
activation="relu",
kernel_initializer="glorot_uniform",
label_smoothing=0.0,
momentum=0.99,
n_layers=3,
n_units="auto",
optimizer_params=None,
rate=0.0, 
):
if n_units == "auto":
n_units = 0.5 *(input_dim + output_dim)
 inputs = tf.keras.layers.Input(shape=input_dim)
 x = inputs
for i in range(n_layers - 2):
x = tf.keras.layers.Dense(n_units, kernel_initializer=kernel_initializer )(x)
 x = tf.keras.layers.BatchNormalization(momentum=momentum )(x)
 x = tf.keras.layers.Activation(activation )(x)
 x = tf.keras.layers.Dropout(rate )(x)
 x = tf.keras.layers.Dense(output_dim )(x)
 outputs = tf.keras.layers.Activation("sigmoid" )(x)
 model = ModelWithCutmix(inputs=inputs, outputs=outputs)
 if optimizer_params is None:
optimizer_params = {}
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing)
 optimizer = tf.keras.optimizers.Adam(**optimizer_params)
 model.compile(loss=loss, optimizer=optimizer)
 return model<compute_test_metric>
learn.fit_one_cycle(5 )
def score(Y, Y_pred, eps=1e-15, label_smoothing=0.0):
Y = np.asarray(Y)
 Y = np.ravel(Y)
 if label_smoothing > 0.0:
Y = Y *(1.0 - label_smoothing)+ 0.5 * label_smoothing
Y_pred = np.asarray(Y_pred)
 Y_pred = np.ravel(Y_pred)
 Y_pred = np.clip(Y_pred, eps, 1.0 - eps)
 return -np.mean(Y * np.log(Y_pred)+(1.0 - Y)* np.log(1.0 - Y_pred))<set_options>
learn.save('stage1' )
def set_seed(seed=0):
os.environ["PYTHONHASHSEED"] = str(seed)
 rn.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 graph = tf.compat.v1.get_default_graph()
session_conf = tf.compat.v1.ConfigProto(
 inter_op_parallelism_threads=1, intra_op_parallelism_threads=1 
)
 sess = tf.compat.v1.Session(graph=graph, config=session_conf)
 tf.compat.v1.keras.backend.set_session(sess )<randomize_order>
learn.load('stage1' )
def cutmix_for_tabular(x, y=None, alpha=1.0, p=1.0, random_state=None):
n, d = x.shape
if n is not None and rn.random() < p:
random_state = check_random_state(random_state)
 l = random_state.beta(alpha, alpha)
 mask = random_state.choice([False, True], size=d, p=[l, 1.0 - l])
 mask = np.where(mask)[0]
shuffle = random_state.choice(n, n, replace=False)
 x[:, mask] = x[shuffle, mask]
if y is not None:
y = l * y +(1.0 - l)* y[shuffle]
return x, y
class ModelWithCutmix(tf.keras.models.Model):
def __init__(self, *args, alpha=1.0, p=1.0, **kwargs):
super().__init__(*args, **kwargs)
 self.alpha = alpha
self.p = p
def train_step(self, data):
data = cutmix_for_tabular(*data, alpha=self.alpha, p=self.p)
 return super().train_step(data )<features_selection>
learn.lr_find()
class MultilabelStratifiedGroupKFold(_BaseKFold):
def __init__(self, n_splits=5, random_state=None, shuffle=False):
super().__init__(n_splits=n_splits, random_state=random_state, shuffle=shuffle)
 def _iter_test_indices(self, X=None, y=None, groups=None):
cv = MultilabelStratifiedKFold(
 n_splits=self.n_splits,
random_state=self.random_state,
shuffle=self.shuffle, 
)
 value_counts = groups.value_counts()
regluar_indices = value_counts.loc[value_counts <= 18].index.sort_values()
irregluar_indices = value_counts.loc[value_counts > 18].index.sort_values()
group_to_fold = {}
tmp = y.groupby(groups ).mean().loc[regluar_indices]
for fold,(_, test)in enumerate(cv.split(tmp, tmp)) :
group_to_fold.update({group: fold for group in tmp.index[test]})
 sample_to_fold = {}
tmp = y.loc[groups.isin(irregluar_indices)]
for fold,(_, test)in enumerate(cv.split(tmp, tmp)) :
sample_to_fold.update({sample: fold for sample in tmp.index[test]})
 folds = groups.map(group_to_fold)
 is_na = folds.isna()
folds[is_na] = folds[is_na].index.map(sample_to_fold ).values
for i in range(self.n_splits):
yield np.where(folds == i)[0]<load_from_csv>
learn.unfreeze()
learn.fit_one_cycle(10, max_lr=slice(1e-6,1e-4))
dtype = {"cp_type": "category", "cp_dose": "category"}
index_col = "sig_id"
train_features = pd.read_csv(
 ".. /input/lish-moa/train_features.csv", dtype=dtype, index_col=index_col 
)
 test_features = pd.read_csv(
 ".. /input/lish-moa/test_features.csv", dtype=dtype, index_col=index_col 
)
 X = train_features.select_dtypes("number")
 X_test = test_features.select_dtypes("number")
 Y = pd.read_csv(".. /input/lish-moa/train_targets_scored.csv", index_col=index_col)
 groups = pd.read_csv(
 ".. /input/lish-moa/train_drug.csv", index_col=index_col, squeeze=True 
)
 train_size, n_features = X.shape
test_size, _ = X_test.shape
_, n_classes = Y.shape<train_model>
learn.lr_find()
<train_on_grid><EOS> 
learn.unfreeze()
learn.fit_one_cycle(6 )