Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<predict_on_test>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
predict4 = lgbm.predict(df_test)
 predict3 = xgb.predict(df_test)
 predict_y =(predict3*0.45 + predict4 * 0.55 )<save_to_csv>
mnist_train_complete = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 mnist_test_complete = pd.read_csv("/kaggle/input/digit-recognizer/test.csv")
 mnist_train_complete.head(5 )
submission = pd.DataFrame({
"Id": test["Id"],
"SalePrice": predict_y
})
 submission.to_csv('submission.csv', index=False )<import_modules>
train_y = mnist_train_complete.iloc[:, 0].values.astype('int32')
 train_x = mnist_train_complete.iloc[:, 1:].values.astype('float32')
 test_x = mnist_test_complete.values.astype('float32')
 train_x = train_x.reshape(train_x.shape[0], 28, 28)
 test_x = test_x.reshape(test_x.shape[0], 28, 28 )
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub<load_from_url>
train_x = train_x.astype('float32')/np.max(train_x)
 test_x = test_x.astype('float32')/np.max(test_x)
 mean = np.std(train_x)
 train_x -= mean
mean = np.std(test_x)
 test_x -= mean
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py<import_modules>
splitted_train_X, splitted_test_X, splitted_train_y, splitted_test_y = train_test_split(train_x, train_y, test_size=0.2, random_state=81)
 ohe_splitted_train_y = tf_utils.to_categorical(splitted_train_y, 10)
 ohe_splitted_test_y = tf_utils.to_categorical(splitted_test_y, 10)
 print('One-hot labels:')
 print(splitted_train_y[:10] )
import tokenization<categorify>
model_sol_1 = tf.keras.models.Sequential()
model_sol_1.add(tf.keras.layers.Flatten(input_shape = splitted_train_X.shape[1:]))
model_sol_1.add(tf.keras.layers.Dense(512, activation='relu'))
model_sol_1.add(tf.keras.layers.Dropout(0.2))
model_sol_1.add(tf.keras.layers.Dense(512, activation='relu'))
model_sol_1.add(tf.keras.layers.Dropout(0.2))
model_sol_1.add(tf.keras.layers.Dense(10, activation='softmax'))
model_sol_1.summary()
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )<choose_model_class>
model_sol_1.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'] )
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 return model<load_from_csv>
score = model_sol_1.evaluate(splitted_test_X, ohe_splitted_test_y, verbose=0)
 accuracy = 100 * score[1]
print('Test accuracy: %4f%%' % accuracy )
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )<choose_model_class>
checkpointer = ModelCheckpoint(filepath='mnist.model.best.hdf5', verbose=1, save_best_only=True)
 hist_sol_1 = model_sol_1.fit(splitted_train_X, ohe_splitted_train_y, batch_size=128, epochs=10,
validation_split=0.2, callbacks=[checkpointer],
verbose=2, shuffle=True )
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )<feature_engineering>
model_sol_1.load_weights('mnist.model.best.hdf5')
 score = model_sol_1.evaluate(splitted_test_X, ohe_splitted_test_y, verbose=0)
 accuracy = 100 * score[1]
print('Test accuracy: %.4f%%' % accuracy )
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )<categorify>
predictions = model_sol_1.predict(test_x)
 predictions = [ np.argmax(x)for x in predictions ]
train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values<train_model>
submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
 submission.drop('Label', axis=1, inplace=True)
 submission['Label'] = predictions
submission.to_csv('submission1.csv', index=False )
callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=20,
batch_size=8,
callbacks=[callback] 
)
 model.save('model_bert.h5' )<predict_on_test>
extended_splitted_train_X = splitted_train_X[..., tf.newaxis]
extended_splitted_test_X = splitted_test_X[..., tf.newaxis]
extended_splitted_test_X.shape
prediction= model.predict(test_input )<save_to_csv>
model_sol_2 = Sequential()
model_sol_2.add(Conv2D(filters=32, kernel_size=3, padding='same', activation='relu', input_shape=extended_splitted_train_X.shape[1:]))
model_sol_2.add(MaxPooling2D(pool_size=2))
model_sol_2.add(Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))
model_sol_2.add(MaxPooling2D(pool_size=2))
model_sol_2.add(Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))
model_sol_2.add(MaxPooling2D(pool_size=2))
model_sol_2.add(Flatten())
 model_sol_2.add(Dense(64))
model_sol_2.add(Activation('relu'))
model_sol_2.add(Dropout(0.2))
model_sol_2.add(Dense(10, activation='softmax'))
submission['target'] = prediction.round().astype(int)
 submission.to_csv('submission.csv', index=False )<train_model>
model_sol_2.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'] )
train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=2,
batch_size=8 
)
 model.save('model_bert.h5' )<set_options>
score = model_sol_2.evaluate(extended_splitted_test_X, ohe_splitted_test_y, verbose=0)
 accuracy = 100 * score[1]
print('Test accuracy: %4f%%' % accuracy )
py.init_notebook_mode(connected=True)
 pio.templates.default = "plotly_dark"
pd.set_option('max_columns', 50)
<install_modules>
checkpointer = ModelCheckpoint(filepath='mnist.model.best.hdf5', verbose=1, save_best_only=True)
 hist_sol_2 = model_sol_2.fit(extended_splitted_train_X, ohe_splitted_train_y, batch_size=128,
epochs=10, callbacks=[checkpointer],
verbose=2, validation_data=(extended_splitted_test_X, ohe_splitted_test_y), shuffle=True )
!pip install detectron2 -f \
https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.7/index.html
!pip install pytorch-pfn-extras timm<load_pretrained>
model_sol_2.load_weights('mnist.model.best.hdf5')
 score = model_sol_2.evaluate(extended_splitted_test_X, ohe_splitted_test_y, verbose=0)
 accuracy = 100 * score[1]
print('Test accuracy: %.4f%%' % accuracy )
def save_yaml(filepath: str, content: Any, width: int = 120):
with open(filepath, "w")as f:
yaml.dump(content, f, width=width )<init_hyperparams>
extended_test_x = test_x[..., tf.newaxis]
predictions = model_sol_2.predict(extended_test_x)
 predictions = [ np.argmax(x)for x in predictions ]
submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
 submission.drop('Label', axis=1, inplace=True)
 submission['Label'] = predictions
submission.to_csv('submission2.csv', index=False )
@dataclass
class Flags:
debug: bool = True
outdir: str = "results/det"
device: str = "cuda:0"
imgdir_name: str = "vinbigdata-chest-xray-resized-png-256x256"
seed: int = 111
target_fold: int = 0
label_smoothing: float = 0.0
model_name: str = "resnet18"
model_mode: str = "normal"
epoch: int = 20
batchsize: int = 8
valid_batchsize: int = 16
num_workers: int = 4
snapshot_freq: int = 5
ema_decay: float = 0.999
scheduler_type: str = ""
scheduler_kwargs: Dict[str, Any] = field(default_factory=lambda: {})
 scheduler_trigger: List[Union[int, str]] = field(default_factory=lambda: [1, "iteration"])
 aug_kwargs: Dict[str, Dict[str, Any]] = field(default_factory=lambda: {})
 mixup_prob: float = -1.0
def update(self, param_dict: Dict)-> "Flags":
for key, value in param_dict.items() :
if not hasattr(self, key):
raise ValueError(f"[ERROR] Unexpected key for flag = {key}")
 setattr(self, key, value)
 return self<init_hyperparams>
image_augmentator = ImageDataGenerator(
 rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.2,
zoom_range=0.1,
fill_mode='nearest')
 batch_size = 32
train_batches = image_augmentator.flow(extended_splitted_train_X, ohe_splitted_train_y, batch_size=batch_size)
 val_batches = image_augmentator.flow(extended_splitted_test_X, ohe_splitted_test_y, batch_size=batch_size )
flags_dict = {
"debug": False,
"outdir": "results/tmp_debug",
"imgdir_name": "vinbigdata-chest-xray-resized-png-256x256",
"model_name": "resnet18",
"num_workers": 4,
"epoch": 15,
"batchsize": 8,
"scheduler_type": "CosineAnnealingWarmRestarts",
"scheduler_kwargs": {"T_0": 28125},
"scheduler_trigger": [1, "iteration"],
"aug_kwargs": {
"HorizontalFlip": {"p": 0.5},
"ShiftScaleRotate": {"scale_limit": 0.15, "rotate_limit": 10, "p": 0.5},
"RandomBrightnessContrast": {"p": 0.5},
"CoarseDropout": {"max_holes": 8, "max_height": 25, "max_width": 25, "p": 0.5},
"Blur": {"blur_limit": [3, 7], "p": 0.5},
"Downscale": {"scale_min": 0.25, "scale_max": 0.9, "p": 0.3},
"RandomGamma": {"gamma_limit": [80, 120], "p": 0.6},
}
}<load_from_csv>
model_sol_3 = Sequential()
model_sol_3.add(Conv2D(filters=16, kernel_size=3, padding='same', activation='relu', input_shape=extended_splitted_train_X.shape[1:]))
model_sol_3.add(Conv2D(filters=16, kernel_size=3, padding='same', activation='relu'))
model_sol_3.add(MaxPooling2D(pool_size=2))
model_sol_3.add(Dropout(0.1))
model_sol_3.add(Conv2D(filters=16, kernel_size=3, padding='same', activation='relu'))
model_sol_3.add(MaxPooling2D(pool_size=2))
model_sol_3.add(Flatten())
 model_sol_3.add(Dense(64))
model_sol_3.add(Activation('relu'))
model_sol_3.add(Dropout(0.2))
model_sol_3.add(Dense(10, activation='softmax'))
print("torch", torch.__version__)
 flags = Flags().update(flags_dict)
 print("flags", flags)
 debug = flags.debug
outdir = Path(flags.outdir)
 os.makedirs(str(outdir), exist_ok=True)
 flags_dict = dataclasses.asdict(flags)
 save_yaml(str(outdir / "flags.yaml"), flags_dict)
 inputdir = Path("/kaggle/input")
 datadir = inputdir / "vinbigdata-chest-xray-abnormalities-detection"
imgdir = inputdir / flags.imgdir_name
train = pd.read_csv(datadir / "train.csv")
<groupby>
model_sol_3.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'] )
is_normal_df = train.groupby("image_id")["class_id"].agg(lambda s:(s == 14 ).sum() ).reset_index().rename({"class_id": "num_normal_annotations"}, axis=1)
 is_normal_df.head()<categorify>
checkpointer = ModelCheckpoint(filepath='mnist.model.best.hdf5', verbose=1, save_best_only=True)
 hist_sol_3 = model_sol_3.fit_generator(generator=train_batches, steps_per_epoch =extended_splitted_train_X.shape[0] // batch_size,
epochs=32, callbacks=[checkpointer],
validation_data=val_batches, validation_steps=extended_splitted_test_X.shape[0] // batch_size,
verbose=2 )
num_normal_anno_counts_df = num_normal_anno_counts.reset_index()
num_normal_anno_counts_df["name"] = num_normal_anno_counts_df["index"].map({0: "Abnormal", 3: "Normal"})
 num_normal_anno_counts_df<define_variables>
model_sol_3.load_weights('mnist.model.best.hdf5')
 score = model_sol_3.evaluate(extended_splitted_test_X, ohe_splitted_test_y, verbose=0)
 accuracy = 100 * score[1]
print('Test accuracy: %.4f%%' % accuracy )
def get_vinbigdata_dicts(
 imgdir: Path,
train_df: pd.DataFrame,
train_data_type: str = "original",
use_cache: bool = True,
debug: bool = True,
target_indices: Optional[np.ndarray] = None, 
):
debug_str = f"_debug{int(debug)}"
train_data_type_str = f"_{train_data_type}"
cache_path = Path(".")/ f"dataset_dicts_cache{train_data_type_str}{debug_str}.pkl"
if not use_cache or not cache_path.exists() :
print("Creating data...")
 train_meta = pd.read_csv(imgdir / "train_meta.csv")
 if debug:
train_meta = train_meta.iloc[:500]
image_id = train_meta.loc[0, "image_id"]
image_path = str(imgdir / "train" / f"{image_id}.png")
 image = cv2.imread(image_path)
 resized_height, resized_width, ch = image.shape
print(f"image shape: {image.shape}")
 dataset_dicts = []
for index, train_meta_row in tqdm(train_meta.iterrows() , total=len(train_meta)) :
record = {}
image_id, height, width = train_meta_row.values
filename = str(imgdir / "train" / f"{image_id}.png")
 record["file_name"] = filename
record["image_id"] = image_id
record["height"] = resized_height
record["width"] = resized_width
objs = []
for index2, row in train_df.query("image_id == @image_id" ).iterrows() :
class_id = row["class_id"]
if class_id == 14:
pass
else:
h_ratio = resized_height / height
w_ratio = resized_width / width
bbox_resized = [
int(row["x_min"])* w_ratio,
int(row["y_min"])* h_ratio,
int(row["x_max"])* w_ratio,
int(row["y_max"])* h_ratio,
]
obj = {
"bbox": bbox_resized,
"bbox_mode": BoxMode.XYXY_ABS,
"category_id": class_id,
}
objs.append(obj)
 record["annotations"] = objs
dataset_dicts.append(record)
 with open(cache_path, mode="wb")as f:
pickle.dump(dataset_dicts, f)
 print(f"Load from cache {cache_path}")
 with open(cache_path, mode="rb")as f:
dataset_dicts = pickle.load(f)
 if target_indices is not None:
dataset_dicts = [dataset_dicts[i] for i in target_indices]
return dataset_dicts
def get_vinbigdata_dicts_test(
 imgdir: Path, test_meta: pd.DataFrame, use_cache: bool = True, debug: bool = True, 
):
debug_str = f"_debug{int(debug)}"
cache_path = Path(".")/ f"dataset_dicts_cache_test{debug_str}.pkl"
if not use_cache or not cache_path.exists() :
print("Creating data...")
 if debug:
test_meta = test_meta.iloc[:500]
image_id = test_meta.loc[0, "image_id"]
image_path = str(imgdir / "test" / f"{image_id}.png")
 image = cv2.imread(image_path)
 resized_height, resized_width, ch = image.shape
print(f"image shape: {image.shape}")
 dataset_dicts = []
for index, test_meta_row in tqdm(test_meta.iterrows() , total=len(test_meta)) :
record = {}
image_id, height, width = test_meta_row.values
filename = str(imgdir / "test" / f"{image_id}.png")
 record["file_name"] = filename
record["image_id"] = image_id
record["height"] = resized_height
record["width"] = resized_width
dataset_dicts.append(record)
 with open(cache_path, mode="wb")as f:
pickle.dump(dataset_dicts, f)
 print(f"Load from cache {cache_path}")
 with open(cache_path, mode="rb")as f:
dataset_dicts = pickle.load(f)
 return dataset_dicts 
<categorify>
extended_test_x = test_x[..., tf.newaxis]
predictions = model_sol_3.predict(extended_test_x)
 predictions = [ np.argmax(x)for x in predictions ]
submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
 submission.drop('Label', axis=1, inplace=True)
 submission['Label'] = predictions
submission.to_csv('submission3.csv', index=False )
class DatasetMixin(Dataset):
def __init__(self, transform=None):
self.transform = transform
def __getitem__(self, index):

 if torch.is_tensor(index):
index = index.tolist()
if isinstance(index, slice):
current, stop, step = index.indices(len(self))
return [self.get_example_wrapper(i)for i in
six.moves.range(current, stop, step)]
elif isinstance(index, list)or isinstance(index, numpy.ndarray):
return [self.get_example_wrapper(i)for i in index]
else:
return self.get_example_wrapper(index)
 def __len__(self):

 raise NotImplementedError
def get_example_wrapper(self, i):

 example = self.get_example(i)
 if self.transform:
example = self.transform(example)
 return example
def get_example(self, i):

 raise NotImplementedError 
<categorify>
model_sol_4_1 = Sequential()
model_sol_4_1.add(Conv2D(filters=16, kernel_size=3, padding='same', activation='relu', input_shape=extended_splitted_train_X.shape[1:]))
model_sol_4_1.add(BatchNormalization())
 model_sol_4_1.add(Conv2D(filters=16, kernel_size=3, padding='same', activation='relu'))
model_sol_4_1.add(MaxPooling2D(pool_size=2))
model_sol_4_1.add(Dropout(0.1))
model_sol_4_1.add(BatchNormalization())
 model_sol_4_1.add(Conv2D(filters=16, kernel_size=3, padding='same', activation='relu'))
model_sol_4_1.add(MaxPooling2D(pool_size=2))
model_sol_4_1.add(Flatten())
 model_sol_4_1.add(BatchNormalization())
 model_sol_4_1.add(Dense(64))
model_sol_4_1.add(Activation('relu'))
model_sol_4_1.add(Dropout(0.2))
model_sol_4_1.add(BatchNormalization())
 model_sol_4_1.add(Dense(10, activation='softmax'))
class VinbigdataTwoClassDataset(DatasetMixin):
def __init__(self, dataset_dicts, image_transform=None, transform=None, train: bool = True,
mixup_prob: float = -1.0, label_smoothing: float = 0.0):
super(VinbigdataTwoClassDataset, self ).__init__(transform=transform)
 self.dataset_dicts = dataset_dicts
self.image_transform = image_transform
self.train = train
self.mixup_prob = mixup_prob
self.label_smoothing = label_smoothing
def _get_single_example(self, i):
d = self.dataset_dicts[i]
filename = d["file_name"]
img = cv2.imread(filename)
 if self.image_transform:
img = self.image_transform(img)
 img = torch.tensor(np.transpose(img,(2, 0, 1)).astype(np.float32))
if self.train:
label = int(len(d["annotations"])> 0)
 if self.label_smoothing > 0:
if label == 0:
return img, float(label)+ self.label_smoothing
else:
return img, float(label)- self.label_smoothing
else:
return img, float(label)
 else:
return img, None
def get_example(self, i):
img, label = self._get_single_example(i)
 if self.mixup_prob > 0.and np.random.uniform() < self.mixup_prob:
j = np.random.randint(0, len(self.dataset_dicts))
p = np.random.uniform()
img2, label2 = self._get_single_example(j)
 img = img * p + img2 *(1 - p)
 if self.train:
label = label * p + label2 *(1 - p)
 if self.train:
label_logit = torch.tensor([1 - label, label], dtype=torch.float32)
 return img, label_logit
else:
return img
def __len__(self):
return len(self.dataset_dicts )<create_dataframe>
model_sol_4_1.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'] )
dataset_dicts = get_vinbigdata_dicts(imgdir, train, debug=debug)
 dataset = VinbigdataTwoClassDataset(dataset_dicts )<normalization>
checkpointer = ModelCheckpoint(filepath='mnist.model.best.hdf5', verbose=1, save_best_only=True)
 hist_sol_4 = model_sol_4_1.fit_generator(generator=train_batches, steps_per_epoch =extended_splitted_train_X.shape[0] // batch_size,
epochs=32, callbacks=[checkpointer],
validation_data=val_batches, validation_steps=extended_splitted_test_X.shape[0] // batch_size,
verbose=2 )
class Transform:
def __init__(
 self, hflip_prob: float = 0.5, ssr_prob: float = 0.5, random_bc_prob: float = 0.5 
):
self.transform = A.Compose(
 [
A.HorizontalFlip(p=hflip_prob),
A.ShiftScaleRotate(
 shift_limit=0.0625, scale_limit=0.1, rotate_limit=10, p=ssr_prob 
),
A.RandomBrightnessContrast(p=random_bc_prob),
] 
)
 def __call__(self, image):
image = self.transform(image=image)["image"]
return image 
<create_dataframe>
model_sol_4_1.load_weights('mnist.model.best.hdf5')
 score = model_sol_4_1.evaluate(extended_splitted_test_X, ohe_splitted_test_y, verbose=0)
 accuracy = 100 * score[1]
print('Test accuracy: %.4f%%' % accuracy )
aug_dataset = VinbigdataTwoClassDataset(dataset_dicts, image_transform=Transform() )<categorify>
model_sol_4_2 = Sequential()
model_sol_4_2.add(Conv2D(filters=32, kernel_size=3, padding='same', activation='relu', input_shape=extended_splitted_train_X.shape[1:]))
model_sol_4_2.add(BatchNormalization())
 model_sol_4_2.add(Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))
model_sol_4_2.add(MaxPooling2D(pool_size=2))
model_sol_4_2.add(Dropout(0.1))
model_sol_4_2.add(BatchNormalization())
 model_sol_4_2.add(Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))
model_sol_4_2.add(MaxPooling2D(pool_size=2))
model_sol_4_2.add(Flatten())
 model_sol_4_2.add(BatchNormalization())
 model_sol_4_2.add(Dense(64))
model_sol_4_2.add(Activation('relu'))
model_sol_4_2.add(Dropout(0.2))
model_sol_4_2.add(BatchNormalization())
 model_sol_4_2.add(Dense(10, activation='softmax'))
class Transform:
def __init__(self, aug_kwargs: Dict):
self.transform = A.Compose(
 [getattr(A, name )(**kwargs)for name, kwargs in aug_kwargs.items() ] 
)
 def __call__(self, image):
image = self.transform(image=image)["image"]
return image<init_hyperparams>
model_sol_4_2.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'] )
class CNNFixedPredictor(nn.Module):
def __init__(self, cnn: nn.Module, num_classes: int = 2):
super(CNNFixedPredictor, self ).__init__()
self.cnn = cnn
self.lin = Linear(cnn.num_features, num_classes)
 print("cnn.num_features", cnn.num_features)
 for param in self.cnn.parameters() :
param.requires_grad = False
def forward(self, x):
feat = self.cnn(x)
 return self.lin(feat)
<choose_model_class>
checkpointer = ModelCheckpoint(filepath='mnist.model.best.hdf5', verbose=1, save_best_only=True)
 hist_sol_4 = model_sol_4_2.fit_generator(generator=train_batches, steps_per_epoch=extended_splitted_train_X.shape[0] // batch_size,
epochs=32, callbacks=[checkpointer],
validation_data=val_batches, validation_steps=extended_splitted_test_X.shape[0] // batch_size,
verbose=2 )
def build_predictor(model_name: str, model_mode: str = "normal"):
if model_mode == "normal":
return timm.create_model(model_name, pretrained=True, num_classes=2, in_chans=3)
 elif model_mode == "cnn_fixed":
timm_model = timm.create_model(model_name, pretrained=True, num_classes=0, in_chans=3)
 return CNNFixedPredictor(timm_model, num_classes=2)
 else:
raise ValueError(f"[ERROR] Unexpected value model_mode={model_mode}")
<compute_test_metric>
model_sol_4_2.load_weights('mnist.model.best.hdf5')
 score = model_sol_4_2.evaluate(extended_splitted_test_X, ohe_splitted_test_y, verbose=0)
 accuracy = 100 * score[1]
print('Test accuracy: %.4f%%' % accuracy )
def accuracy(y: torch.Tensor, t: torch.Tensor)-> torch.Tensor:

 assert y.shape[:-1] == t.shape, f"y {y.shape}, t {t.shape} is inconsistent."
pred_label = torch.max(y.detach() , dim=-1)[1]
count = t.nelement()
correct =(pred_label == t ).sum().float()
acc = correct / count
return acc
def accuracy_with_logits(y: torch.Tensor, t: torch.Tensor)-> torch.Tensor:

 assert y.shape == t.shape
gt_label = torch.max(t.detach() , dim=-1)[1]
return accuracy(y, gt_label )<compute_train_metric>
extended_test_x = test_x[..., tf.newaxis]
predictions = model_sol_4_2.predict(extended_test_x)
 predictions = [ np.argmax(x)for x in predictions ]
submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
 submission.drop('Label', axis=1, inplace=True)
 submission['Label'] = predictions
submission.to_csv('submission4.csv', index=False )
def cross_entropy_with_logits(input, target, dim=-1):
loss = torch.sum(- target * F.log_softmax(input, dim), dim)
 return loss.mean() 
<find_best_params>
train_y_sol5 = mnist_train_complete.iloc[:, 0].values.astype('int32')
 train_x_sol5 = mnist_train_complete.iloc[:, 1:].values.astype('float32')
 test_x_sol5 = mnist_test_complete.values.astype('float32')
 train_x_sol5 = train_x_sol5.reshape(train_x_sol5.shape[0], 28, 28)
 test_x_sol5 = test_x_sol5.reshape(test_x_sol5.shape[0], 28, 28)
 train_x_sol5 = train_x_sol5[..., tf.newaxis]
test_x_sol5 = test_x_sol5[..., tf.newaxis]
class Classifier(nn.Module):

 def __init__(self, predictor, lossfun=cross_entropy_with_logits):
super().__init__()
self.predictor = predictor
self.lossfun = lossfun
self.prefix = ""
def forward(self, image, targets):
outputs = self.predictor(image)
 loss = self.lossfun(outputs, targets)
 metrics = {
f"{self.prefix}loss": loss.item() ,
f"{self.prefix}acc": accuracy_with_logits(outputs, targets ).item()
}
ppe.reporting.report(metrics, self)
 return loss, metrics
def predict(self, data_loader):
pred = self.predict_proba(data_loader)
 label = torch.argmax(pred, dim=1)
 return label
def predict_proba(self, data_loader):
device: torch.device = next(self.parameters() ).device
y_list = []
self.eval()
with torch.no_grad() :
for batch in data_loader:
if isinstance(batch,(tuple, list)) :
batch = batch[0].to(device)
 else:
batch = batch.to(device)
 y = self.predictor(batch)
 y = torch.softmax(y, dim=-1)
 y_list.append(y)
 pred = torch.cat(y_list)
 return pred 
<find_best_params>
mean_px = train_x_sol5.mean().astype(np.float32)
 std_px = train_x_sol5.std().astype(np.float32)
 def standardize(x):
return(x-mean_px)/std_px
supported_models = timm.list_models()
print(f"{len(supported_models)} models are supported in timm.")
 print(supported_models )<import_modules>
s5_train_x, s5_test_x, s5_train_y, s5_test_y = train_test_split(train_x_sol5, train_y_sol5,
test_size=0.2,
random_state=81)
 ohe_s5_train_y = tf_utils.to_categorical(s5_train_y, 10)
 ohe_s5_test_y = tf_utils.to_categorical(s5_test_y, 10)
 train_batches_sol5 = image_augmentator.flow(s5_train_x, ohe_s5_train_y, batch_size=64)
 val_batches_sol5 = image_augmentator.flow(s5_test_x, ohe_s5_test_y, batch_size=64 )
class EMA(object):

 def __init__(
 self,
model: nn.Module,
decay: float,
strict: bool = True,
use_dynamic_decay: bool = True, 
):
self.decay = decay
self.model = model
self.strict = strict
self.use_dynamic_decay = use_dynamic_decay
self.logger = getLogger(__name__)
 self.n_step = 0
self.shadow = {}
self.original = {}
self._assigned = False
for name, param in model.named_parameters() :
if param.requires_grad:
self.shadow[name] = param.data.clone()
def step(self):
self.n_step += 1
if self.use_dynamic_decay:
_n_step = float(self.n_step)
 decay = min(self.decay,(1.0 + _n_step)/(10.0 + _n_step))
else:
decay = self.decay
for name, param in self.model.named_parameters() :
if param.requires_grad:
assert name in self.shadow
new_average =(1.0 - decay)* param.data + decay * self.shadow[name]
self.shadow[name] = new_average.clone()
__call__ = step
def assign(self):

 if self._assigned:
if self.strict:
raise ValueError("[ERROR] `assign` is called again before `resume`.")
 else:
self.logger.warning(
 "`assign` is called again before `resume`."
"shadow parameter is already assigned, skip." 
)
 return
for name, param in self.model.named_parameters() :
if param.requires_grad:
assert name in self.shadow
self.original[name] = param.data.clone()
param.data = self.shadow[name]
self._assigned = True
def resume(self):

 if not self._assigned:
if self.strict:
raise ValueError("[ERROR] `resume` is called before `assign`.")
 else:
self.logger.warning("`resume` is called before `assign`, skip.")
 return
for name, param in self.model.named_parameters() :
if param.requires_grad:
assert name in self.shadow
param.data = self.original[name]
self._assigned = False 
<set_options>
model_sol_5 = Sequential()
model_sol_5.add(Lambda(standardize, input_shape=(28,28,1)))
 model_sol_5.add(Conv2D(filters=32, kernel_size=3, padding='same', activation='relu',
kernel_regularizer=regularizers.l2(0.1), 
))
model_sol_5.add(BatchNormalization())
 model_sol_5.add(Conv2D(filters=32, kernel_size=3, padding='same', activation='relu' 
))
model_sol_5.add(MaxPooling2D(pool_size=2))
model_sol_5.add(BatchNormalization())
 model_sol_5.add(Conv2D(filters=32, kernel_size=3, padding='same', activation='relu' 
))
model_sol_5.add(MaxPooling2D(pool_size=2))
model_sol_5.add(Flatten())
 model_sol_5.add(BatchNormalization())
 model_sol_5.add(Dense(64))
model_sol_5.add(Activation('relu'))
model_sol_5.add(Dropout(0.2))
model_sol_5.add(BatchNormalization())
 model_sol_5.add(Dense(10, activation='softmax'))
class LRScheduler(Extension):

 trigger = 1, 'iteration'
priority = PRIORITY_READER
name = None
def __init__(self, optimizer: optim.Optimizer, scheduler_type: str, scheduler_kwargs: Mapping[str, Any])-> None:
super().__init__()
self.scheduler = getattr(optim.lr_scheduler, scheduler_type )(optimizer, **scheduler_kwargs)
 def __call__(self, manager: ExtensionsManager)-> None:
self.scheduler.step()
def state_dict(self)-> None:
return self.scheduler.state_dict()
def load_state_dict(self, to_load)-> None:
self.scheduler.load_state_dict(to_load)
<train_model>
model_sol_5.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=['accuracy'] )
def create_trainer(model, optimizer, device)-> Engine:
model.to(device)
 def update_fn(engine, batch):
model.train()
optimizer.zero_grad()
loss, metrics = model(*[elem.to(device)for elem in batch])
 loss.backward()
optimizer.step()
return metrics
trainer = Engine(update_fn)
 return trainer 
<import_modules>
checkpointer = ModelCheckpoint(filepath='mnist.model.best.hdf5', verbose=1, save_best_only=True)
 hist_sol_5 = model_sol_5.fit_generator(generator=train_batches_sol5, steps_per_epoch=s5_train_x.shape[0] // 64,
epochs=32, callbacks=[checkpointer],
validation_data=val_batches_sol5, validation_steps=s5_test_x.shape[0] // 64, verbose=2 )
import dataclasses
import os
import sys
from pathlib import Path
import numpy as np
import pandas as pd
import pytorch_pfn_extras.training.extensions as E
import torch
from ignite.engine import Events
from pytorch_pfn_extras.training import IgniteExtensionsManager
from sklearn.model_selection import StratifiedKFold
from torch import nn, optim
from torch.utils.data.dataloader import DataLoader<split>
model_sol_5.load_weights('mnist.model.best.hdf5')
 score = model_sol_5.evaluate(s5_test_x, ohe_s5_test_y, verbose=0)
 accuracy = 100 * score[1]
print('Test accuracy: %.4f%%' % accuracy )
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=flags.seed)
 y = np.array([int(len(d["annotations"])> 0)for d in dataset_dicts])
 split_inds = list(skf.split(dataset_dicts, y))
train_inds, valid_inds = split_inds[flags.target_fold]
train_dataset = VinbigdataTwoClassDataset(
 [dataset_dicts[i] for i in train_inds],
image_transform=Transform(flags.aug_kwargs),
mixup_prob=flags.mixup_prob,
label_smoothing=flags.label_smoothing, 
)
 valid_dataset = VinbigdataTwoClassDataset([dataset_dicts[i] for i in valid_inds])
<choose_model_class>
predictions = model_sol_5.predict(test_x_sol5)
 predictions = [ np.argmax(x)for x in predictions ]
submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
 submission.drop('Label', axis=1, inplace=True)
 submission['Label'] = predictions
submission.to_csv('submission5.csv', index=False )
train_loader = DataLoader(
 train_dataset,
batch_size=flags.batchsize,
num_workers=flags.num_workers,
shuffle=True,
pin_memory=True, 
)
 valid_loader = DataLoader(
 valid_dataset,
batch_size=flags.valid_batchsize,
num_workers=flags.num_workers,
shuffle=False,
pin_memory=True, 
)
 device = torch.device(flags.device)
 predictor = build_predictor(model_name=flags.model_name, model_mode=flags.model_mode)
 classifier = Classifier(predictor)
 model = classifier
optimizer = optim.Adam([param for param in model.parameters() if param.requires_grad], lr=1e-3)
 trainer = create_trainer(model, optimizer, device)
 ema = EMA(predictor, decay=flags.ema_decay)
 def eval_func(*batch):
loss, metrics = model(*[elem.to(device)for elem in batch])
 if flags.ema_decay > 0:
classifier.prefix = "ema_"
ema.assign()
loss, metrics = model(*[elem.to(device)for elem in batch])
 ema.resume()
classifier.prefix = ""
valid_evaluator = E.Evaluator(
 valid_loader, model, progress_bar=False, eval_func=eval_func, device=device 
)
 log_trigger =(1, "epoch")
 log_report = E.LogReport(trigger=log_trigger)
 extensions = [
log_report,
E.ProgressBarNotebook(update_interval=10 if debug else 100),
E.PrintReportNotebook() ,
E.FailOnNonNumber() ,
]
epoch = flags.epoch
models = {"main": model}
optimizers = {"main": optimizer}
manager = IgniteExtensionsManager(
 trainer, models, optimizers, epoch, extensions=extensions, out_dir=str(outdir), 
)
 manager.extend(valid_evaluator)
 manager.extend(
 E.snapshot_object(predictor, "predictor.pt"), trigger=(flags.snapshot_freq, "epoch")
)
 if flags.scheduler_type != "":
scheduler_type = flags.scheduler_type
print(f"using {scheduler_type} scheduler with kwargs {flags.scheduler_kwargs}")
 manager.extend(
 LRScheduler(optimizer, scheduler_type, flags.scheduler_kwargs),
trigger=flags.scheduler_trigger, 
)
 manager.extend(E.observe_lr(optimizer=optimizer), trigger=log_trigger)
 if flags.ema_decay > 0:
manager.extend(lambda manager: ema() , trigger=(1, "iteration"))
def save_ema_model(manager):
ema.assign()
torch.save(predictor.state_dict() , outdir / "predictor_ema.pt")
 ema.resume()
manager.extend(save_ema_model, trigger=(flags.snapshot_freq, "epoch"))
_ = trainer.run(train_loader, max_epochs=epoch )<save_to_csv>
model_sol_6 = Sequential()
model_sol_6.add(Lambda(standardize, input_shape=(28,28,1)))
 model_sol_6.add(Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))
model_sol_6.add(BatchNormalization())
 model_sol_6.add(Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))
model_sol_6.add(MaxPooling2D(pool_size=2))
model_sol_6.add(Dropout(0.1))
model_sol_6.add(BatchNormalization())
 model_sol_6.add(Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))
model_sol_6.add(MaxPooling2D(pool_size=2))
model_sol_6.add(Flatten())
 model_sol_6.add(BatchNormalization())
 model_sol_6.add(Dense(64))
model_sol_6.add(Activation('relu'))
model_sol_6.add(Dropout(0.2))
model_sol_6.add(BatchNormalization())
 model_sol_6.add(Dense(10, activation='softmax'))
torch.save(predictor.state_dict() , outdir / "predictor_last.pt")
 df = log_report.to_dataframe()
df.to_csv(outdir / "log.csv", index=False)
 df<save_to_csv>
model_sol_6.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=['accuracy'] )
print("Training done! Start prediction...")
 valid_pred = classifier.predict_proba(valid_loader ).cpu().numpy()
valid_pred_df = pd.DataFrame({
"image_id": [dataset_dicts[i]["image_id"] for i in valid_inds],
"class0": valid_pred[:, 0],
"class1": valid_pred[:, 1]
})
 valid_pred_df.to_csv(outdir/"valid_pred.csv", index=False)
 test_meta = pd.read_csv(inputdir / "vinbigdata-testmeta" / "test_meta.csv")
 dataset_dicts_test = get_vinbigdata_dicts_test(imgdir, test_meta, debug=debug)
 test_dataset = VinbigdataTwoClassDataset(dataset_dicts_test, train=False)
 test_loader = DataLoader(
 test_dataset,
batch_size=flags.valid_batchsize,
num_workers=flags.num_workers,
shuffle=False,
pin_memory=True, 
)
 test_pred = classifier.predict_proba(test_loader ).cpu().numpy()
test_pred_df = pd.DataFrame({
"image_id": [d["image_id"] for d in dataset_dicts_test],
"class0": test_pred[:, 0],
"class1": test_pred[:, 1]
})
 test_pred_df.to_csv(outdir/"test_pred.csv", index=False )<load_from_csv>
checkpointer = ModelCheckpoint(filepath='mnist.model.best.hdf5', verbose=1, save_best_only=True)
 hist_sol_6 = model_sol_6.fit_generator(generator=train_batches_sol5, steps_per_epoch=s5_train_x.shape[0] // 64,
epochs=32, callbacks=[checkpointer],
validation_data=val_batches_sol5, validation_steps=s5_test_x.shape[0] // 64, verbose=2 )
pred_2class = pd.read_csv(inputdir/"vinbigdata2classpred/test_pred.csv")
 low_threshold = 0.0
high_threshold = 0.976
pred_2class<load_from_csv>
model_sol_6.load_weights('mnist.model.best.hdf5')
 score = model_sol_6.evaluate(s5_test_x, ohe_s5_test_y, verbose=0)
 accuracy = 100 * score[1]
print('Test accuracy: %.4f%%' % accuracy )
NORMAL = "14 1 0 0 1 1"
pred_det_df = pd.read_csv(inputdir/"vinbigdata-detectron2-prediction/results/20210125_all_alb_aug_512_cos/submission.csv")
 n_normal_before = len(pred_det_df.query("PredictionString == @NORMAL"))
merged_df = pd.merge(pred_det_df, pred_2class, on="image_id", how="left")
 if "target" in merged_df.columns:
merged_df["class0"] = 1 - merged_df["target"]
c0, c1, c2 = 0, 0, 0
for i in range(len(merged_df)) :
p0 = merged_df.loc[i, "class0"]
if p0 < low_threshold:
c0 += 1
elif low_threshold <= p0 and p0 < high_threshold:
merged_df.loc[i, "PredictionString"] += f" 14 {p0} 0 0 1 1"
c1 += 1
else:
merged_df.loc[i, "PredictionString"] = NORMAL
c2 += 1
n_normal_after = len(merged_df.query("PredictionString == @NORMAL"))
print(
 f"n_normal: {n_normal_before} -> {n_normal_after} with threshold {low_threshold} & {high_threshold}" 
)
 print(f"Keep {c0} Add {c1} Replace {c2}")
 submission_filepath = str(outdir / "submission.csv")
 submission_df = merged_df[["image_id", "PredictionString"]]
submission_df.to_csv(submission_filepath, index=False)
 print(f"Saved to {submission_filepath}" )<load_from_csv>
model_sol_6.optimizer.lerning_rate=0.01
gen = ImageDataGenerator()
batches = gen.flow(train_x_sol5, tf_utils.to_categorical(train_y_sol5, 10), batch_size=64)
 hist_sol_6 = model_sol_6.fit_generator(generator=batches, steps_per_epoch=train_x_sol5.shape[0] // 64,
epochs=50, verbose=2)
train = pd.read_csv("/kaggle/input/titanic/train.csv")
 test = pd.read_csv("/kaggle/input/titanic/test.csv")
 train<load_from_csv>
predictions = model_sol_6.predict(test_x_sol5)
 predictions = [ np.argmax(x)for x in predictions ]
submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
 submission.drop('Label', axis=1, inplace=True)
 submission['Label'] = predictions
submission.to_csv('submission6.csv', index=False )
submission=pd.read_csv("/kaggle/input/titanic/gender_submission.csv")
 print(train.columns)
 print(test.shape)
 train.drop(['Name', 'Ticket', 'Fare', 'Embarked'], axis=1, inplace=True)
 test.drop(['Name', 'Ticket', 'Fare', 'Embarked'], axis=1, inplace=True)
 train['Sex'].replace({'male':0, 'female':1}, inplace=True)
 test['Sex'].replace({'male':0, 'female':1}, inplace=True)
 train.drop(['Cabin'], axis=1,inplace=True)
 test.drop(['Cabin'], axis=1,inplace=True )<prepare_x_and_y>
os.remove('submission1.csv')
 os.remove('submission2.csv')
 os.remove('submission3.csv')
 os.remove('submission4.csv')
 os.remove('submission5.csv')
 os.remove('submission6.csv' )
X = train.drop(['Survived', 'PassengerId'], axis=1)
 y = train['Survived']
X_test = test.drop(['PassengerId'], axis=1)
<train_model>
final_train_x = train_x[..., tf.newaxis]
final_ohe_train_y = tf_utils.to_categorical(train_y, 10)
 final_train_batches = image_augmentator.flow(final_train_x, final_ohe_train_y, batch_size=64)
 final_model = Sequential()
final_model.add(Conv2D(filters=32, kernel_size=3, padding='same', activation='relu', input_shape=extended_splitted_train_X.shape[1:]))
final_model.add(BatchNormalization())
 final_model.add(Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))
final_model.add(MaxPooling2D(pool_size=2))
final_model.add(Dropout(0.1))
final_model.add(BatchNormalization())
 final_model.add(Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))
final_model.add(MaxPooling2D(pool_size=2))
final_model.add(Flatten())
 final_model.add(BatchNormalization())
 final_model.add(Dense(64))
final_model.add(Activation('relu'))
final_model.add(Dropout(0.2))
final_model.add(BatchNormalization())
 final_model.add(Dense(10, activation='softmax'))
final_model.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=['accuracy'])
 final_model.fit_generator(generator=final_train_batches, steps_per_epoch=final_train_batches.n,
epochs=1, verbose=1 )
<save_to_csv><EOS> 
predictions = final_model.predict(test_x[..., tf.newaxis])
 predictions = [ np.argmax(x)for x in predictions ]
submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
 submission.drop('Label', axis=1, inplace=True)
 submission['Label'] = predictions
submission.to_csv('submission.csv', index=False )