Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<save_to_csv>
warnings.filterwarnings("ignore" )
test_df.to_csv('submission.csv', index=False )<import_modules>
training_folder = ".. /input/vinbigdata-chest-xray-abnormalities-detection/train/"
df = pd.read_csv(".. /input/vinbigdata-chest-xray-abnormalities-detection/train.csv")
 df = df.query("class_id<14")
 df = df.query("rad_id=='R9'" )
fastai.__version__<define_variables>
df["several_issues"] = df.duplicated(subset=['image_id'])
 df["box_size"] = [(row.y_max-row.y_min)*(row.x_max-row.x_min)for idx, row in df.iterrows() ]
path = Path('.. /input/imet-2019-fgvc6/' )<load_pretrained>
df.groupby("class_id")["box_size"].mean()
Path('models' ).mkdir(exist_ok=True)
 !cp '.. /input/densenet201/densenet201.pth' 'models/'
def load_url(*args, **kwargs):
model_dir = Path('models')
 filename = 'densenet201.pth'
if not(model_dir/filename ).is_file() : raise FileNotFoundError
return torch.load(model_dir/filename)
 model_zoo.load_url = load_url<load_from_csv>
df.groupby("class_id" ).image_id.count()
train_df = pd.read_csv(path/'train.csv')
 train_df.head()<load_from_csv>
selected_classes = [0,3,5,7,10]
category_list = ["Aortic enlargement", "Cardiomegaly", "ILD", "Lung Opacity", "Pleural effusion"]
filtered_df = df.query("class_id in @selected_classes" )
labels_df = pd.read_csv(path/'labels.csv')
 labels_df.head()<load_from_csv>
selected_classes_dict = {"0":0,"3":1,"5":2,"7":3,"10":4}
filtered_df["reclass_id"] = [selected_classes_dict[str(row.class_id)] for idx, row in filtered_df.iterrows() ]
test_df = pd.read_csv(path/'sample_submission.csv')
 test_df.head()<categorify>
def get_mask(img_dimensions, x_min, y_min, x_max, y_max):
img_height, img_width = img_dimensions
img_mask = np.full(( img_height,img_width),0)
 img_mask[y_min:y_max,x_min:x_max] = 255
return img_mask.astype(np.float32)
 def rle_encoding(x):
dots = np.where(x.T.flatten() == 255)[0]
run_lengths = []
prev = -2
for b in dots:
if(b>prev+1): run_lengths.extend(( b + 1, 0))
run_lengths[-1] += 1
prev = b
return ' '.join([str(x)for x in run_lengths] )
train, test = [ImageList.from_df(df, path=path, cols='id', folder=folder, suffix='.png')
 for df, folder in zip([train_df, test_df], ['train', 'test'])]
data =(train.split_by_rand_pct(0.2, seed=42)
.label_from_df(cols='attribute_ids', label_delim=' ')
.add_test(test)
.transform(get_transforms() , size=128)
.databunch(path=Path('.'), bs=64 ).normalize() )<define_variables>
def read_xray(path, voi_lut = True, fix_monochrome = True):
dicom = pydicom.read_file(path)
 if voi_lut:
data = apply_voi_lut(dicom.pixel_array, dicom)
 else:
data = dicom.pixel_array
if fix_monochrome and dicom.PhotometricInterpretation == "MONOCHROME1":
data = np.amax(data)- data
data = data - np.min(data)
 data = data / np.max(data)
 data =(data * 255 ).astype(np.uint8)
 return data
data.show_batch()<normalization>
filtered_df.groupby("class_id" ).image_id.count()
class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, input, target):
if not(target.size() == input.size()):
raise ValueError("Target size({})must be the same as input size({})" 
.format(target.size() , input.size()))
max_val =(-input ).clamp(min=0)
 loss = input - input * target + max_val + \ 
(( -max_val ).exp() +(-input - max_val ).exp() ).log()
invprobs = F.logsigmoid(-input *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
return loss.sum(dim=1 ).mean()
class FbetaLoss(nn.Module):
def __init__(self, beta=1):
super(FbetaLoss, self ).__init__()
self.small_value = 1e-6
self.beta = beta
def forward(self, logits, labels):
beta = self.beta
batch_size = logits.size() [0]
p = F.sigmoid(logits)
 l = labels
num_pos = torch.sum(p, 1)+ self.small_value
num_pos_hat = torch.sum(l, 1)+ self.small_value
tp = torch.sum(l * p, 1)
 precise = tp / num_pos
recall = tp / num_pos_hat
fs =(1 + beta * beta)* precise * recall /(beta * beta * precise + recall + self.small_value)
 loss = fs.sum() / batch_size
return 1 - loss
class CombineLoss(nn.Module):
def __init__(self):
super(CombineLoss, self ).__init__()
self.fbeta_loss = FbetaLoss(beta=2)
 self.focal_loss = FocalLoss()
def forward(self, logits, labels):
loss_beta = self.fbeta_loss(logits, labels)
 loss_focal = self.focal_loss(logits, labels)
 return 0.5 * loss_beta + 0.5 * loss_focal<choose_model_class>
balanced_filtered_df = pd.DataFrame()
samples_per_class = 500
for class_name in filtered_df.class_name.unique() :
balanced_filtered_df = balanced_filtered_df.append(filtered_df.query("class_name==@class_name")[:samples_per_class],
ignore_index=True )
learn = cnn_learner(data, base_arch=models.densenet201, loss_func=CombineLoss() , metrics=fbeta )<find_best_params>
samples_df = pd.DataFrame(diagnostic_per_image)
 samples_df["Height"] = image_size
samples_df["Width"] = image_size
learn.lr_find()
learn.recorder.plot()<train_model>
DATA_DIR = Path('.. /working/')
 ROOT_DIR = ".. /.. /working"
NUM_CATS = len(selected_classes)
 IMAGE_SIZE = 512
os.chdir('.. /working/maskrcnn-tf2-keras')
 sys.path.append(ROOT_DIR+'/maskrcnn-tf2-keras')
lr = 3e-2
learn.fit_one_cycle(3, slice(lr))<train_model>
COCO_WEIGHTS_PATH = '.. /.. /input/coco-weights/mask_rcnn_coco.h5'
class DiagnosticConfig(Config):
NAME = "Diagnostic"
NUM_CLASSES = NUM_CATS + 1
GPU_COUNT = 1
IMAGES_PER_GPU = 2
BACKBONE = 'resnet50'
IMAGE_MIN_DIM = IMAGE_SIZE
IMAGE_MAX_DIM = IMAGE_SIZE
IMAGE_RESIZE_MODE = 'none'
POST_NMS_ROIS_TRAINING = 250
POST_NMS_ROIS_INFERENCE = 150
MAX_GROUNDTRUTH_INSTANCES = 5
BACKBONE_STRIDES = [4, 8, 16, 32, 64]
BACKBONESHAPE =(8, 16, 24, 32, 48)
 RPN_ANCHOR_SCALES =(8,16,24,32,48)
 ROI_POSITIVE_RATIO = 0.33
DETECTION_MAX_INSTANCES = 300
DETECTION_MIN_CONFIDENCE = 0.7
STEPS_PER_EPOCH = int(len(samples_df)*0.9/IMAGES_PER_GPU)
 VALIDATION_STEPS = len(samples_df)-int(len(samples_df)*0.9/IMAGES_PER_GPU)
 config = DiagnosticConfig()
config.display()
lr = 1e-3
learn.fit_one_cycle(21, slice(lr/10, lr))<predict_on_test>
class DiagnosticDataset(utils.Dataset):
def __init__(self, df):
super().__init__(self)
 for i, name in enumerate(category_list):
self.add_class("diagnostic", i+1, name)
 for i, row in df.iterrows() :
self.add_image("diagnostic",
image_id=row.name,
path=".. /"+resized_folder+str(row.image_id)+".jpg",
labels=row['CategoryId'],
annotations=row['EncodedPixels'],
height=row['Height'], width=row['Width'])
 def image_reference(self, image_id):
info = self.image_info[image_id]
return info['path'], [category_list[int(x)] for x in info['labels']]
def load_image(self, image_id):
return cv2.imread(self.image_info[image_id]['path'])
 def load_mask(self, image_id):
info = self.image_info[image_id]
mask = np.zeros(( IMAGE_SIZE, IMAGE_SIZE, len(info['annotations'])) , dtype=np.uint8)
 labels = []
for m,(annotation, label)in enumerate(zip(info['annotations'], info['labels'])) :
sub_mask = np.full(info['height']*info['width'], 0, dtype=np.uint8)
 annotation = [int(x)for x in annotation.split(' ')]
for i, start_pixel in enumerate(annotation[::2]):
sub_mask[start_pixel: start_pixel+annotation[2*i+1]] = 1
sub_mask = sub_mask.reshape(( info['height'], info['width']), order='F')
 sub_mask = cv2.resize(sub_mask,(IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_NEAREST)
 mask[:, :, m] = sub_mask
labels.append(int(label)+1)
 return mask, np.array(labels )
valid_preds = learn.get_preds(DatasetType.Valid)
 best_thr = find_best_fixed_threshold(*valid_preds )<predict_on_test>
training_percentage = 0.9
training_set_size = int(training_percentage*len(samples_df))
validation_set_size = int(( 1-training_percentage)*len(samples_df))
train_dataset = DiagnosticDataset(samples_df[:training_set_size])
 train_dataset.prepare()
valid_dataset = DiagnosticDataset(samples_df[training_set_size:training_set_size+validation_set_size])
 valid_dataset.prepare()
for i in range(10):
image_id = random.choice(train_dataset.image_ids)
 image = train_dataset.load_image(image_id)
 mask, class_ids = train_dataset.load_mask(image_id)
 visualize.display_top_masks(image, mask, class_ids, train_dataset.class_names, limit=5 )
test_preds = learn.get_preds(DatasetType.Test)
 test_df.attribute_ids = join_preds(test_preds, best_thr)
 test_df.head()<save_to_csv>
LR = 1e-4
EPOCHS = [1,18]
model = modellib.MaskRCNN(mode='training', config=config, model_dir="")
 model.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=['mrcnn_class_logits', 'mrcnn_bbox_fc', 'mrcnn_bbox', 'mrcnn_mask'] )
test_df.to_csv('submission.csv', index=False )<install_modules>
%%time
model.train(train_dataset, valid_dataset,
learning_rate=LR,
epochs=EPOCHS[0],
layers='heads')
 history = model.keras_model.history.history
!pip install imgaug --upgrade<set_options>
%%time
model.train(train_dataset, valid_dataset,
learning_rate=LR/10,
epochs=EPOCHS[1],
layers='all')
 new_history = model.keras_model.history.history
for k in new_history: history[k] = history[k] + new_history[k]
WORKERS = 2
CHANNEL = 3
warnings.filterwarnings("ignore")
 SIZE = 300
NUM_CLASSES = 1103
beta_f2=2<init_hyperparams>
best_epoch = np.argmin(history["val_loss"][1:])+ 1
print("Best epoch: ", best_epoch)
 print("Valid loss: ", history["val_loss"][1:][best_epoch-1] )
class AdamAccumulate_v1(Optimizer):
def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., amsgrad=False, accum_iters=20, **kwargs):
super(AdamAccumulate, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.effective_iterations = K.variable(0, dtype='int64', name='effective_iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.amsgrad = amsgrad
self.accum_iters = K.variable(accum_iters, dtype='int64')
 @interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update(self.iterations, self.iterations + 1)]
flag = K.equal(self.iterations % self.accum_iters, self.accum_iters - 1)
 flag = K.cast(flag, K.floatx())
 self.updates.append(K.update(self.effective_iterations,
self.effective_iterations + K.cast(flag, 'int64')))
 lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.effective_iterations,
K.dtype(self.decay))))
t = K.cast(self.effective_iterations, K.floatx())+ 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) / 
(1.- K.pow(self.beta_1, t)))
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
else:
vhats = [K.zeros(1)for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat, gg in zip(params, grads, ms, vs, vhats, gs):
gg_t =(1 - flag)*(gg + g)
 m_t =(self.beta_1 * m)+(1.- self.beta_1)*(gg + flag * g)/ K.cast(self.accum_iters, K.floatx())
 v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(
(gg + flag * g)/ K.cast(self.accum_iters, K.floatx()))
if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 p_t = p - flag * lr_t * m_t /(K.sqrt(vhat_t)+ self.epsilon)
 self.updates.append(K.update(vhat, vhat_t))
else:
p_t = p - flag * lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)
 self.updates.append(( m, flag * m_t +(1 - flag)* m))
self.updates.append(( v, flag * v_t +(1 - flag)* v))
self.updates.append(( gg, gg_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad}
base_config = super(AdamAccumulate, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
class AdamAccumulate(Optimizer):
def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., amsgrad=False, accum_iters=1, **kwargs):
if accum_iters < 1:
raise ValueError('accum_iters must be >= 1')
 super(AdamAccumulate, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.amsgrad = amsgrad
self.accum_iters = K.variable(accum_iters, K.dtype(self.iterations))
self.accum_iters_float = K.cast(self.accum_iters, K.floatx())
 @interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
completed_updates = K.cast(K.tf.floor(self.iterations / self.accum_iters), K.floatx())
 if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * completed_updates))
t = completed_updates + 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) /(1.- K.pow(self.beta_1, t)))
 update_switch = K.equal(( self.iterations + 1)% self.accum_iters, 0)
 update_switch = K.cast(update_switch, K.floatx())
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
else:
vhats = [K.zeros(1)for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):
sum_grad = tg + g
avg_grad = sum_grad / self.accum_iters_float
m_t =(self.beta_1 * m)+(1.- self.beta_1)* avg_grad
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(avg_grad)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 p_t = p - lr_t * m_t /(K.sqrt(vhat_t)+ self.epsilon)
 self.updates.append(K.update(vhat,(1 - update_switch)* vhat + update_switch * vhat_t))
else:
p_t = p - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)
 self.updates.append(K.update(m,(1 - update_switch)* m + update_switch * m_t))
self.updates.append(K.update(v,(1 - update_switch)* v + update_switch * v_t))
self.updates.append(K.update(tg,(1 - update_switch)* sum_grad))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p,(1 - update_switch)* p + update_switch * new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad}
base_config = super(AdamAccumulate, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<load_from_csv>
class InferenceConfig(DiagnosticConfig):
GPU_COUNT = 1
IMAGES_PER_GPU = 1
IMAGE_MIN_DIM = IMAGE_SIZE
IMAGE_MAX_DIM = IMAGE_SIZE
IMAGE_RESIZE_MODE = 'none'
DETECTION_MIN_CONFIDENCE = 0.8
DETECTION_NMS_THRESHOLD = 0.5
inference_config = InferenceConfig()
model = modellib.MaskRCNN(mode='inference',
config=inference_config,
model_dir="" )
path_to_train = '.. /input/imet-2019-fgvc6/train/'
data = pd.read_csv('.. /input/imet-2019-fgvc6/train.csv')
 train_dataset_info = []
for name, labels in zip(data['id'], data['attribute_ids'].str.split(' ')) :
train_dataset_info.append({
'path':os.path.join(path_to_train, name),
'labels':np.array([int(label)for label in labels])})
 train_dataset_info = np.array(train_dataset_info )<compute_test_metric>
glob_list = glob.glob(f'diagnostic*/mask_rcnn_diagnostic_{best_epoch:04d}.h5')
 model_path = glob_list[0] if glob_list else ''
model.load_weights(model_path, by_name=True )
gamma = 2.0
epsilon = K.epsilon()
def focal_loss(y_true, y_pred):
pt = y_pred * y_true +(1-y_pred)*(1-y_true)
 pt = K.clip(pt, epsilon, 1-epsilon)
 CE = -K.log(pt)
 FL = K.pow(1-pt, gamma)* CE
loss = K.sum(FL, axis=1)
 return loss<normalization>
def refine_masks(masks, rois):
areas = np.sum(masks.reshape(-1, masks.shape[-1]), axis=0)
 mask_index = np.argsort(areas)
 union_mask = np.zeros(masks.shape[:-1], dtype=bool)
 for m in mask_index:
masks[:, :, m] = np.logical_and(masks[:, :, m], np.logical_not(union_mask))
union_mask = np.logical_or(masks[:, :, m], union_mask)
 for m in range(masks.shape[-1]):
mask_pos = np.where(masks[:, :, m]==True)
 if np.any(mask_pos):
y1, x1 = np.min(mask_pos, axis=1)
 y2, x2 = np.max(mask_pos, axis=1)
 rois[m, :] = [y1, x1, y2, x2]
return masks, rois
def decode_rle(rle, height, width):
s = rle.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(height*width, dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(( height, width)).T
def annotations_to_mask(annotations, height, width):
if isinstance(annotations, list):
mask = np.zeros(( height, width, len(annotations)))
 for i, rle_code in enumerate(annotations):
mask[:, :, i] = decode_rle(rle_code, height, width)
 else:
error_message = "{} is expected to be a list or str but received {}".format(annotation, type(annotation))
raise TypeError(error_message)
 return mask
def find_anomalies(dicom_image, display=False):
image_dimensions = dicom_image.shape
resized_img = cv2.resize(dicom_image,(image_size,image_size), interpolation = cv2.INTER_AREA)
 saved_filename = resized_test_folder+"temp_image.jpg"
cv2.imwrite(saved_filename, resized_img)
 img = cv2.imread(saved_filename)
 result = model.detect([img])
 r = result[0]
if r['masks'].size > 0:
masks = np.zeros(( img.shape[0], img.shape[1], r['masks'].shape[-1]), dtype=np.uint8)
 for m in range(r['masks'].shape[-1]):
masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), 
(img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST)
 y_scale = image_dimensions[0]/IMAGE_SIZE
x_scale = image_dimensions[1]/IMAGE_SIZE
rois =(r['rois'] * [y_scale, x_scale, y_scale, x_scale] ).astype(int)
 masks, rois = refine_masks(masks, rois)
 else:
masks, rois = r['masks'], r['rois']
if display:
visualize.display_instances(img, rois, masks, r['class_ids'],
['bg']+category_list, r['scores'],
title="prediction", figsize=(12, 12))
return rois, r['class_ids'], r['scores']
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
 class data_generator(Sequence):
def mix_up(x, y):
x = np.array(x, np.float32)
 lam = np.random.beta(0.2, 0.4)
 ori_index = np.arange(int(len(x)))
 index_array = np.arange(int(len(x)))
 np.random.shuffle(index_array)
 mixed_x = lam * x[ori_index] +(1 - lam)* x[index_array]
mixed_y = lam * y[ori_index] +(1 - lam)* y[index_array]
return mixed_x, mixed_y
def create_train(dataset_info, batch_size, shape, augument=True, mix=False):
assert shape[2] == 3
while True:
dataset_info = shuffle(dataset_info)
 for start in range(0, len(dataset_info), batch_size):
end = min(start + batch_size, len(dataset_info))
batch_images = []
X_train_batch = dataset_info[start:end]
batch_labels = np.zeros(( len(X_train_batch), NUM_CLASSES))
for i in range(len(X_train_batch)) :
image = data_generator.load_image(
 X_train_batch[i]['path'], augument)
 batch_images.append(preprocess_input(image))
batch_labels[i][X_train_batch[i]['labels']] = 1
if(mix):
batch_images, batch_labels = data_generator.mix_up(batch_images, batch_labels)
 yield np.array(batch_images, np.float32), batch_labels
def create_valid(dataset_info, batch_size, shape, augument=False):
assert shape[2] == 3
while True:
for start in range(0, len(dataset_info), batch_size):
end = min(start + batch_size, len(dataset_info))
batch_images = []
X_train_batch = dataset_info[start:end]
batch_labels = np.zeros(( len(X_train_batch), NUM_CLASSES))
for i in range(len(X_train_batch)) :
image = data_generator.load_image(
 X_train_batch[i]['path'], augument)
 batch_images.append(preprocess_input(image))
batch_labels[i][X_train_batch[i]['labels']] = 1
yield np.array(batch_images, np.float32), batch_labels
def load_image(path, augument=False):
image = cv2.imread(path+'.png')
 h, w, _ = image.shape
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 if augument:
image = iaa.CropToFixedSize(height=min(SIZE*2, h),
width=min(SIZE*2, w),
position='uniform' ).augment_image(image)
 image = data_generator.augment(image)
 else:
image = iaa.CropToFixedSize(height=min(SIZE*2, h),
width=min(SIZE*2, w),
position='center' ).augment_image(image)
 return cv2.resize(image,(SIZE, SIZE))
def augment(image):
augment_img = iaa.Sequential([
iaa.OneOf([
iaa.Affine(rotate=0),
iaa.Affine(rotate=90),
]),
iaa.OneOf([
iaa.Fliplr(0.5),
])], random_order=True)
 image_aug = augment_img.augment_image(image)
 return image_aug 
<import_modules>
test_folder = ".. /.. /input/vinbigdata-chest-xray-abnormalities-detection/test/"
test_file_list = os.listdir(test_folder)[:5]
for test_file in test_file_list:
dicom_image = read_xray(test_folder+test_file)
 find_anomalies(dicom_image, display=True )
BatchNormalization, Input, Conv2D, GlobalAveragePooling2D)
<choose_model_class>
submission_df = pd.DataFrame(results )
def ResNet(stack_fn,
preact,
use_bias,
model_name='resnet',
include_top=True,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000,
**kwargs):

 global backend, layers, models, keras_utils
backend, layers, models, keras_utils = keras.backend, keras.layers, keras.models, keras.utils
if not(weights in {'imagenet', None} or os.path.exists(weights)) :
raise ValueError('The `weights` argument should be either '
'`None`(random initialization), `imagenet` '
'(pre-training on ImageNet), '
'or the path to the weights file to be loaded.')
 if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as `"imagenet"` with `include_top`'
' as true, `classes` should be 1000')
 input_shape = utils._obtain_input_shape(input_shape,
default_size=224,
min_size=32,
data_format=backend.image_data_format() ,
require_flatten=include_top,
weights=weights)
 if input_tensor is None:
img_input = layers.Input(shape=input_shape)
 else:
if not backend.is_keras_tensor(input_tensor):
img_input = layers.Input(tensor=input_tensor, shape=input_shape)
 else:
img_input = input_tensor
bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1
x = layers.ZeroPadding2D(padding=(( 3, 3),(3, 3)) , name='conv1_pad' )(img_input)
 x = layers.Conv2D(64, 7, strides=2, use_bias=use_bias, name='conv1_conv' )(x)
 if preact is False:
x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5,
name='conv1_bn' )(x)
 x = layers.Activation('relu', name='conv1_relu' )(x)
 x = layers.ZeroPadding2D(padding=(( 1, 1),(1, 1)) , name='pool1_pad' )(x)
 x = layers.MaxPooling2D(3, strides=2, name='pool1_pool' )(x)
 x = stack_fn(x)
 if preact is True:
x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5,
name='post_bn' )(x)
 x = layers.Activation('relu', name='post_relu' )(x)
 if include_top:
x = layers.GlobalAveragePooling2D(name='avg_pool' )(x)
 x = layers.Dense(classes, activation='softmax', name='probs' )(x)
 else:
if pooling == 'avg':
x = layers.GlobalAveragePooling2D(name='avg_pool' )(x)
 elif pooling == 'max':
x = layers.GlobalMaxPooling2D(name='max_pool' )(x)
 if input_tensor is not None:
inputs = keras_utils.get_source_inputs(input_tensor)
 else:
inputs = img_input
model = models.Model(inputs, x, name=model_name)
 if(weights == 'imagenet')and(model_name in WEIGHTS_HASHES):
if include_top:
file_name = model_name + '_weights_tf_dim_ordering_tf_kernels.h5'
file_hash = WEIGHTS_HASHES[model_name][0]
else:
file_name = model_name + '_weights_tf_dim_ordering_tf_kernels_notop.h5'
file_hash = WEIGHTS_HASHES[model_name][1]
weights_path = keras_utils.get_file(file_name,
BASE_WEIGHTS_PATH + file_name,
cache_subdir='models',
file_hash=file_hash)
 model.load_weights(weights_path)
 elif weights is not None:
model.load_weights(weights)
 return model<categorify>
submission_df.to_csv('.. /submission.csv', index=False )
<choose_model_class><EOS> 
!rm -rf.. /.. /working/resized_train/
!rm -rf.. /.. /working/resized_test/