Kaggle competition: Freesound Audio Tagging 2019
path = ".. /input/efficient-net-deps-1"
sys.path.append(path)
<choose_model_class>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 999
seed_everything(SEED )
model = EfficientNet.from_name('efficientnet-b0')
 model.load_state_dict(torch.load('.. /input/efficientnet-pytorch/efficientnet-b0-08094119.pth'))
num_ftrs = model._fc.in_features
model._fc = nn.Linear(num_ftrs, 1)
 for param in model.parameters() :
param.requires_grad = True
model = model.to('cuda')
 optimizer = torch.optim.Adam(model.parameters() , lr=0.00015, weight_decay=1e-5)
 loss_func = nn.MSELoss()
train_on_gpu = torch.cuda.is_available()
model.load_state_dict(torch.load(".. /input/weights/weights.pt"))
model.eval()<categorify>
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class
data_transf = transforms.Compose([
transforms.ToPILImage() ,
transforms.RandomRotation(( -180, 180)) ,
transforms.RandomHorizontalFlip(p=0.4),
transforms.RandomVerticalFlip(p=0.5),
transforms.ToTensor() ,
transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225))
])
 test_data = Imagedata(df = label, data_dir = test_dir, transform = data_transf)
 test_loader = DataLoader(dataset = test_data, batch_size = 64 )<predict_on_test>
TRAIN_MODE = False
CONTINUOUS_TRAIN = True
MIXMATCH_SSL = 0
num = len(pd.read_csv(".. /input/aptos2019-blindness-detection/sample_submission.csv"))
test_preds = np.zeros(( num, 1))
device = torch.device("cuda:0")
 for j in range(5):
i = 0
tk0 = tqdm(test_loader)
 for i, x_batch in enumerate(tk0):
pred = model(x_batch.to(device))
test_preds[i * 64:(i + 1)* 64] += pred.detach().cpu().squeeze().numpy().ravel().reshape(-1, 1)* 0.2
test_preds<save_to_csv>
DATA = Path('.. /input/freesound-audio-tagging-2019')
 PREPROCESSED_N1K = Path('.. /input/fat2019_prep_mels1')
 PREPROCESSED_MP = Path('.. /input/fat2019-multipreprocessed-package')
 LAST_WEIGHTS = Path('.. /input/fat19-fastai-weights-of-mixup-mp')
 WORK = Path('work')
 Path(WORK ).mkdir(exist_ok=True, parents=True)
 CSV_TRAIN_CURATED = PREPROCESSED_MP/'train_curated_valid.csv'
CSV_TRAIN_NOISY = PREPROCESSED_MP/'train_noisy_valid.csv'
CSV_TRAIN_NOISY_BEST50S = PREPROCESSED_N1K/'trn_noisy_best50s.csv'
CSV_SUBMISSION = DATA/'sample_submission.csv'
SAVED_WEIGHT = LAST_WEIGHTS/'last'
DEPLOYED_MODEL = LAST_WEIGHTS/'export.pkl'
def add_prefix_path(df, prefix):
fnames = df.fname.copy()
df.fname = prefix + os.sep
df.fname = df.fname.str.cat(fnames)
 return df
def filter_df(df):
few_labels = list(map(lambda p: len(p[1])<= 3, df.labels.str.split(',' ).iteritems()))
return df[few_labels]
df_curated = add_prefix_path(pd.read_csv(CSV_TRAIN_CURATED), 'train_curated')
 df_noisy = add_prefix_path(pd.read_csv(CSV_TRAIN_NOISY), 'train_noisy')
 df_noisy50s = add_prefix_path(pd.read_csv(CSV_TRAIN_NOISY_BEST50S), 'train_noisy')
 df_submission = pd.read_csv(CSV_SUBMISSION)
 df_test = add_prefix_path(df_submission.copy() , 'test')
 df_train = df_curated
print('Training dataset size:', len(df_train))
df_train.head()
coef = [0.5, 1.5, 2.5, 3.5]
for i, pred in enumerate(test_preds):
if pred < coef[0]:
test_preds[i] = 0
elif pred >= coef[0] and pred < coef[1]:
test_preds[i] = 1
elif pred >= coef[1] and pred < coef[2]:
test_preds[i] = 2
elif pred >= coef[2] and pred < coef[3]:
test_preds[i] = 3
else:
test_preds[i] = 4
sample = pd.read_csv(".. /input/aptos2019-blindness-detection/sample_submission.csv")
 sample.diagnosis = test_preds.astype(int)
 sample.to_csv("submission.csv", index=False)
 sample<load_from_csv>
USE_MASK_FREQ = True
MASK_FREQ_RANGE = 8
MASK_FREQ_MAX_COUNT = 3
USE_MASK_TIME = True
MASK_TIME_RANGE = 8
MASK_TIME_MAX_COUNT = 3
def freq_mask(x, num=1, mask_size=10, mask_value=None, inplace=False):
cloned = x.clone() if not inplace else x
num_bins = cloned.shape[1]
mask_value = cloned.mean() if mask_value is None else mask_value
for i in range(num):
f = random.randrange(0, mask_size + 1)
 if f == 0 or f >= num_bins: continue
f_low = random.randrange(0, num_bins - f)
 f_high = f_low + f
cloned[:, f_low:f_high] = mask_value
return cloned
def time_mask(x, num=1, mask_size=10, mask_value=None, inplace=False):
cloned = x.clone() if not inplace else x
num_frames = cloned.shape[2]
mask_value = cloned.mean() if mask_value is None else mask_value
for i in range(num):
t = random.randrange(0, mask_size + 1)
 if t == 0 or t >= num_frames: continue
t_beg = random.randrange(0, num_frames - t)
 t_end = t_beg + t
cloned[:, :, t_beg:t_end] = mask_value
return cloned
TRAIN_PATH_NEW = '.. /input/upload3/train_images/train_images/'
TRAIN_PATH_OLD = '.. /input/upload3/train_2015/train_2015/'
TEST_PATH = '.. /input/aptos2019-blindness-detection/test_images/'
df_old = pd.read_csv('.. /input/upload2/trainlabels_2015.csv')
 df_old.columns = ['id_code', 'diagnosis']
df_old['id_code'] = TRAIN_PATH_OLD + df_old['id_code'] + '.jpeg'
df_new = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 df_new['id_code'] = TRAIN_PATH_NEW + df_new['id_code'] + ".png"
df = pd.concat([df_old, df_new], axis = 0)
 df_old['diagnosis'] = df_old['diagnosis'].astype(str)
 df_new['diagnosis'] = df_new['diagnosis'].astype(str)
 train, test = train_test_split(df_new, test_size =.2, random_state = 123, stratify = df_new['diagnosis'])
 batch_size = 8
image_size =(456,456)
 figax, ax = plt.subplots(1,1,figsize =(10,5))
sns.barplot(x = df['diagnosis'].value_counts().index, y = df['diagnosis'].value_counts() ).set_title("Diagnosis Distribution")
 sns.despine(bottom = True, left = True)
 sns.set_style('dark')
 ax.set_ylabel("")
 ax.get_yaxis().set_visible(False)
 plt.show()<drop_column>
class AugmentationConfig:
padding_scale = 1.
 whitenoise = True
whitenoise_level = 1e-3
pitchshift = True
pitchshift_steps = 2.
 class PreproConfig:
sr = 44100
duration = 2.
 n_out = 128
n_mels = 128
n_fft = n_mels * 20
hop_len = int(sr * duration // n_out)
 sample_size = int(sr * duration)
 padding_size = int(sample_size * AugmentationConfig.padding_scale)
 def load_audio(file, eps=1e-6):
pcm, sr = librosa.load(file, sr=PreproConfig.sr)
 if len(pcm)<= 0:
pcm = np.zeros(PreproConfig.sample_size)+ 1e-8
sr = PreproConfig.sr
pcm_max = np.max(np.abs(pcm))
if pcm_max > eps:
pcm = pcm / pcm_max
min_size = PreproConfig.padding_size
if len(pcm)> min_size: pcm, _ = librosa.effects.trim(pcm)
 pcm = pad_zeros(pcm, min_size)
 return pcm, pcm_max
def pad_zeros(x, size):
if len(x)> size:
return x
else:
padding = size - len(x)
 offset = padding // 2
return np.pad(x,(offset, size - len(x)- offset), 'constant')
 def conv_pcm_to_magphs(x):
fft = librosa.core.stft(x, n_fft=PreproConfig.n_fft, hop_length=PreproConfig.hop_len)
 return np.abs(fft), np.angle(fft)
 def conv_magphs_to_mels(magphs):
mel_basis = librosa.filters.mel(PreproConfig.sr, PreproConfig.n_fft, n_mels=PreproConfig.n_mels)
 return np.dot(mel_basis, magphs)
 def conv_mag_to_melspec(mag):
mels = conv_magphs_to_mels(mag ** 2.)
mels = librosa.power_to_db(mels)
 m_max = mels.max()
m_min = mels.min()
return(mels - m_min)/(m_max - m_min)
 def normalize_melspec(melspec, eps=1e-6, dtype=np.uint8):
mean = melspec.mean()
std = melspec.std()
ms_std =(melspec - mean)/(std + eps)
 ms_min, ms_max = ms_std.min() , ms_std.max()
if(ms_max - ms_min)> eps:
ms_norm = 255.*(ms_std - ms_min)/(ms_max - ms_min)
 return ms_norm.astype(dtype)
 else:
return np.zeros_like(ms_std, dtype=dtype)
 def preprocess(pcm, debug=False, dtype=np.uint8):
melspec, melphss = conv_pcm_to_magphs(pcm)
 melspec = conv_mag_to_melspec(melspec)
 melspec = normalize_melspec(melspec, dtype=dtype)
 return melspec
pretrain_generator = ImgGenerator(df = df_old,
x_col = "id_code",
y_col = 'diagnosis',
batch_size = batch_size,
img_size = image_size,
postprocessing_function = postprocess_input,
should_aug = True,
should_mix = False,
iaa_seq = seq)
 pretrain_validation_generator = ImgGenerator(df = df_new,
x_col = "id_code",
y_col = 'diagnosis',
batch_size = batch_size,
img_size = image_size,
postprocessing_function = postprocess_input,
should_aug = False,
should_mix = False)
 train_generator = ImgGenerator(df = train,
x_col = "id_code",
y_col = 'diagnosis',
batch_size = batch_size,
img_size = image_size,
postprocessing_function = postprocess_input,
should_aug = True,
should_mix = False,
iaa_seq = seq)
 validation_generator = ImgGenerator(df = test,
x_col = "id_code",
y_col = 'diagnosis',
batch_size = batch_size,
img_size = image_size,
postprocessing_function = postprocess_input,
should_aug = False,
should_mix = False )<choose_model_class>
class MPLoader:
cache = dict()
cache_vmem_percent = 45.0
data_type = 'msp'
use_augmentation = True
max_of_augid = 2
def reset() :
MPLoader.cache = dict()
def get(fname, augmentation=False, cache=False):
if fname in MPLoader.cache:
data = MPLoader.cache[fname]
else:
data = MPLoader.load(fname, cache)
 if augmentation and MPLoader.use_augmentation:
return random.choice(data)
 else:
return data[0]
def load(fname, cache=False):
data_raw = np.load(( PREPROCESSED_MP/fname ).with_suffix('.wav.npz'))
data = list()
if MPLoader.use_augmentation:
for i in range(1, MPLoader.max_of_augid+1):
tid = f'{MPLoader.data_type}{i}'
if tid in data_raw:
data.append(MPLoader._mp2tensor(data_raw[tid]))
else:
break
else:
tid = f'{MPLoader.data_type}1'
data.append(MPLoader._mp2tensor(data_raw[tid]))
if cache: MPLoader.cache[fname] = data
return data
def load_and_process(fname, cache=False):
pcm, _ = load_audio(DATA/fname)
 mels = preprocess(pcm)
 data = [MPLoader._mp2tensor(mels)]
if cache: MPLoader.cache[fname] = data
return data
def _mp2tensor(mp):
mp = torch.FloatTensor(mp ).div_(255)
 return mp
def full_load(df, use_preprocess=True):
vmem_limit = MPLoader.cache_vmem_percent
virt = psutil.virtual_memory()
if virt.percent >= vmem_limit:
vmem_limit = virt.percent + 0.1
for i, row in tqdm_notebook(df.iterrows() , total=len(df)) :
if use_preprocess:
MPLoader.load(row.fname, cache=True)
 else:
MPLoader.load_and_process(row.fname, cache=True)
 if i % 10 == 0:
virt = psutil.virtual_memory()
if virt.percent >= vmem_limit:
mb = 1024 * 1024
print(f'Stopped loading preprocessed data as cache, '
f'cause memory usage reached {vmem_limit}%.')
 print('Memory usage:', virt.available>>20, '/', virt.total>>20, 'MB')
 break
elif i % 1000 == 0:
print(virt.percent, '%', virt.available>>20, 'MB')
 print('Completed loading preprocessed data.')
 if TRAIN_MODE:
MPLoader.full_load(df_train)
 if MIXMATCH_SSL > 0:
MPLoader.full_load(df_noisy )
class AdamAccumulate(Optimizer):
def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., amsgrad=False, accum_iters=1, **kwargs):
if accum_iters < 1:
raise ValueError('accum_iters must be >= 1')
 super(AdamAccumulate, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.amsgrad = amsgrad
self.accum_iters = K.variable(accum_iters, K.dtype(self.iterations))
self.accum_iters_float = K.cast(self.accum_iters, K.floatx())
 @interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
completed_updates = K.cast(K.tf.floordiv(self.iterations, self.accum_iters), K.floatx())
 if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * completed_updates))
t = completed_updates + 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) /(1.- K.pow(self.beta_1, t)))
 update_switch = K.equal(( self.iterations + 1)% self.accum_iters, 0)
 update_switch = K.cast(update_switch, K.floatx())
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
gs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
else:
vhats = [K.zeros(1)for _ in params]
self.weights = [self.iterations] + ms + vs + vhats
for p, g, m, v, vhat, tg in zip(params, grads, ms, vs, vhats, gs):
sum_grad = tg + g
avg_grad = sum_grad / self.accum_iters_float
m_t =(self.beta_1 * m)+(1.- self.beta_1)* avg_grad
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(avg_grad)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 p_t = p - lr_t * m_t /(K.sqrt(vhat_t)+ self.epsilon)
 self.updates.append(K.update(vhat,(1 - update_switch)* vhat + update_switch * vhat_t))
else:
p_t = p - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)
 self.updates.append(K.update(m,(1 - update_switch)* m + update_switch * m_t))
self.updates.append(K.update(v,(1 - update_switch)* v + update_switch * v_t))
self.updates.append(K.update(tg,(1 - update_switch)* sum_grad))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p,(1 - update_switch)* p + update_switch * new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad}
base_config = super(AdamAccumulate, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<init_hyperparams>
TIME_DIM = 128
def open_fat2019_image(fn, convert_mode, after_open)->Image:
fname = '/'.join(fn.split('/')[-2:])
 x = MPLoader.get(fname, augmentation=True)
 base_dim, time_dim = x.shape
if time_dim < TIME_DIM:
x2 = torch.zeros(( base_dim,TIME_DIM), dtype=x.dtype)
 crop = random.randint(0, TIME_DIM - time_dim)
 x2[:, crop:crop+time_dim] = x
x = x2
else:
crop = random.randint(0, time_dim - TIME_DIM)
 x = x[:, crop:crop+TIME_DIM]
x = torch.stack(( x,x,x), dim=0)
 if USE_MASK_FREQ: freq_mask(
 x,
num=random.randrange(MASK_FREQ_MAX_COUNT),
mask_size=MASK_FREQ_RANGE,
mask_value=0,
inplace=True, 
)
 if USE_MASK_TIME: time_mask(
 x,
num=random.randrange(MASK_TIME_MAX_COUNT),
mask_size=MASK_TIME_RANGE,
mask_value=0,
inplace=True, 
)
 return Image(x)
 vision.data.open_image = open_fat2019_image
class EvaluationMetrics(Callback):
def __init__(self, validation_data = None, batch_size = 32):
super(Callback, self ).__init__()
self.validation_generator = validation_data
self.batch_size = batch_size
def on_train_begin(self, logs={}):
self._val_qwk = []
self._val_pred_counts = pd.DataFrame(index = [0,1,2,3,4])
 self._val_confusion_matrices = []
self._epoch = 1
def on_epoch_end(self, batch, logs={}):
y_val = np.array([])
 for i in range(len(validation_generator)) :
y_val_batch = validation_generator[i][1]
y_val = np.append(y_val, y_val_batch)
 y_val.astype(int)
 y_predict = model.predict_generator(generator=validation_generator,
steps=np.ceil(float(len(y_val)) / float(batch_size)) ,
verbose = 1 ).tolist()
for i in range(len(y_predict)) :
if y_predict[i][0] <= 0.5:
y_predict[i][0] = 0
elif y_predict[i][0] <= 1.5:
y_predict[i][0] = 1
elif y_predict[i][0] <= 2.5:
y_predict[i][0] = 2
elif y_predict[i][0] <= 3.5:
y_predict[i][0] = 3
else:
y_predict[i][0] = 4
y_predict = np.array(y_predict)
 self.qwk_hx(y_val, y_predict)
 self.value_counts_hx(y_predict)
 self.confusion_matrix_hx(y_val, y_predict)
 self._epoch += 1
return
def qwk_hx(self, y_val, y_predict):
qwk = cohen_kappa_score(y_val,
y_predict,
labels = [0,1,2,3,4],
weights='quadratic')
 print("
quadratic weighted kappa score: {0}
".format(qwk))
self._val_qwk.append(qwk)
 return
def value_counts_hx(self, y_predict):
self._val_pred_counts['Epoch {0}'.format(self._epoch)] = pd.DataFrame(y_predict)[0].value_counts()
self._val_pred_counts.fillna(0,inplace=True)
 self._val_pred_counts = self._val_pred_counts.astype(int)
 print("Value counts at Epoch {0}:
".format(self._epoch))
print(self._val_pred_counts.head())
 return
def confusion_matrix_hx(self, y_val, y_predict):
self._val_confusion_matrices.append(confusion_matrix(y_val, y_predict))
return
def get_qwk_hx(self):
return self._val_qwk
def get_value_counts_hx(self):
return self._val_pred_counts
def get_confusion_matrix_hx(self):
return self.val_confusion_matrices
eval_metrics = EvaluationMetrics(validation_data = validation_generator, batch_size = batch_size )<compute_train_metric>
BATCH_SIZE = 48
tfms = get_transforms(do_flip=True, max_rotate=0, max_lighting=0.1, max_zoom=0, max_warp=0.)
src =(ImageList.from_df(df_train, WORK, folder='')
.split_none() 
.label_from_df(label_delim=',')
)
 data =(src.transform(tfms, size=128)
.databunch(bs=BATCH_SIZE ).normalize(imagenet_stats)
)
 if MIXMATCH_SSL > 0:
noisy_src =(ImageList.from_df(df_noisy, WORK, folder='')
.split_none() 
.label_from_df(label_delim=',')
)
 noisy_data =(noisy_src.transform(tfms, size=128)
.databunch(bs=BATCH_SIZE ).normalize(imagenet_stats)
 )
def mse(y_true, y_pred):
return K.mean(K.square(y_pred -y_true), axis=-1)
 inputs = Input(( 456, 456, 3))
base_model = EfficientNetB5(include_top=False, weights=None, input_tensor = inputs)
 base_model.load_weights('.. /input/efficientnet-keras-weights-b0b5/efficientnet-b5_imagenet_1000_notop.h5')
 out = GlobalAveragePooling2D()(base_model.output)
 out = Dropout(0.5 )(out)
 out = Dense(1024,activation="relu", kernel_regularizer=l2(0.01))(out)
 out = Dropout(0.5 )(out)
 out = Dense(1,activation="linear", kernel_regularizer=l2(0.01))(out)
 model = Model(inputs, out)
 learning_rate_reduction = ReduceLROnPlateau(monitor='val_loss',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.0000001)
 early_stopping = EarlyStopping(monitor = 'val_loss',
patience = 10,
restore_best_weights = True)
 for layer in model.layers[:-5]:
layer.trainable = False
model.compile(loss='mse', optimizer=AdamAccumulate(lr=0.001, accum_iters = 2), metrics=[mse])
 print("
Warming up model: " + "
"*2)
 epochs = 5
history = model.fit_generator(pretrain_generator,
epochs=epochs,
steps_per_epoch =(df_old.shape[0] // batch_size)+ 1,
validation_data = pretrain_validation_generator,
validation_steps =(df_new.shape[0] // batch_size)+ 1,
workers = 2,
use_multiprocessing = True,
verbose=1)
 for layer in model.layers:
layer.trainable = True
print('
Begining training session: ' + '
'*2)
 model.compile(loss='mse', optimizer=AdamAccumulate(lr=0.0005, accum_iters = 3), metrics=[mse])
 epochs = 10
history2 = model.fit_generator(pretrain_generator,
epochs=epochs,
steps_per_epoch =(df_old.shape[0] // batch_size)+ 1,
validation_data = pretrain_validation_generator,
validation_steps =(df_new.shape[0] // batch_size)+ 1,
callbacks=[learning_rate_reduction, early_stopping],
workers = 2,
use_multiprocessing = True,
verbose=1)
 epochs = 40
history3 = model.fit_generator(train_generator,
epochs=epochs,
steps_per_epoch =(train.shape[0] // batch_size)+ 1,
validation_data = validation_generator,
validation_steps =(test.shape[0] // batch_size)+ 1,
callbacks=[learning_rate_reduction, early_stopping],
workers = 2,
use_multiprocessing = True,
verbose=1)
 model.save('model.h5' )<predict_on_test>
if TRAIN_MODE: data.show_batch(3 )
y_val = np.array([])
 for i in range(len(validation_generator)) :
v_val_batch = validation_generator[i][1]
y_val = np.append(y_val, v_val_batch)
 y_val.astype(int)
 y_pred = model.predict_generator(generator=validation_generator,
steps=np.ceil(float(len(y_val)) / float(batch_size)) ,
verbose=1)
 y_pred[y_pred <= 0.5] = 0
y_pred[(y_pred <= 1.5)&(y_pred > 0.5)] = 1
y_pred[(y_pred <= 2.5)&(y_pred > 1.5)] = 2
y_pred[(y_pred <= 3.5)&(y_pred > 2.5)] = 3
y_pred[y_pred > 3.5] = 4
score = cohen_kappa_score(y_val,
y_pred.astype(int),
labels=[0,1,2,3,4],
weights='quadratic')
 print("Quadratic weighted kappa: {0}".format(score))<load_from_csv>
def lwlrap(y_pred,y_true):
score, weight = calculate_per_class_lwlrap(y_true.cpu().numpy() , y_pred.cpu().numpy())
 lwlrap =(score * weight ).sum()
return torch.from_numpy(np.array(lwlrap))
sub_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 sub_df['id_code'] = sub_df['id_code']+'.png'
preds = []
for ix, idx in enumerate(sub_df['id_code']):
path = '.. /input/aptos2019-blindness-detection/test_images/' + idx
image = cv2.imread(path)
 image = cv2.resize(image,(456,456))
score_predict = [model.predict(( postprocess_input(seq_tta.augment_image(image)) [np.newaxis])/255)for i in range(5)]
score_predict.append(model.predict(( postprocess_input(image)[np.newaxis])/255))
score_predict = sum(score_predict)/ float(len(score_predict))
if score_predict <= 0.5:
score_predict = 0
elif score_predict <= 1.5:
score_predict = 1
elif score_predict <= 2.5:
score_predict = 2
elif score_predict <= 3.5:
score_predict = 3
else:
score_predict = 4
preds.append(int(score_predict))
sub_df['diagnosis'] = np.array(preds, np.int)
 sub_df['id_code'] = sub_df['id_code'].str.split(".png", expand = True)[0]
sub_df.to_csv('submission.csv', index = False)
 sub_df.head()<load_from_csv>
class MixMatchCallback(LearnerCallback):
def __init__(self, learn:Learner,
unlabeled_dl:DeviceDataLoader,
temperature:float=0.5, n_augment:int=2,
alpha:float=0.75, lambda_u:float=100, rampup:int=16):
super().__init__(learn)
 self.unlabeled_dl = unlabeled_dl
self.T = temperature
self.K = n_augment
self.beta_distirb = torch.distributions.beta.Beta(alpha, alpha)
 self.lambda_u = lambda_u
self.rampup = rampup
self.n_iterations = len(learn.data.train_dl)
 def on_train_begin(self, **kwargs):
self.unlabeled_iter = iter(self.unlabeled_dl)
 self.learn.loss_func = MultiSemiLoss(self.learn.loss_func, lambda_u=self.lambda_u, rampup=self.rampup)
 def on_train_end(self, **kwargs):
self.unlabeled_iter = None
self.learn.loss_func = self.learn.loss_func.get_old()
def unlabeled_next_batch(self):
try:
return next(self.unlabeled_iter)
 except:
self.unlabeled_iter = iter(self.unlabeled_dl)
 return next(self.unlabeled_iter)
 def sharpen(self, y):
y = y.pow(1 / self.T)
 return y / y.sum(dim=1, keepdim=True)
 def on_batch_begin(self, epoch, iteration, last_input, last_target, train, **kwargs):
if not train: return
with torch.no_grad() :
bs = len(last_input)
 u_x_hat = [self.unlabeled_next_batch() [0] for _ in range(self.K)]
y_hat = sum([self.learn.model(u_x_hat[i] ).sigmoid_() for i in range(self.K)])/ self.K
y_hat = self.sharpen(y_hat)
 y_hat = y_hat.repeat(self.K, 1)
 u_x_hat = torch.cat([last_input] + u_x_hat, dim=0)
 y_hat = torch.cat(( last_target, y_hat), dim=0)
 index = torch.randperm(u_x_hat.shape[0])
 lam = self.beta_distirb.sample().item()
lam = max(lam, 1-lam)
 mixed_input = lam * u_x_hat +(1-lam)* u_x_hat[index]
mixed_input = self.interleave(torch.split(mixed_input, bs), bs)
 mixed_target = lam * y_hat +(1-lam)* y_hat[index]
loss_func = self.learn.loss_func
loss_func.labeled_bs = bs
loss_func.epoch = iteration / self.n_iterations
return {'last_input': mixed_input, 'last_target': mixed_target}
def on_loss_begin(self, last_input, last_output, iteration, **kwargs):
bs = len(last_input)
 last_output = self.interleave(torch.split(last_output, bs), bs)
 return {'last_output': last_output}
def interleave_offsets(self, batch, nu):
groups = [batch //(nu + 1)] *(nu + 1)
 for x in range(batch - sum(groups)) :
groups[-x - 1] += 1
offsets = [0]
for g in groups:
offsets.append(offsets[-1] + g)
 return offsets
def interleave(self, xy, batch):
nu = len(xy)- 1
offsets = self.interleave_offsets(batch, nu)
 xy = [[v[offsets[p]:offsets[p + 1]] for p in range(nu + 1)] for v in xy]
for i in range(1, nu + 1):
xy[0][i], xy[i][i] = xy[i][i], xy[0][i]
return torch.cat([torch.cat(v, dim=0)for v in xy], dim=0)
 MM_LOG = pd.DataFrame(columns=['loss', 'loss_labeled', 'loss_unlabeled', 'weight'])
 class MultiSemiLoss(nn.Module):
"Adapt the loss function `crit` to go with mixmatch."
def __init__(self, crit, lambda_u=100.0, rampup=16):
super().__init__()
self.crit = crit
self.labeled_bs = 0
self.lambda_u = lambda_u
self.rampup = rampup
self.epoch = 0
def forward(self, output, target):
if self.labeled_bs == 0: return self.crit(output, target)
 output_x, target_x = output[:self.labeled_bs], target[:self.labeled_bs]
output_u, target_u = output[self.labeled_bs:], target[self.labeled_bs:]
q = output_u.sigmoid()
Lx = self.crit(output_x, target_x)
 Lu = torch.mean(( q - target_u)** 2)
 w = self.lambda_u *(np.clip(self.epoch / self.rampup, 0., 1.) if self.rampup > 0 else 1.)
self.labeled_bs = 0
d = Lx + w * Lu
MM_LOG.loc[self.epoch] = [float(d), float(Lx), float(Lu), w]
return d
def get_old(self):
return self.crit
def _mixmatch(learn:Learner, unlabeled_data:ImageDataBunch,
temperature:float=0.5, n_augment:int=2,
alpha:float=0.75, lambda_u:float=100, rampup:int=16)-> Learner:
"Add mixup https://arxiv.org/abs/1905.02249 to `learn`."
unlabeled_dl = unlabeled_data.dl(DatasetType.Train)
 learn.callback_fns.append(partial(MixMatchCallback,
unlabeled_dl=unlabeled_dl,
temperature=temperature, n_augment=n_augment,
alpha=alpha, lambda_u=lambda_u, rampup=rampup))
return learn
Learner.mixmatch = _mixmatch
sales = pd.read_csv(os.path.join(dirname, 'sales_train.csv'))
items = pd.read_csv(os.path.join(dirname, 'items.csv'))
item_cat = pd.read_csv(os.path.join(dirname, 'item_categories.csv'))
shops = pd.read_csv(os.path.join(dirname, 'shops.csv'))
test = pd.read_csv(os.path.join(dirname, 'test.csv'))
submission = pd.read_csv(os.path.join(dirname, 'sample_submission.csv')) 
<set_options>
class MixUpCallback(LearnerCallback):
"Callback that creates the mixed-up input and target."
def __init__(self, learn:Learner, alpha:float=0.4, stack_x:bool=False, stack_y:bool=True):
super().__init__(learn)
 self.alpha,self.stack_x,self.stack_y = alpha,stack_x,stack_y
def on_train_begin(self, **kwargs):
if self.stack_y: self.learn.loss_func = MixUpLoss(self.learn.loss_func)
 def on_batch_begin(self, last_input, last_target, train, **kwargs):
"Applies mixup to `last_input` and `last_target` if `train`."
if not train: return
lambd = np.random.beta(self.alpha, self.alpha, last_target.size(0))
lambd = np.concatenate([lambd[:,None], 1-lambd[:,None]], 1 ).max(1)
 lambd = last_input.new(lambd)
 shuffle = torch.randperm(last_target.size(0)).to(last_input.device)
 x1, y1 = last_input[shuffle], last_target[shuffle]
if self.stack_x:
new_input = [last_input, last_input[shuffle], lambd]
else:
new_input =(last_input * lambd.view(lambd.size(0),1,1,1)+ x1 *(1-lambd ).view(lambd.size(0),1,1,1))
if self.stack_y:
new_target = torch.cat([last_target[:,None].float() , y1[:,None].float() , lambd[:,None].float() ], 1)
 else:
if len(last_target.shape)== 2:
lambd = lambd.unsqueeze(1 ).float()
new_target = last_target.float() * lambd + y1.float() *(1-lambd)
 return {'last_input': new_input, 'last_target': new_target}
def on_train_end(self, **kwargs):
if self.stack_y: self.learn.loss_func = self.learn.loss_func.get_old()
class MixUpLoss(nn.Module):
"Adapt the loss function `crit` to go with mixup."
def __init__(self, crit, reduction='mean'):
super().__init__()
if hasattr(crit, 'reduction'):
self.crit = crit
self.old_red = crit.reduction
setattr(self.crit, 'reduction', 'none')
 else:
self.crit = partial(crit, reduction='none')
 self.old_crit = crit
self.reduction = reduction
def forward(self, output, target):
if len(target.size())== 2:
loss1, loss2 = self.crit(output,target[:,0].long()), self.crit(output,target[:,1].long())
 d =(loss1 * target[:,2] + loss2 *(1-target[:,2])).mean()
else: d = self.crit(output, target)
 if self.reduction == 'mean': return d.mean()
elif self.reduction == 'sum': return d.sum()
return d
def get_old(self):
if hasattr(self, 'old_crit'): return self.old_crit
elif hasattr(self, 'old_red'):
setattr(self.crit, 'reduction', self.old_red)
 return self.crit
def _mixup(learn:Learner, alpha:float=0.4, stack_x:bool=False, stack_y:bool=True)-> Learner:
"Add mixup https://arxiv.org/abs/1710.09412 to `learn`."
learn.callback_fns.append(partial(MixUpCallback, alpha=alpha, stack_x=stack_x, stack_y=stack_y))
return learn
Learner.mixup = _mixup
%matplotlib inline 
<import_modules>
class ConvBlock(nn.Module):
def __init__(self, in_channels, out_channels):
super().__init__()
self.conv1 = nn.Sequential(
 nn.Conv2d(in_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self.conv2 = nn.Sequential(
 nn.Conv2d(out_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self._init_weights()
def _init_weights(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight)
 if m.bias is not None:
nn.init.zeros_(m.bias)
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.zeros_(m.bias)
 def forward(self, x):
x = self.conv1(x)
 x = self.conv2(x)
 x = F.avg_pool2d(x, 2)
 return x
class Classifier(nn.Module):
def __init__(self, num_classes=1000):
super().__init__()
self.conv = nn.Sequential(
 ConvBlock(in_channels=3, out_channels=64),
ConvBlock(in_channels=64, out_channels=128),
ConvBlock(in_channels=128, out_channels=256),
ConvBlock(in_channels=256, out_channels=512), 
)
 self.avgpool = nn.AdaptiveAvgPool2d(( 4, 1))
self.fc = nn.Sequential(
 nn.Dropout(0.5),
nn.Linear(512*4, 256),
nn.PReLU() ,
nn.BatchNorm1d(256),
nn.Dropout(0.2),
nn.Linear(256, num_classes), 
)
 def forward(self, x):
x = self.conv(x)
 x = self.avgpool(x)
 x = self.fc(x)
 return x
for p in [np, pd, scipy,sklearn, seaborn, lgb]:
print(p.__name__, p.__version__ )<data_type_conversions>
labels = df_submission.columns[1:].tolist()
label_size = len(labels)
 def calc_P_R_AP(y_true, y_pred):
P = [None] * label_size
R = [None] * label_size
AP = np.zeros(label_size)
 for i in range(label_size):
P[i], R[i], _ = precision_recall_curve(y_true[:,i], y_pred[:,i])
 AP[i] = average_precision_score(y_true[:,i], y_pred[:,i])
 df_ap = pd.DataFrame(data=AP, index=labels, columns=['AP'])
 return P, R, AP, df_ap
def downcast_dtypes(df):

 float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype == "int64"]
df[float_cols] = df[float_cols].astype(np.float32)
 df[int_cols] = df[int_cols].astype(np.int32)
 return df<compute_test_metric>
def normalize_predict(y):
min_pred = y.min(axis=1 ).reshape(-1,1)
 max_pred = y.max(axis=1 ).reshape(-1,1)
 return(y - min_pred)/(max_pred - min_pred )
def get_rmse(actual, predicted):

 mse = mean_squared_error(actual, predicted)
 rmse = sqrt(mse)
 return rmse<feature_engineering>
def borrowed_model(pretrained=False, **kwargs):
return Classifier(**kwargs)
 if TRAIN_MODE:
f_score = partial(fbeta, thresh=0.2)
 learn = cnn_learner(
 data,
borrowed_model, pretrained=False,
metrics=[lwlrap],
loss_func=nn.MultiLabelSoftMarginLoss() 
)
 if MIXMATCH_SSL > 0:
if CONTINUOUS_TRAIN:
learn.mixmatch(noisy_data, temperature=0.5, n_augment=MIXMATCH_SSL, alpha=0.4, lambda_u=50., rampup=0)
 else:
learn.mixmatch(noisy_data, temperature=0.5, n_augment=MIXMATCH_SSL, alpha=0.4, lambda_u=10., rampup=1000)
 else:
learn.mixup(alpha=0.4, stack_y=False)
 if CONTINUOUS_TRAIN: learn.load(Path('.. /.. ')/SAVED_WEIGHT)
 learn.unfreeze()
sales_test = sales.copy()
test_grid = test[["shop_id", "item_id"]]
test_grid["date_block_num"] = 34
test_grid["month"] = 11
test_grid["date"] = test_grid["item_price"] = test_grid["item_cnt_day"] = 0
test_grid = test_grid.reindex(columns=sales.columns.values)
 sales_test = sales_test.append(test_grid )<merge>
if TRAIN_MODE and CONTINUOUS_TRAIN:
df_ap = pd.read_csv(LAST_WEIGHTS/'labels_ap.csv', index_col=0)
 loss_weights = torch.FloatTensor(( 1/df_ap.AP ).values ** 4 ).cuda()
print(loss_weights)
 learn.loss_func = nn.MultiLabelSoftMarginLoss(weight=loss_weights)
 else:
loss_weights = None
cols_to_rename = list(all_data.columns.difference(index_cols))
shift_range = [1, 2, 3, 4, 5, 12]
for month_shift in notebook.tqdm(shift_range):
train_shift = all_data[index_cols + cols_to_rename].copy()
train_shift['date_block_num'] = train_shift['date_block_num'] + month_shift
foo = lambda x: '{}_lag_{}'.format(x, month_shift)if x in cols_to_rename else x
train_shift = train_shift.rename(columns=foo)
 all_data = pd.merge(all_data, train_shift, on=index_cols, how='left' ).fillna(0)
 del train_shift
fit_cols = [col for col in all_data.columns if col[-1] in [str(item)for item in shift_range]]
to_drop_cols = list(set(list(all_data.columns)) -(set(fit_cols)|set(index_cols)))+ ['date_block_num']
item_category_mapping = items[['item_id','item_category_id']].drop_duplicates()
all_data = pd.merge(all_data, item_category_mapping, how='left', on='item_id')
 all_data = downcast_dtypes(all_data)
 gc.collect() ;<feature_engineering>
if TRAIN_MODE:
learn.lr_find()
learn.recorder.plot(suggestion=True )
shop_first_block = sales_test.groupby("shop_id" ).date_block_num.min()
all_data["shop_1st"] = all_data["shop_id"].map(shop_first_block)
 all_data["shop_1st"] = all_data["date_block_num"] - all_data["shop_1st"]
item_first_block = sales_test.groupby("item_id" ).date_block_num.min()
all_data["item_1st"] = all_data["item_id"].map(item_first_block)
 all_data["item_1st"] = all_data["date_block_num"] - all_data["item_1st"]
all_data.shop_1st.value_counts()<feature_engineering>
if TRAIN_MODE:
gc.collect()
callbacks = [
SaveModelCallback(learn, every='improvement', monitor='lwlrap', name='best'),
]
if CONTINUOUS_TRAIN:
learn.fit_one_cycle(50, slice(1e-7,1e-4), callbacks=callbacks)
 else:
learn.fit_one_cycle(300, 2e-2, callbacks=callbacks )
all_data["month"]=(all_data["date_block_num"]+1)%12<define_variables>
USE_MASK_FREQ = USE_MASK_TIME = False
MPLoader.use_augmentation = False
if TRAIN_MODE:
y_pred, y_true = learn.get_preds(ds_type=DatasetType.Train)
 y_pred = torch.from_numpy(normalize_predict(y_pred.cpu().numpy()))
print('Local LwLRAP score(Train dataset):', float(lwlrap(y_pred, y_true ).float()))
dates = all_data['date_block_num']
last_block = dates.max()
print('Test `date_block_num` is %d' % last_block )<prepare_x_and_y>
if TRAIN_MODE:
learn.save('last')
 learn.export()
dates_train = dates[dates < last_block]
dates_test = dates[dates == last_block]
X_train = all_data.loc[dates < last_block].drop(to_drop_cols, axis=1)
 X_test = all_data.loc[dates == last_block].drop(to_drop_cols, axis=1)
 y_train = all_data.loc[dates < last_block, 'target'].values.clip(0,20)
 y_test = all_data.loc[dates == last_block, 'target'].values.clip(0,20)
 print(X_train.head() )<feature_engineering>
if TRAIN_MODE:
y_true2 = y_true.numpy()
y_pred2 = y_pred.numpy()
labels = df_submission.columns[1:].tolist()
label_size = len(labels)
 P = [None] * label_size
R = [None] * label_size
AP = np.zeros(label_size)
 for i in range(label_size):
P[i], R[i], _ = precision_recall_curve(y_true2[:,i], y_pred2[:,i])
 AP[i] = average_precision_score(y_true2[:,i], y_pred2[:,i])
 df_ap = pd.DataFrame(data=AP, index=labels, columns=['AP'])
 df_ap.to_csv('labels_ap.csv')
 P_micro, R_micro, _ = precision_recall_curve(y_true2.ravel() , y_pred2.ravel())
 AP_micro = average_precision_score(y_true2, y_pred2, average='micro')
 fig = plt.figure(figsize=(8,11))
ax = fig.add_subplot(1,1,1)
 df_ap.sort_values('AP' ).plot.barh(
 title='Average Precision per Labels(μAP=%.5f)' % AP_micro,
grid=True, legend=False, xlim=(0.,1.) , ax=ax, 
)
 plt.tight_layout()
plt.show()
X_test_compare = pd.DataFrame(data=X_test, columns=["ID", "shop_id", "item_id"])
 X_test_compare.reset_index(inplace=True, drop=True)
 X_test_compare["ID"]=X_test_compare.index
print(test.columns, test.shape, test.head())
 print(X_test_compare.columns, X_test_compare.shape, X_test_compare.head())
 print("Are the same",(test == X_test_compare ).value_counts() )<predict_on_test>
MPLoader.reset()
MPLoader.full_load(df_test, use_preprocess=False )
reg = make_pipeline(StandardScaler() , SGDRegressor(loss="epsilon_insensitive"))
reg.fit(X_train.values, y_train)
 pred_reg = reg.predict(X_test.values ).clip(0,20 )<compute_test_metric>
USE_MASK_FREQ = USE_MASK_TIME = False
test = ImageList.from_df(df_test, WORK, folder='')
 learn = load_learner(WORK, test=test)if TRAIN_MODE else load_learner('.', DEPLOYED_MODEL, test=test)
 preds, _ = learn.TTA(ds_type=DatasetType.Test, num_pred=50 )
print("r2 train: ", r2_score(y_train, reg.predict(X_train.values ).clip(0,20)) )<load_pretrained>
df_submission[learn.data.classes] = preds
df_submission.to_csv('submission.csv', index=False)
 df_submission.head()