Kaggle competition: Digit Recognizer
MODE = 1
FUDGE = 2.0
FILE = '.. /input/rfcx-minimal/submission.csv'
df = pd.read_csv(FILE)
 for k in range(24):
df.iloc[:,1+k] -= df.iloc[:,1+k].min()
df.iloc[:,1+k] /= df.iloc[:,1+k].max()
def scale(probs, factor):
probs = probs.copy()
idx = np.where(probs!=1)[0]
odds = factor * probs[idx] /(1-probs[idx])
 probs[idx] = odds/(1+odds)
 return probs
d1 = df.iloc[:,1:].mean().values
d2 = np.array([113,204,44,923,53,41,3,213,44,23,26,149,255,14,123,222,46,6,474,4,17,18,23,72])/1000.
 for k in range(24):
if MODE==1: d = FUDGE
if MODE==2: d = d1[k]/(1-d1[k])
 if MODE==3: s = d2[k] / d1[k]
else: s =(d2[k]/(1-d2[k])) /d
df.iloc[:,k+1] = scale(df.iloc[:,k+1].values,s)
 df.to_csv('submission_with_pp.csv',index=False )<define_variables>
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )
save_to_disk = 0<install_modules>
X_train_raw = train.drop(columns=['label'] ).to_numpy()
y_train_raw = train['label'].to_numpy()
!pip install resnest > /dev/null<normalization>
x_mean = np.mean(X_train_raw)
 x_std = np.std(X_train_raw)
 def standarize(X):
return(X - x_mean)/ x_std
X_train = np.reshape(X_train_raw,(-1,28,28,1))
X_train = standarize(X_train)
 y_train_cat = keras.utils.to_categorical(y_train_raw )
def horizontal_flip(img):
horizontal_flip_img = img[:, ::-1]
return addChannels(horizontal_flip_img)
 def vertical_flip(img):
vertical_flip_img = img[::-1, :]
return addChannels(vertical_flip_img)
 def addNoisy(img):
noise_img = util.random_noise(img)
 return addChannels(noise_img)
 def contrast_stretching(img):
contrast_img = exposure.rescale_intensity(img)
 return addChannels(contrast_img)
 def randomGaussian(img):
gaussian_img = gaussian(img)
 return addChannels(gaussian_img)
 def grayScale(img):
gray_img = rgb2gray(img)
 return addChannels(gray_img)
 def randomGamma(img):
img_gamma = exposure.adjust_gamma(img)
 return addChannels(img_gamma)
 def addChannels(img):
return np.stack(( img, img, img))
def spec_to_image(spec):
spec = resize(spec,(224, 400))
eps=1e-6
mean = spec.mean()
std = spec.std()
spec_norm =(spec - mean)/(std + eps)
 spec_min, spec_max = spec_norm.min() , spec_norm.max()
spec_scaled = 255 *(spec_norm - spec_min)/(spec_max - spec_min)
 spec_scaled = spec_scaled.astype(np.uint8)
 spec_scaled = np.asarray(spec_scaled)
 return spec_scaled<choose_model_class>
data_generator = keras.preprocessing.image.ImageDataGenerator(
 rotation_range=10,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.1 
)
 data_generator.fit(X_train )
def get_model() :
resnet_model = resnest50(pretrained=True)
 num_ftrs = resnet_model.fc.in_features
resnet_model.fc = nn.Linear(num_ftrs, num_birds)
 resnet_model = resnet_model.to(device)
 return resnet_model<define_variables>
def make_model() :
model = keras.models.Sequential()
model.add(keras.layers.Conv2D(32, kernel_size=3,activation='relu',kernel_initializer='he_normal',input_shape=(28, 28, 1)))
 model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.Conv2D(32, kernel_size=3,activation='relu'))
model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.Conv2D(32, kernel_size=5, strides=2, padding='same',activation='relu'))
model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.Dropout(0.4))
model.add(keras.layers.Conv2D(64, kernel_size=3, activation ='relu'))
model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.Conv2D(64, kernel_size=3, activation ='relu'))
model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.Conv2D(64, kernel_size=5, strides=2, padding='same',activation='relu'))
model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.Dropout(0.4))
model.add(keras.layers.Flatten())
 model.add(keras.layers.Dense(128, activation = "relu"))
model.add(keras.layers.BatchNormalization())
 model.add(keras.layers.Dropout(0.4))
model.add(keras.layers.Dense(10, activation = "softmax"))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 return model
class AudioData(Dataset):
def __init__(self, X, y, data_type):
self.data = []
self.labels = []
self.augs = [
addNoisy, contrast_stretching,
randomGaussian, grayScale,
randomGamma, vertical_flip,
horizontal_flip, addChannels
]
self.data_type=data_type
for i in range(0, len(X)) :
recording_id = X[i]
label = y[i]
mel_spec = audio_data[recording_id]['original']
self.data.append(mel_spec)
 self.labels.append(label)
 def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
if self.data_type == "train":
aug= random.choice(self.augs)
 data = aug(self.data[idx])
 else:
data = addChannels(self.data[idx])
 return data, self.labels[idx]<import_modules>
ens_size=16
model = [0]*ens_size
for i in range(ens_size):
model[i] = make_model()
import torch
import torch.nn as nn
import copy<init_hyperparams>
history = [0]*ens_size
start = time.perf_counter()
for i in range(ens_size):
callbacks = [keras.callbacks.ModelCheckpoint('/kaggle/working/mdl-{}-of-{}.hdf5' 
.format(i,ens_size-1),save_best_only=True, monitor='val_accuracy', mode='max')]
X_train_ens, X_valid_ens, y_train_ens, y_valid_ens = train_test_split(X_train, y_train_cat, train_size=0.9)
 train_aug = data_generator.flow(X_train_ens, y_train_ens)
 history[i] = model[i].fit_generator(train_aug, epochs=40,
validation_data=(X_valid_ens, y_valid_ens),
callbacks=callbacks, verbose = 0)
 print("CNN {}: Train accuracy={:0.5f}, Validation accuracy={:0.5f}" 
.format(i,max(history[i].history['accuracy']),max(history[i].history['val_accuracy'])))
 stop = time.perf_counter()
print(f"{stop - start:0.4f}" )
class Adas(Optimizer):
r
def __init__(self, params,
lr = 0.001, lr2 =.005, lr3 =.0005,
beta_1 = 0.999, beta_2 = 0.999, beta_3 = 0.9999,
epsilon = 1e-8, **kwargs):
if not 0.0 <= lr:
raise ValueError("Invalid lr: {}".format(lr))
if not 0.0 <= lr2:
raise ValueError("Invalid lr2: {}".format(lr))
if not 0.0 <= lr3:
raise ValueError("Invalid lr3: {}".format(lr))
if not 0.0 <= epsilon:
raise ValueError("Invalid epsilon value: {}".format(eps))
if not 0.0 <= beta_1 < 1.0:
raise ValueError("Invalid beta_1 parameter: {}".format(betas[0]))
if not 0.0 <= beta_2 < 1.0:
raise ValueError("Invalid beta_2 parameter: {}".format(betas[1]))
if not 0.0 <= beta_3 < 1.0:
raise ValueError("Invalid beta_3 parameter: {}".format(betas[2]))
defaults = dict(lr=lr, lr2=lr2, lr3=lr3, beta_1=beta_1, beta_2=beta_2, beta_3=beta_3, epsilon=epsilon)
 self._varn = None
self._is_create_slots = None
self._curr_var = None
self._lr = lr
self._lr2 = lr2
self._lr3 = lr3
self._beta_1 = beta_1
self._beta_2 = beta_2
self._beta_3 = beta_3
self._epsilon = epsilon
super(Adas, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(Adas, self ).__setstate__(state)
 @torch.no_grad()
def _add(self,x,y):
x.add_(y)
 return x
@torch.no_grad()
def _derivatives_normalizer(self,derivative,beta):
steps = self._make_variable(0,() ,derivative.dtype)
 self._add(steps,1)
 factor =(1.-(self._beta_1 ** steps)).sqrt()
m = self._make_variable(0,derivative.shape,derivative.dtype)
 moments = self._make_variable(0,derivative.shape,derivative.dtype)
 m.mul_(self._beta_1 ).add_(( 1 - self._beta_1)* derivative * derivative)
 np_t = derivative * factor /(m.sqrt() + self._epsilon)
 return(moments,np_t,lambda: moments.mul_(beta ).add_(( 1 - beta)* np_t))
def _make_variable(self,value,shape,dtype):
self._varn += 1
name = 'unnamed_variable' + str(self._varn)
 if self._is_create_slots:
self.state[self._curr_var][name] = torch.full(size=shape,fill_value=value,dtype=dtype,device=self._curr_var.device)
 return self.state[self._curr_var][name]
@torch.no_grad()
def _get_updates_universal_impl(self, grad, param):
lr = self._make_variable(value = self._lr,shape=param.shape[1:], dtype=param.dtype)
 moment, deriv, f = self._derivatives_normalizer(grad,self._beta_3)
 param.add_(- torch.unsqueeze(lr,0)* deriv)
 lr_deriv = torch.sum(moment * grad,0)
 f()
master_lr = self._make_variable(self._lr2,() ,dtype=torch.float32)
 m2,d2, f = self._derivatives_normalizer(lr_deriv,self._beta_2)
 self._add(lr,master_lr * lr * d2)
 master_lr_deriv2 = torch.sum(m2 * lr_deriv)
 f()
m3,d3,f = self._derivatives_normalizer(master_lr_deriv2,0.)
self._add(master_lr,self._lr3 * master_lr * d3)
 f()
@torch.no_grad()
def _get_updates_universal(self, param, grad, is_create_slots):
self._curr_var = param
self._is_create_slots = is_create_slots
self._varn = 0
return self._get_updates_universal_impl(grad,self._curr_var.data)
 @torch.no_grad()
def step(self, closure=None):

 loss = None
if closure is not None:
with torch.enable_grad() :
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data
if grad.is_sparse:
raise RuntimeError('Adas does not support sparse gradients')
 self._get_updates_universal(p,grad,len(self.state[p])== 0)
 return loss<set_options>
X_test = np.reshape(test.to_numpy() ,(-1,28,28,1))
X_test = standarize(X_test )
num_birds = 24
if torch.cuda.is_available() :
device=torch.device('cuda:0')
 else:
device=torch.device('cpu' )<train_model>
preds = np.zeros(( X_test.shape[0],10))
for i in range(ens_size):
mdl = keras.models.load_model('/kaggle/working/mdl-{}-of-{}.hdf5'.format(i,ens_size-1))
preds = preds + mdl.predict(X_test )
learning_rate = 2e-4
epochs = 20
loss_fn = nn.CrossEntropyLoss()
def train(model, loss_fn, train_loader, valid_loader, epochs, optimizer, scheduler):
best_model_wts = copy.deepcopy(model.state_dict())
 best_acc = 0.0
train_losses = []
valid_losses = []
for epoch in tqdm(range(1,epochs+1)) :
model.train()
batch_losses=[]
for i, data in enumerate(train_loader):
x, y = data
optimizer.zero_grad()
x = x.to(device, dtype=torch.float32)
 y = y.to(device, dtype=torch.long)
 y_hat = model(x)
 loss = loss_fn(y_hat, y)
 loss.backward()
batch_losses.append(loss.item())
 optimizer.step()
train_losses.append(batch_losses)
 print(f'Epoch - {epoch} Train-Loss : {np.mean(train_losses[-1])}')
 model.eval()
batch_losses=[]
trace_y = []
trace_yhat = []
for i, data in enumerate(valid_loader):
x, y = data
x = x.to(device, dtype=torch.float32)
 y = y.to(device, dtype=torch.long)
 y_hat = model(x)
 loss = loss_fn(y_hat, y)
 trace_y.append(y.cpu().detach().numpy())
 trace_yhat.append(y_hat.cpu().detach().numpy())
 batch_losses.append(loss.item())
 valid_losses.append(batch_losses)
 trace_y = np.concatenate(trace_y)
 trace_yhat = np.concatenate(trace_yhat)
 accuracy = np.mean(trace_yhat.argmax(axis=1)==trace_y)
 print(f'Epoch - {epoch} Valid-Loss : {np.mean(valid_losses[-1])} Valid-Accuracy : {accuracy}')
 scheduler.step(np.mean(valid_losses[-1]))
if accuracy > best_acc:
best_acc = accuracy
best_model_wts = copy.deepcopy(model.state_dict())
 model.load_state_dict(best_model_wts)
 return model<load_from_csv>
submit_pred = np.argmax(preds,axis=1)
 submit_pred.shape
fft = 2048
hop = 512
sr = 48000
length = 10 * sr
with open('/kaggle/input/rfcx-species-audio-detection/train_tp.csv')as f:
reader = csv.reader(f)
 next(reader, None)
 data = list(reader)
 fmin = 24000
fmax = 0
for i in range(0, len(data)) :
if fmin > float(data[i][4]):
fmin = float(data[i][4])
 if fmax < float(data[i][6]):
fmax = float(data[i][6])
 fmin = int(fmin * 0.9)
 fmax = int(fmax * 1.1)
 print('Minimum frequency: ' + str(fmin)+ ', maximum frequency: ' + str(fmax))
label_list = []
data_list = []
audio_data = {}
for d in data:
recording_id = d[0]
species_id = int(d[1])
 data_list.append(recording_id)
 label_list.append(species_id)
 audio_data[recording_id] = {}
wav, sr = librosa.load('/kaggle/input/rfcx-species-audio-detection/train/' + recording_id + '.flac', sr=None)
 t_min = float(d[3])* sr
t_max = float(d[5])* sr
center = np.round(( t_min + t_max)/ 2)
 beginning = center - length / 2
if beginning < 0:
beginning = 0
ending = beginning + length
if ending > len(wav):
ending = len(wav)
 beginning = ending - length
slice = wav[int(beginning):int(ending)]
spec=librosa.feature.melspectrogram(slice, sr=sr,n_fft=fft,hop_length=hop,fmin=fmin,fmax=fmax)
 spec_db=librosa.power_to_db(spec,top_db=80)
 img = spec_to_image(spec_db)
 audio_data[recording_id]["original"] = img<train_model>
submition = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
 submition['Label'] = submit_pred
nfold = 5
skf = KFold(n_splits=nfold, shuffle=True, random_state=32)
 for fold_id,(train_index, val_index)in enumerate(skf.split(data_list, label_list)) :
print("Fold", fold_id)
 X_train = np.take(data_list, train_index)
 y_train = np.take(label_list, train_index, axis = 0)
 X_val = np.take(data_list, val_index)
 y_val = np.take(label_list, val_index, axis = 0)
 train_data = AudioData(X_train, y_train, "train")
 valid_data = AudioData(X_val, y_val, "valid")
 train_loader = DataLoader(train_data, batch_size=8, shuffle=True, drop_last=True)
 valid_loader = DataLoader(valid_data, batch_size=8, shuffle=True, drop_last=True)
 resnet_model = get_model()
optimizer = Adas(resnet_model.parameters() , lr=learning_rate)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=3)
 resnet_model = train(resnet_model, loss_fn, train_loader, valid_loader, epochs, optimizer, scheduler)
 torch.save(resnet_model.state_dict() , "model"+str(fold_id)+".pt")
 del train_data, valid_data, train_loader, valid_loader, resnet_model, X_train, X_val, y_train, y_val<load_pretrained>
submition.to_csv('/kaggle/working/submition_ens3.csv', index=False )