Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<choose_model_class>
%matplotlib inline
sns.set(style='white', context='notebook', palette='deep')
 np.random.seed(9 )
class ILNnet(nn.Module):
def __init__(self, input_dim, id_num_embd, id_embd_dim,
strg_dim, site_num_embd, imu_dim,
ninp, nhead, nhid, nlayers, dropout):
super(ILNnet, self ).__init__()

 self.id_embd = nn.Embedding(id_num_embd, id_embd_dim,
padding_idx=0)
 self.strg_ln = nn.LayerNorm(input_dim)
 self.strg_lin = nn.Linear(input_dim, strg_dim)
 self.site_embd = nn.Embedding(site_num_embd, ninp)
 cat_dim = input_dim*id_embd_dim + strg_dim
self.main_seq1 = nn.Sequential(nn.Linear(cat_dim, ninp),
nn.LayerNorm(ninp),
nn.ReLU() ,
nn.Dropout(dropout))
self.dltimu_lin = nn.Linear(4+imu_dim, ninp)

self.pos_encoder = PositionalEncoding(ninp, dropout)
 self.attn_wt_coef = nn.Parameter(torch.clamp(torch.randn(nhead), min=1e-3))
self.customTR1 = CustomEncoderLayer(ninp, nhead, nhid, dropout)
 self.customTR2 = CustomEncoderLayer(ninp, nhead, nhid, dropout)

self.lstm1 = nn.LSTM(batch_first=True,
bidirectional=True, num_layers=1,
input_size=ninp, hidden_size=ninp//2)
 self.lstm2 = nn.LSTM(batch_first=True,
bidirectional=True, num_layers=1,
input_size=ninp, hidden_size=ninp//2)
 self.main_seq2 = nn.Sequential(nn.Linear(ninp,ninp),
nn.LayerNorm(ninp),
nn.Dropout(dropout),
nn.Linear(ninp,2))
self.init_weights()
def init_weights(self):
self.id_embd.weight.data.zero_()
self.site_embd.weight.data.zero_()
def forward(self, x_id, x_strg, x_dlt, x_imu, x_site, x_pos, x_mask):

 x_id = self.id_embd(x_id)
 x_id = rearrange(x_id, 'bn seq d1 d2 -> bn seq(d1 d2)')
 x_strg = self.strg_ln(x_strg)
 x_strg = torch.relu(self.strg_lin(x_strg))
x_site = self.site_embd(x_site)
 x = torch.cat(( x_id, x_strg), dim=-1)
 x = self.main_seq1(x)+ x_site

 x_dltimu = torch.cat(( x_dlt, x_imu),-1)
 x_dltimu = torch.relu(self.dltimu_lin(x_dltimu))
x = x + x_dltimu

 xt = self.pos_encoder(x, None, x_mask)
 attn_wt =(rearrange(x_pos, 'bn seq -> bn 1 seq 1')
 -rearrange(x_pos, 'bn seq -> bn 1 1 seq')).abs()
attn_coef = rearrange(self.attn_wt_coef, 'nh -> 1 nh 1 1')
 xt = self.customTR1(xt, x_mask, torch.log(1.0 + attn_wt)* attn_coef)
 xt = self.customTR2(xt, x_mask, torch.log(1.0 + attn_wt)* attn_coef)
 x = x + xt

 lengths = x_mask.logical_not().sum(dim=1 ).to('cpu')
 x = rnn.pack_padded_sequence(x, lengths, batch_first=True,
enforce_sorted=False)
 x,_ = self.lstm1(x)
 x,_ = self.lstm2(x)
 x = rnn.pad_packed_sequence(x, batch_first=True)[0]
if x.size(1)!=x_mask.size(1):
pad_len = x_mask.size(1)-x.size(1)
 x = torch.cat([x, torch.zeros(x.size(0),pad_len,x.size(2)).to(device)],
dim=1)
 assert x.size(1)==x_mask.size(1)
 x = self.main_seq2(x)
 return x<compute_test_metric>
import matplotlib.image as mpimg
from sklearn.metrics import confusion_matrix
import itertools
import tensorflow as tf
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense,Dropout,Flatten,Conv2D,MaxPool2D,BatchNormalization
from keras.optimizers import RMSprop
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ReduceLROnPlateau
from keras.callbacks import LearningRateScheduler
def ILN_loss(output, filter_, delta, label):
metr1 =(output[:,:,:2]-label[:,:,:2] ).square()
metr1 = metr1.sum(dim=-1)+1e-6
metr1 = metr1.sqrt().sum()
metric = metr1/filter_.sum()
loss = metric
return loss, metric<init_hyperparams>
if tf.test.gpu_device_name() :
print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))
else:
print("Please install GPU version of TF")
 print(tf.test.is_built_with_cuda() )
class LearningRateScheduler:
def __init__(self, lr:list,
switch_epoch: list):
self.lr = lr
self.switch_epoch = switch_epoch
def __call__(self, epoch:int):
idx = [i>epoch for i
in self.switch_epoch+[1e9]].index(True)
 return self.lr[idx]<train_model>
train=pd.read_csv('.. /input/digit-recognizer/train.csv')
 test=pd.read_csv('.. /input/digit-recognizer/test.csv' )
def train_val(net, trainloader, valloader, bsz):

 loss_sum, metric_sum = 0., 0.
 net.train()
optimizer.zero_grad()
with tqdm(trainloader, ncols=80)as pbar:
prefix = f"epoch {i_epoch+1} train"
pbar.set_description(prefix)
 descpost = None
for i,(tr_id, tr_strg, tr_dlt, tr_imu, tr_site, tr_pos,
tr_mask, tr_xy)in enumerate(pbar):
output = net(tr_id, tr_strg, tr_dlt, tr_imu, tr_site, tr_pos, tr_mask)
 filter_ = tr_mask.logical_not()
output *= filter_.unsqueeze(-1)
 loss, metric = ILN_loss(output, filter_, tr_dlt, tr_xy)
 loss.backward()
optimizer.step()
optimizer.zero_grad()
loss_sum += loss.item()
metric_sum += metric.item()
descpost = f'{loss_sum/(i+1):.2f}/{metric_sum/(i+1):.2f}'
pbar.set_postfix({"l/m":descpost})
 history['epoch'].append(i_epoch+1)
 history['train_loss'].append(loss_sum/(i+1))
history['train_metric'].append(metric_sum/(i+1))

 with torch.no_grad() :
loss_sum, metric_sum = 0., 0.
 net.eval()
with tqdm(valloader, ncols=80)as pbar:
prefix = ' '*(len(prefix)-10)+'validation'
pbar.set_description(prefix)
 descpost = None
for i,(vl_id, vl_strg, vl_dlt, vl_imu, vl_site, vl_pos, vl_mask, vl_xy)\
in enumerate(pbar):
output = net(vl_id, vl_strg, vl_dlt, vl_imu, vl_site, vl_pos, vl_mask)
 filter_ = vl_mask.logical_not()
output *= filter_.unsqueeze(-1)
 loss, metric = ILN_loss(output, filter_, vl_dlt, vl_xy)
 loss_sum += loss.item()
metric_sum += metric.item()
descpost = f'{loss_sum/(i+1):.2f}/{metric_sum/(i+1):.2f}'
pbar.set_postfix({"l/m":descpost})
 history['val_loss'].append(loss_sum/(i+1))
history['val_metric'].append(metric_sum/(i+1))
lr_scheduler.step()
return net, history['val_metric'][-1]<categorify>
y_train=train['label']
X_train=train.drop(columns='label' )
te_id, te_strg, te_dlt, te_imu, te_site, te_pos, te_mask = \
[],[],[],[],[],[],[]
for gid, grp in tqdm(test_data.groupby('seq_label'), ncols=60):
te_id.append(torch.from_numpy(grp[ID_FEATS].values ).long())
 te_strg.append(torch.from_numpy(grp[STRG_FEATS].values ).float())
 te_dlt.append(torch.from_numpy(grp[DELTA_FEATS].values ).float())
 te_imu.append(torch.from_numpy(grp[IMU_FEATS].values ).float())
 te_site.append(torch.from_numpy(grp['site'].values ).long())
 te_pos.append(torch.from_numpy(grp['len_pos'].values ).float())

te_mask.append(te_id[-1].sum(dim=-1)==0)
 te_id = rnn.pad_sequence(te_id, batch_first=True ).to(device)
 te_strg = rnn.pad_sequence(te_strg, batch_first=True ).to(device)
 te_dlt = rnn.pad_sequence(te_dlt, batch_first=True ).to(device)
 te_imu = rnn.pad_sequence(te_imu, batch_first=True ).to(device)
 te_site = rnn.pad_sequence(te_site, batch_first=True ).to(device)
 te_pos = rnn.pad_sequence(te_pos, batch_first=True ).to(device)
 te_mask = rnn.pad_sequence(te_mask, batch_first=True,
padding_value=True ).to(device )<import_modules>
del train
import numpy as np
import pandas as pd
import scipy.stats as stats
from pathlib import Path
import glob
import pickle
import psutil
import random
import os
import time
import sys
import math
from contextlib import contextmanager
from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.decomposition import PCA
import tensorflow as tf
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
import tensorflow.keras.backend as K
import tensorflow_addons as tfa
from tensorflow_addons.layers import WeightNormalization
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping<define_variables>
X_train.isna().any().sum()
MODEL_NAME='TF_comp_user_fix'
SEED = 47
N_SPLITS = 10
NUM_FEATS = 100
N_COMPONENTS = 15
INFERENCE_MODE = True
MODEL_DATASET='indoor-models'
TARGET_FOLDS = [0]
MAX_EPOCHS = 36
PLATEAU = 0
TRIAL_ROUND = 0<define_variables>
test.isna().any().sum()
set_seed(SEED )<load_pretrained>
X_train /=255.0
test /=255.0
with open(f'.. /input/indoor-with-delta/train_all.pkl', 'rb')as f:
data = pickle.load(f)
 with open(f'.. /input/indoor-interpolated-with-gap/test_all.pkl', 'rb')as f:
test_data = pickle.load(f )<define_variables>
y_train=to_categorical(y_train,num_classes=10 )
BSSID_FEATS = [f'bssid_{i}' for i in range(NUM_FEATS)]
RSSI_FEATS = [f'rssi_{i}' for i in range(NUM_FEATS)]
GAP_FEATS = [f'gap_{i}' for i in range(NUM_FEATS)]
DELTA_FEATS = ['delta_x_hat', 'delta_y_hat']<feature_engineering>
data['site_id'] = data['site']<concatenate>
nets=6
model=[0]*nets
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
wifi_bssids = []
for i in BSSID_FEATS:
wifi_bssids.extend(data.loc[:,i].values.tolist())
 wifi_bssids = list(set(wifi_bssids))
wifi_bssids_size = len(wifi_bssids)
 print(f'BSSID TYPES: {wifi_bssids_size}')
 wifi_bssids_test = []
for i in BSSID_FEATS:
wifi_bssids_test.extend(test_data.loc[:,i].values.tolist())
 wifi_bssids_test = list(set(wifi_bssids_test))
wifi_bssids_size = len(wifi_bssids_test)
 print(f'BSSID TYPES: {wifi_bssids_size}' )<concatenate>
model[0]=Sequential()
model[0].add(Conv2D(filters=32,kernel_size=(5,5),padding='Same',activation='relu',input_shape=(28,28,1)))
 model[0].add(Conv2D(filters=32,kernel_size=(5,5),padding='Same',activation='relu',input_shape=(28,28,1)))
 model[0].add(MaxPool2D(pool_size=(2,2)))
 model[0].add(Dropout(0.25))
model[0].add(Conv2D(filters=64,kernel_size=(3,3),padding='Same',activation='relu',input_shape=(28,28,1)))
 model[0].add(Conv2D(filters=64,kernel_size=(3,3),padding='Same',activation='relu',input_shape=(28,28,1)))
 model[0].add(MaxPool2D(pool_size=(2,2)))
 model[0].add(Dropout(0.25))
model[0].add(Flatten())
 model[0].add(Dense(256,activation='relu'))
model[0].add(Dropout(0.5))
model[0].add(Dense(10,activation='softmax'))
model[0].compile(optimizer=optimizer,loss="categorical_crossentropy",metrics=['accuracy'] )
wifi_bssids.extend(wifi_bssids_test)
 wifi_bssids = list(set(wifi_bssids))
wifi_bssids_size = len(wifi_bssids )<categorify>
model[1]=Sequential()
model[1].add(Conv2D(filters=32,kernel_size=(5,5),padding='Same',activation='relu',input_shape=(28,28,1)))
 model[1].add(Conv2D(filters=32,kernel_size=(5,5),padding='Same',activation='relu',input_shape=(28,28,1)))
 model[1].add(MaxPool2D(pool_size=(2,2)))
 model[1].add(Dropout(0.25))
model[1].add(Conv2D(filters=64,kernel_size=(3,3),padding='Same',activation='relu',input_shape=(28,28,1)))
 model[1].add(Conv2D(filters=64,kernel_size=(3,3),padding='Same',activation='relu',input_shape=(28,28,1)))
 model[1].add(MaxPool2D(pool_size=(2,2)))
 model[1].add(Dropout(0.25))
model[1].add(Flatten())
 model[1].add(Dense(256,activation='relu'))
model[1].add(Dropout(0.5))
model[1].add(Dense(10,activation='softmax'))
model[1].compile(optimizer=optimizer,loss="categorical_crossentropy",metrics=['accuracy'] )
le = LabelEncoder()
le.fit(wifi_bssids)
 le_site = LabelEncoder()
le_site.fit(data['site_id'])
 data.loc[:, 'site_id'] = le_site.transform(data.loc[:, 'site_id'])
 for i in BSSID_FEATS:
data.loc[:,i] = le.transform(data.loc[:,i])
 data.loc[:,i] = data.loc[:,i] + 1
test_data.loc[:, 'site_id'] = le_site.transform(test_data.loc[:, 'site_id'])
 for i in BSSID_FEATS:
test_data.loc[:,i] = le.transform(test_data.loc[:,i])
 test_data.loc[:,i] = test_data.loc[:,i] + 1
site_count = len(data['site_id'].unique() )<feature_engineering>
model[2]=Sequential()
model[2].add(Conv2D(filters=32,kernel_size=(5,5),padding='Same',activation='relu',input_shape=(28,28,1)))
 model[2].add(BatchNormalization())
 model[2].add(Conv2D(filters=32,kernel_size=(5,5),padding='Same',activation='relu',input_shape=(28,28,1)))
 model[2].add(BatchNormalization())
 model[2].add(MaxPool2D(pool_size=(2,2)))
 model[2].add(Dropout(0.25))
model[2].add(Conv2D(filters=64,kernel_size=(3,3),padding='Same',activation='relu',input_shape=(28,28,1)))
 model[2].add(BatchNormalization())
 model[2].add(Conv2D(filters=64,kernel_size=(3,3),padding='Same',activation='relu',input_shape=(28,28,1)))
 model[2].add(BatchNormalization())
 model[2].add(MaxPool2D(pool_size=(2,2)))
 model[2].add(Dropout(0.25))
model[2].add(Flatten())
 model[2].add(Dense(256,activation='relu'))
model[2].add(BatchNormalization())
 model[2].add(Dropout(0.5))
model[2].add(Dense(10,activation='softmax'))
model[2].compile(optimizer=optimizer,loss="categorical_crossentropy",metrics=['accuracy'] )
sort = data.sort_values(['path', 'timestamp'])
 sort['x_shift'] = sort.groupby(['path'])['x'].shift()
sort['y_shift'] = sort.groupby(['path'])['y'].shift()
sort['dist'] = sort.apply(lambda x: math.sqrt(x['delta_x_hat'] ** 2 + x['delta_y_hat'] ** 2), axis = 1)
 sort = sort.sort_index()
sort = sort[sort['dist'] >= 25][['x', 'x_shift', 'delta_x_hat', 'y', 'y_shift', 'delta_y_hat', 'dist']]
sort['delta_x_hat'] = sort.apply(lambda x: x['x'] - x['x_shift'], axis=1)
 sort['delta_y_hat'] = sort.apply(lambda x: x['y'] - x['y_shift'], axis=1)
 data.loc[sort.index, 'delta_x_hat'] = sort['delta_x_hat'].values
data.loc[sort.index, 'delta_y_hat'] = sort['delta_y_hat'].values<normalization>
model[3]=Sequential()
model[3].add(Conv2D(filters=32,kernel_size=(5,5),padding='Same',activation='relu',input_shape=(28,28,1)))
 model[3].add(BatchNormalization())
 model[3].add(Conv2D(filters=32,kernel_size=(5,5),padding='Same',activation='relu',input_shape=(28,28,1)))
 model[3].add(BatchNormalization())
 model[3].add(MaxPool2D(pool_size=(2,2)))
 model[3].add(Dropout(0.25))
model[3].add(Conv2D(filters=64,kernel_size=(3,3),padding='Same',activation='relu',input_shape=(28,28,1)))
 model[3].add(BatchNormalization())
 model[3].add(Conv2D(filters=64,kernel_size=(3,3),padding='Same',activation='relu',input_shape=(28,28,1)))
 model[3].add(BatchNormalization())
 model[3].add(MaxPool2D(pool_size=(2,2)))
 model[3].add(Dropout(0.25))
model[3].add(Flatten())
 model[3].add(Dense(256,activation='relu'))
model[3].add(BatchNormalization())
 model[3].add(Dropout(0.5))
model[3].add(Dense(10,activation='softmax'))
model[3].compile(optimizer=optimizer,loss="categorical_crossentropy",metrics=['accuracy'] )
ss = StandardScaler()
ss.fit(data[DELTA_FEATS])
 ss.transform(data[DELTA_FEATS])
 data[DELTA_FEATS] = ss.transform(data[DELTA_FEATS] )<feature_engineering>
model[4]=Sequential()
model[4].add(Conv2D(filters=64, kernel_size=3, padding='same', activation='relu', input_shape=(28,28,1)))
 model[4].add(Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))
model[4].add(Conv2D(filters=128, kernel_size=3, padding='same', activation='relu'))
model[4].add(MaxPool2D(pool_size=2))
model[4].add(Conv2D(filters=128, kernel_size=3, padding='same', activation='relu'))
model[4].add(Conv2D(filters=192, kernel_size=3, padding='same', activation='relu'))
model[4].add(MaxPool2D(pool_size=2))
model[4].add(Conv2D(filters=192, kernel_size=5, padding='same', activation='relu'))
model[4].add(MaxPool2D(pool_size=2, padding='same'))
model[4].add(Flatten())
 model[4].add(Dense(256, activation='relu'))
model[4].add(Dense(10, activation='softmax'))
model[4].compile(optimizer=optimizer,loss="categorical_crossentropy",metrics=['accuracy'] )
a = data[BSSID_FEATS]
a.columns = [str(i)for i in range(len(BSSID_FEATS)) ]
b = data[RSSI_FEATS]
b.columns = [str(i)for i in range(len(BSSID_FEATS)) ]
x = a.mask(b == -999, 0)
 x.columns = BSSID_FEATS
data[BSSID_FEATS] = x
a = test_data[BSSID_FEATS]
a.columns = [str(i)for i in range(len(BSSID_FEATS)) ]
b = test_data[RSSI_FEATS]
b.columns = [str(i)for i in range(len(BSSID_FEATS)) ]
x = a.mask(b == -999, 0)
 x.columns = BSSID_FEATS
test_data[BSSID_FEATS] = x<normalization>
model[5]=Sequential()
model[5].add(Conv2D(filters=64, kernel_size=3, padding='same', activation='relu', input_shape=(28,28,1)))
 model[5].add(Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))
model[5].add(Conv2D(filters=128, kernel_size=3, padding='same', activation='relu'))
model[5].add(MaxPool2D(pool_size=2))
model[5].add(Conv2D(filters=128, kernel_size=3, padding='same', activation='relu'))
model[5].add(Conv2D(filters=192, kernel_size=3, padding='same', activation='relu'))
model[5].add(MaxPool2D(pool_size=2))
model[5].add(Conv2D(filters=192, kernel_size=5, padding='same', activation='relu'))
model[5].add(MaxPool2D(pool_size=2, padding='same'))
model[5].add(Flatten())
 model[5].add(Dense(256, activation='relu'))
model[5].add(Dense(10, activation='softmax'))
model[5].compile(optimizer=optimizer,loss="categorical_crossentropy",metrics=['accuracy'] )
all_rssis = data['rssi_0']
for i in RSSI_FEATS[1:]:
all_rssis = pd.concat([all_rssis, data[i]])
 ss = StandardScaler()
ss.fit(pd.DataFrame(all_rssis))
for i in RSSI_FEATS:
data.loc[:,i] = ss.transform(pd.DataFrame(data.loc[:,i]))
test_data.loc[:,i] = ss.transform(pd.DataFrame(test_data.loc[:,i]))
all_rssis = data['gap_0']
for i in GAP_FEATS[1:]:
all_rssis = pd.concat([all_rssis, data[i]])
 ss = StandardScaler()
ss.fit(pd.DataFrame(all_rssis))
for i in GAP_FEATS:
data.loc[:,i] = ss.transform(pd.DataFrame(data.loc[:,i]))
test_data.loc[:,i] = ss.transform(pd.DataFrame(test_data.loc[:,i]))<concatenate>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
PCA_COLUMNS = [f'rssi_pca_{i}' for i in range(N_COMPONENTS)]
pca = PCA(n_components=N_COMPONENTS, random_state=SEED)
 pca.fit(data.loc[:,RSSI_FEATS])
 data_pca = pd.DataFrame(pca.transform(data.loc[:,RSSI_FEATS]))
data_pca.columns = PCA_COLUMNS
data = pd.concat([data, data_pca], axis=1)
 test_pca = pd.DataFrame(pca.transform(test_data.loc[:,RSSI_FEATS]))
test_pca.columns = PCA_COLUMNS
test_data = pd.concat([test_data, test_pca], axis=1 )<categorify>
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x )
floor_count = len(data['floor'].unique())
 data['floor'] = data['floor'].astype(int)
 y = pd.get_dummies(data.loc[:,'floor'] )<feature_engineering>
history = [0] * nets
user_id = pd.read_csv('.. /input/retrieving-user-id-from-leaked-wifi-feature/df.csv')
 data = data.merge(user_id[['path_id', 'user_id']], left_on='path', right_on='path_id', how='left')
 test_data['path'] = test_data['site_path_timestamp'].apply(lambda x: x.split('_')[1])
 test_data = test_data.merge(user_id[['path_id', 'user_id']], left_on='path', right_on='path_id', how='left')
 data['user_id'] = data['user_id'] + 1
test_data['user_id'] = test_data['user_id'] + 1
shared_user_ids =(set(data['user_id'].unique())& set(test_data['user_id']))
print(len(shared_user_ids))
data['user_id'] = data['user_id'].apply(lambda x: x if x in shared_user_ids else 0)
 test_data['user_id'] = test_data['user_id'].apply(lambda x: x if x in shared_user_ids else 0)
 key_map = {j: i for(i, j)in enumerate(data['user_id'].unique())}
data['user_id'] = data['user_id'].apply(lambda x: key_map[x])
 test_data['user_id'] = test_data['user_id'].apply(lambda x: key_map[x])
 userid_count = len(shared_user_ids )<load_from_csv>
epochs = 20
for j in range(nets):
print("CNN ",j+1)
 X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, y_train, test_size = 0.1)
 history[j] = model[j].fit_generator(datagen.flow(X_train2,Y_train2, batch_size=32),
epochs = epochs, steps_per_epoch = X_train2.shape[0]//32,
validation_data =(X_val2,Y_val2), callbacks=[annealer], verbose=1)
 print("CNN {0:d}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}".format(
 j+1,epochs,max(history[j].history['accuracy']),max(history[j].history['val_accuracy'])) )
test_delta = pd.read_csv('.. /input/indoor-with-delta/delta_for_test_from_4.006.csv')
 test_data = test_data.merge(test_delta, on='site_path_timestamp', how='left')
 ss_delta = StandardScaler()
ss_delta.fit(data[DELTA_FEATS])
 ss_delta.transform(data[DELTA_FEATS])
 data[DELTA_FEATS] = ss_delta.transform(data[DELTA_FEATS])
 test_data[DELTA_FEATS] = ss_delta.transform(test_data[DELTA_FEATS] )<choose_model_class>
results_ens = np.zeros(( X_val2.shape[0],10))
for j in range(nets):
results_ens = results_ens + model[j].predict(X_val2)
 results_ens = np.argmax(results_ens,axis =1 )
def create_model(input_data):
input_dim = input_data[0].shape[1]
input_embd_layer = L.Input(shape=(input_dim,))
x1 = L.Embedding(wifi_bssids_size + 1,128, mask_zero=True )(input_embd_layer)
 x1 = L.Flatten()(x1)
 input_site_layer = L.Input(shape=(1,))
x3 = L.Embedding(site_count, 2 )(input_site_layer)
 x3 = L.Flatten()(x3)
 input_dim = input_data[2].shape[1]
input_layer_2 = L.Input(input_dim,)
 x4 = L.BatchNormalization()(input_layer_2)
 x4 = L.Dense(32, activation='swish' )(x4)
 input_dim = input_data[3].shape[1]
input_layer_3 = L.Input(input_dim,)
 x6 = L.BatchNormalization()(input_layer_3)
 x6 = L.Dense(256, activation='swish' )(x6)
 x6 = L.Reshape(( 1, 256))(x6)
 input_userid_layer = L.Input(shape=(1,))
x7 = L.Embedding(userid_count + 1, 4, mask_zero=True )(input_userid_layer)
 x7 = L.Flatten()(x7)
 input_rssi_gap = []
x5 = []
for c in RSSI_FEATS:
_i = L.Input(2,)
 _x5 = L.BatchNormalization()(_i)
 _x5 = L.Dense(1, activation='swish' )(_x5)
 input_rssi_gap.append(_i)
 x5.append(_x5)
 concatenated = L.Concatenate(axis=1 )([x1, x3, x4, x7] + x5)
 concatenated = L.BatchNormalization()(concatenated)
 concatenated = L.Dropout(0.4 )(concatenated)
 concatenated = L.Dense(256, activation='swish' )(concatenated)
 concatenated = L.Reshape(( 1, -1))(concatenated)
 def attention(query_key, res):
l = L.MultiHeadAttention(num_heads=4, key_dim=4, dropout=0.5 )(query_key, query_key)
 l = L.LayerNormalization(epsilon=1e-6 )(res + l)
 ffl = L.BatchNormalization()(l)
 ffl = L.Dropout(0.4 )(ffl)
 ffl = L.Dense(256, activation='relu' )(ffl)
 ffl = L.BatchNormalization()(ffl)
 ffl = L.Dropout(0.3 )(ffl)
 ffl = L.Dense(64, activation='relu' )(ffl)
 ffl = L.BatchNormalization()(ffl)
 ffl = L.Dropout(0.5 )(ffl)
 ffl = L.Dense(256, activation='relu' )(ffl)
 l = L.LayerNormalization(epsilon=1e-6 )(l + ffl)
 return l
x = attention(concatenated, concatenated)
 x = attention(x, concatenated)
 x = L.Concatenate(axis=1 )([x, x6])
 x = L.Reshape(( 8, -1))(x)
 x = L.LSTM(64, dropout=0.3, recurrent_dropout=0.3, return_sequences=True, activation='relu' )(x)
 x = L.LSTM(16, dropout=0.1, return_sequences=False, activation='swish' )(x)
 output_layer_1 = L.Dense(2, name='xy' )(x)
 output_layer_2 = L.Dense(11, activation='softmax', name='floor' )(x)
 model = M.Model([input_embd_layer, input_site_layer, input_layer_2, input_layer_3, input_userid_layer] + input_rssi_gap,
[output_layer_1, output_layer_2])
 lr = 0.001 *(0.1 ** PLATEAU)
 print(f'lr:{lr}')
 model.compile(optimizer=tf.optimizers.Adam(lr=lr),
loss='mse', metrics=['mse'])
 return model<define_variables>
Y_val2=np.argmax(Y_val2,axis =1)
 ens_acc=accuracy_score(results_ens,Y_val2)
score_df = pd.DataFrame()
oof = list()
predictions = list()
oof_x, oof_y, oof_f = np.zeros(data.shape[0]), np.zeros(data.shape[0]), np.zeros(data.shape[0])
 preds_x, preds_y = 0, 0
preds_f_arr = np.zeros(( test_data.shape[0], N_SPLITS))
for fold,(trn_idx, val_idx)in enumerate(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED ).split(data.loc[:, 'path'], data.loc[:, 'path'])) :
if(not INFERENCE_MODE)&(fold not in TARGET_FOLDS):
continue
_X_train = data.loc[trn_idx, :]
X_train = [_X_train.loc[:,BSSID_FEATS], _X_train.loc[:,'site_id'], _X_train.loc[:,PCA_COLUMNS], _X_train.loc[:,DELTA_FEATS], _X_train.loc[:,'user_id']]
for r, g in zip(RSSI_FEATS, GAP_FEATS):
X_train.append(pd.DataFrame(_X_train.loc[:,r].values, _X_train.loc[:,g].values ).reset_index(drop=False))
y_trainx = data.loc[trn_idx, 'x']
y_trainy = data.loc[trn_idx, 'y']
y_trainf = y.loc[trn_idx, :]
tmp = pd.concat([y_trainx, y_trainy], axis=1)
 y_train = [tmp, y_trainf]
_X_valid = data.loc[val_idx, :]
X_valid = [_X_valid.loc[:,BSSID_FEATS], _X_valid.loc[:,'site_id'], _X_valid.loc[:,PCA_COLUMNS], _X_valid.loc[:,DELTA_FEATS], _X_valid.loc[:,'user_id']]
for r, g in zip(RSSI_FEATS, GAP_FEATS):
X_valid.append(pd.DataFrame(_X_valid.loc[:,r].values, _X_valid.loc[:,g].values ).reset_index(drop=False))
y_validx = data.loc[val_idx, 'x']
y_validy = data.loc[val_idx, 'y']
y_validf = y.loc[val_idx, :]
tmp = pd.concat([y_validx, y_validy], axis=1)
 y_valid = [tmp, y_validf]
with timer("fit"):
model = create_model(X_train)
 if not INFERENCE_MODE:
if TRIAL_ROUND >= 1:
model.load_weights(f'.. /input/{MODEL_DATASET}/{MODEL_NAME}_{SEED}_{fold}.hdf5')
 model.fit(X_train, y_train,
validation_data=(X_valid, y_valid),
batch_size=64, epochs=MAX_EPOCHS + TRIAL_ROUND*MAX_EPOCHS, initial_epoch=TRIAL_ROUND*MAX_EPOCHS,
callbacks=[
ReduceLROnPlateau(monitor='val_xy_loss', factor=0.1, patience=6, verbose=1, min_delta=1e-4, mode='min')
 , ModelCheckpoint(f'{MODEL_NAME}_{SEED}_{fold}.hdf5', monitor = 'val_xy_loss', verbose = 0, save_best_only=True, save_weights_only=True, mode='min')
 , ModelCheckpoint(f'{MODEL_NAME}_{SEED}_{fold}_latest.hdf5', monitor = 'val_xy_loss', verbose = 0, save_best_only=False, save_weights_only=True, mode='min')
 , EarlyStopping(monitor='val_xy_loss', min_delta=1e-4, patience=10, mode='min', baseline = None, restore_best_weights = True)
 ])
 if INFERENCE_MODE:
model.load_weights(f'.. /input/{MODEL_DATASET}/{MODEL_NAME}_{SEED}_{fold}.hdf5')
 val_pred = model.predict(X_valid)
 oof_x[val_idx] = val_pred[0][:,0]
oof_y[val_idx] = val_pred[0][:,1]
_test_data = [test_data.loc[:,BSSID_FEATS], test_data.loc[:,'site_id'], test_data.loc[:,PCA_COLUMNS], test_data.loc[:,DELTA_FEATS], test_data.loc[:,'user_id']]
for r, g in zip(RSSI_FEATS, GAP_FEATS):
_test_data.append(pd.DataFrame(test_data.loc[:,r].values, test_data.loc[:,g].values ).reset_index(drop=False))
pred = model.predict(_test_data)
 preds_x += pred[0][:,0]
preds_y += pred[0][:,1]<compute_test_metric>
print('Ensemble accuracy: {:.4f}'.format(ens_acc))
<prepare_output><EOS> 
results = np.zeros(( test.shape[0],10))
for j in range(nets):
results = results + model[j].predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("predictions.csv",index=False )