Kaggle competition: Indoor Location & Navigation
<SOS> metric: IndoorLocalization Kaggle data source: indoor-location-&-navigation<count_values>
import numpy as np
import pandas as pd
import scipy.stats as stats
from pathlib import Path
import glob
import pickle
import psutil
import random
import os
import time
import sys
import math
from contextlib import contextmanager
from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.decomposition import PCA
import tensorflow as tf
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
import tensorflow.keras.backend as K
import tensorflow_addons as tfa
from tensorflow_addons.layers import WeightNormalization
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping
train_data['Survived'].value_counts()<data_type_conversions>
MODEL_NAME='TF_comp_user_fix'
SEED = 47
N_SPLITS = 10
NUM_FEATS = 100
N_COMPONENTS = 15
INFERENCE_MODE = True
MODEL_DATASET='indoor-models'
TARGET_FOLDS = [0]
MAX_EPOCHS = 36
PLATEAU = 0
TRIAL_ROUND = 0
cat_features = ['Pclass', 'SibSp', 'Parch', 'Embarked']
train_data[cat_features] = train_data[cat_features].astype('O')
 test_data[cat_features] = test_data[cat_features].astype('O' )<prepare_x_and_y>
set_seed(SEED )
features = ["Sex", "Pclass", "SibSp", "Parch", "Age", 'Embarked']
y = train_data["Survived"]
X = train_data[features]
X_test = test_data[features]
X.head()<categorify>
with open(f'.. /input/indoor-with-delta/train_all.pkl', 'rb')as f:
data = pickle.load(f)
 with open(f'.. /input/indoor-interpolated-with-gap/test_all.pkl', 'rb')as f:
test_data = pickle.load(f )
X = pd.get_dummies(X)
 X_test = pd.get_dummies(X_test )<drop_column>
BSSID_FEATS = [f'bssid_{i}' for i in range(NUM_FEATS)]
RSSI_FEATS = [f'rssi_{i}' for i in range(NUM_FEATS)]
GAP_FEATS = [f'gap_{i}' for i in range(NUM_FEATS)]
DELTA_FEATS = ['delta_x_hat', 'delta_y_hat']
X_test.drop(columns=['Parch_9'], inplace=True )<normalization>
data['site_id'] = data['site']
si = SimpleImputer()
X_imp = si.fit_transform(X)
 X_test_imp = si.transform(X_test )<choose_model_class>
wifi_bssids = []
for i in BSSID_FEATS:
wifi_bssids.extend(data.loc[:,i].values.tolist())
 wifi_bssids = list(set(wifi_bssids))
wifi_bssids_size = len(wifi_bssids)
 print(f'BSSID TYPES: {wifi_bssids_size}')
 wifi_bssids_test = []
for i in BSSID_FEATS:
wifi_bssids_test.extend(test_data.loc[:,i].values.tolist())
 wifi_bssids_test = list(set(wifi_bssids_test))
wifi_bssids_size = len(wifi_bssids_test)
 print(f'BSSID TYPES: {wifi_bssids_size}' )
imputer = KNNImputer(n_neighbors=2 )<categorify>
wifi_bssids.extend(wifi_bssids_test)
 wifi_bssids = list(set(wifi_bssids))
wifi_bssids_size = len(wifi_bssids )
pca = PCA(n_components = 10,random_state=42)
 X_pca = pca.fit_transform(X_imp)
 X_test_pca = pca.transform(X_test_imp )<normalization>
le = LabelEncoder()
le.fit(wifi_bssids)
 le_site = LabelEncoder()
le_site.fit(data['site_id'])
 data.loc[:, 'site_id'] = le_site.transform(data.loc[:, 'site_id'])
 for i in BSSID_FEATS:
data.loc[:,i] = le.transform(data.loc[:,i])
 data.loc[:,i] = data.loc[:,i] + 1
test_data.loc[:, 'site_id'] = le_site.transform(test_data.loc[:, 'site_id'])
 for i in BSSID_FEATS:
test_data.loc[:,i] = le.transform(test_data.loc[:,i])
 test_data.loc[:,i] = test_data.loc[:,i] + 1
site_count = len(data['site_id'].unique() )
sclr_pca = MinMaxScaler()
X_min_pca = sclr_pca.fit_transform(X_pca)
 X_test_min_pca = sclr_pca.transform(X_test_pca )<normalization>
sort = data.sort_values(['path', 'timestamp'])
 sort['x_shift'] = sort.groupby(['path'])['x'].shift()
sort['y_shift'] = sort.groupby(['path'])['y'].shift()
sort['dist'] = sort.apply(lambda x: math.sqrt(x['delta_x_hat'] ** 2 + x['delta_y_hat'] ** 2), axis = 1)
 sort = sort.sort_index()
sort = sort[sort['dist'] >= 25][['x', 'x_shift', 'delta_x_hat', 'y', 'y_shift', 'delta_y_hat', 'dist']]
sort['delta_x_hat'] = sort.apply(lambda x: x['x'] - x['x_shift'], axis=1)
 sort['delta_y_hat'] = sort.apply(lambda x: x['y'] - x['y_shift'], axis=1)
 data.loc[sort.index, 'delta_x_hat'] = sort['delta_x_hat'].values
data.loc[sort.index, 'delta_y_hat'] = sort['delta_y_hat'].values
sclr = MinMaxScaler()
X_min = sclr.fit_transform(X_imp)
 X_test_min = sclr.transform(X_test_imp )<import_modules>
ss = StandardScaler()
ss.fit(data[DELTA_FEATS])
 ss.transform(data[DELTA_FEATS])
 data[DELTA_FEATS] = ss.transform(data[DELTA_FEATS] )
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold
from xgboost.sklearn import XGBClassifier<import_modules>
a = data[BSSID_FEATS]
a.columns = [str(i)for i in range(len(BSSID_FEATS)) ]
b = data[RSSI_FEATS]
b.columns = [str(i)for i in range(len(BSSID_FEATS)) ]
x = a.mask(b == -999, 0)
 x.columns = BSSID_FEATS
data[BSSID_FEATS] = x
a = test_data[BSSID_FEATS]
a.columns = [str(i)for i in range(len(BSSID_FEATS)) ]
b = test_data[RSSI_FEATS]
b.columns = [str(i)for i in range(len(BSSID_FEATS)) ]
x = a.mask(b == -999, 0)
 x.columns = BSSID_FEATS
test_data[BSSID_FEATS] = x
from tensorflow import keras
from tensorflow.keras import layers<split>
all_rssis = data['rssi_0']
for i in RSSI_FEATS[1:]:
all_rssis = pd.concat([all_rssis, data[i]])
 ss = StandardScaler()
ss.fit(pd.DataFrame(all_rssis))
for i in RSSI_FEATS:
data.loc[:,i] = ss.transform(pd.DataFrame(data.loc[:,i]))
test_data.loc[:,i] = ss.transform(pd.DataFrame(test_data.loc[:,i]))
all_rssis = data['gap_0']
for i in GAP_FEATS[1:]:
all_rssis = pd.concat([all_rssis, data[i]])
 ss = StandardScaler()
ss.fit(pd.DataFrame(all_rssis))
for i in GAP_FEATS:
data.loc[:,i] = ss.transform(pd.DataFrame(data.loc[:,i]))
test_data.loc[:,i] = ss.transform(pd.DataFrame(test_data.loc[:,i]))
X_train_pca, X_valid_pca, y_train_pca, y_valid_pca = train_test_split(X_min, keras.utils.to_categorical(y), test_size=0.20, random_state=42 )<split>
PCA_COLUMNS = [f'rssi_pca_{i}' for i in range(N_COMPONENTS)]
pca = PCA(n_components=N_COMPONENTS, random_state=SEED)
 pca.fit(data.loc[:,RSSI_FEATS])
 data_pca = pd.DataFrame(pca.transform(data.loc[:,RSSI_FEATS]))
data_pca.columns = PCA_COLUMNS
data = pd.concat([data, data_pca], axis=1)
 test_pca = pd.DataFrame(pca.transform(test_data.loc[:,RSSI_FEATS]))
test_pca.columns = PCA_COLUMNS
test_data = pd.concat([test_data, test_pca], axis=1 )
X_train_min, X_valid_min, y_train_min, y_valid_min = train_test_split(X_min, keras.utils.to_categorical(y), test_size=0.20, random_state=42 )<choose_model_class>
floor_count = len(data['floor'].unique())
 data['floor'] = data['floor'].astype(int)
 y = pd.get_dummies(data.loc[:,'floor'] )
model = keras.Sequential()
model.add(layers.Dense(500, activation='relu', input_dim=X_train_min.shape[1]))
model.add(layers.Dense(100, activation='relu'))
model.add(layers.Dense(50, activation='relu'))
model.add(layers.Dense(2, activation='softmax'))
model.summary()<choose_model_class>
user_id = pd.read_csv('.. /input/retrieving-user-id-from-leaked-wifi-feature/df.csv')
 data = data.merge(user_id[['path_id', 'user_id']], left_on='path', right_on='path_id', how='left')
 test_data['path'] = test_data['site_path_timestamp'].apply(lambda x: x.split('_')[1])
 test_data = test_data.merge(user_id[['path_id', 'user_id']], left_on='path', right_on='path_id', how='left')
 data['user_id'] = data['user_id'] + 1
test_data['user_id'] = test_data['user_id'] + 1
shared_user_ids =(set(data['user_id'].unique())& set(test_data['user_id']))
print(len(shared_user_ids))
data['user_id'] = data['user_id'].apply(lambda x: x if x in shared_user_ids else 0)
 test_data['user_id'] = test_data['user_id'].apply(lambda x: x if x in shared_user_ids else 0)
 key_map = {j: i for(i, j)in enumerate(data['user_id'].unique())}
data['user_id'] = data['user_id'].apply(lambda x: key_map[x])
 test_data['user_id'] = test_data['user_id'].apply(lambda x: key_map[x])
 userid_count = len(shared_user_ids )
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=[keras.metrics.BinaryAccuracy() ] )<train_model>
test_delta = pd.read_csv('.. /input/indoor-with-delta/delta_for_test_from_4.006.csv')
 test_data = test_data.merge(test_delta, on='site_path_timestamp', how='left')
 ss_delta = StandardScaler()
ss_delta.fit(data[DELTA_FEATS])
 ss_delta.transform(data[DELTA_FEATS])
 data[DELTA_FEATS] = ss_delta.transform(data[DELTA_FEATS])
 test_data[DELTA_FEATS] = ss_delta.transform(test_data[DELTA_FEATS] )
history = model.fit(X_train_min, y_train_min, epochs=20, validation_data=(X_valid_min, y_valid_min), verbose=1, batch_size=32 )<predict_on_test>
def create_model(input_data):
input_dim = input_data[0].shape[1]
input_embd_layer = L.Input(shape=(input_dim,))
x1 = L.Embedding(wifi_bssids_size + 1,128, mask_zero=True )(input_embd_layer)
 x1 = L.Flatten()(x1)
 input_site_layer = L.Input(shape=(1,))
x3 = L.Embedding(site_count, 2 )(input_site_layer)
 x3 = L.Flatten()(x3)
 input_dim = input_data[2].shape[1]
input_layer_2 = L.Input(input_dim,)
 x4 = L.BatchNormalization()(input_layer_2)
 x4 = L.Dense(32, activation='swish' )(x4)
 input_dim = input_data[3].shape[1]
input_layer_3 = L.Input(input_dim,)
 x6 = L.BatchNormalization()(input_layer_3)
 x6 = L.Dense(256, activation='swish' )(x6)
 x6 = L.Reshape(( 1, 256))(x6)
 input_userid_layer = L.Input(shape=(1,))
x7 = L.Embedding(userid_count + 1, 4, mask_zero=True )(input_userid_layer)
 x7 = L.Flatten()(x7)
 input_rssi_gap = []
x5 = []
for c in RSSI_FEATS:
_i = L.Input(2,)
 _x5 = L.BatchNormalization()(_i)
 _x5 = L.Dense(1, activation='swish' )(_x5)
 input_rssi_gap.append(_i)
 x5.append(_x5)
 concatenated = L.Concatenate(axis=1 )([x1, x3, x4, x7] + x5)
 concatenated = L.BatchNormalization()(concatenated)
 concatenated = L.Dropout(0.4 )(concatenated)
 concatenated = L.Dense(256, activation='swish' )(concatenated)
 concatenated = L.Reshape(( 1, -1))(concatenated)
 def attention(query_key, res):
l = L.MultiHeadAttention(num_heads=4, key_dim=4, dropout=0.5 )(query_key, query_key)
 l = L.LayerNormalization(epsilon=1e-6 )(res + l)
 ffl = L.BatchNormalization()(l)
 ffl = L.Dropout(0.4 )(ffl)
 ffl = L.Dense(256, activation='relu' )(ffl)
 ffl = L.BatchNormalization()(ffl)
 ffl = L.Dropout(0.3 )(ffl)
 ffl = L.Dense(64, activation='relu' )(ffl)
 ffl = L.BatchNormalization()(ffl)
 ffl = L.Dropout(0.5 )(ffl)
 ffl = L.Dense(256, activation='relu' )(ffl)
 l = L.LayerNormalization(epsilon=1e-6 )(l + ffl)
 return l
x = attention(concatenated, concatenated)
 x = attention(x, concatenated)
 x = L.Concatenate(axis=1 )([x, x6])
 x = L.Reshape(( 8, -1))(x)
 x = L.LSTM(64, dropout=0.3, recurrent_dropout=0.3, return_sequences=True, activation='relu' )(x)
 x = L.LSTM(16, dropout=0.1, return_sequences=False, activation='swish' )(x)
 output_layer_1 = L.Dense(2, name='xy' )(x)
 output_layer_2 = L.Dense(11, activation='softmax', name='floor' )(x)
 model = M.Model([input_embd_layer, input_site_layer, input_layer_2, input_layer_3, input_userid_layer] + input_rssi_gap,
[output_layer_1, output_layer_2])
 lr = 0.001 *(0.1 ** PLATEAU)
 print(f'lr:{lr}')
 model.compile(optimizer=tf.optimizers.Adam(lr=lr),
loss='mse', metrics=['mse'])
 return model
mdl_sqn = model.predict(X_test_min )<data_type_conversions>
score_df = pd.DataFrame()
oof = list()
predictions = list()
oof_x, oof_y, oof_f = np.zeros(data.shape[0]), np.zeros(data.shape[0]), np.zeros(data.shape[0])
 preds_x, preds_y = 0, 0
preds_f_arr = np.zeros(( test_data.shape[0], N_SPLITS))
for fold,(trn_idx, val_idx)in enumerate(StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED ).split(data.loc[:, 'path'], data.loc[:, 'path'])) :
if(not INFERENCE_MODE)&(fold not in TARGET_FOLDS):
continue
_X_train = data.loc[trn_idx, :]
X_train = [_X_train.loc[:,BSSID_FEATS], _X_train.loc[:,'site_id'], _X_train.loc[:,PCA_COLUMNS], _X_train.loc[:,DELTA_FEATS], _X_train.loc[:,'user_id']]
for r, g in zip(RSSI_FEATS, GAP_FEATS):
X_train.append(pd.DataFrame(_X_train.loc[:,r].values, _X_train.loc[:,g].values ).reset_index(drop=False))
y_trainx = data.loc[trn_idx, 'x']
y_trainy = data.loc[trn_idx, 'y']
y_trainf = y.loc[trn_idx, :]
tmp = pd.concat([y_trainx, y_trainy], axis=1)
 y_train = [tmp, y_trainf]
_X_valid = data.loc[val_idx, :]
X_valid = [_X_valid.loc[:,BSSID_FEATS], _X_valid.loc[:,'site_id'], _X_valid.loc[:,PCA_COLUMNS], _X_valid.loc[:,DELTA_FEATS], _X_valid.loc[:,'user_id']]
for r, g in zip(RSSI_FEATS, GAP_FEATS):
X_valid.append(pd.DataFrame(_X_valid.loc[:,r].values, _X_valid.loc[:,g].values ).reset_index(drop=False))
y_validx = data.loc[val_idx, 'x']
y_validy = data.loc[val_idx, 'y']
y_validf = y.loc[val_idx, :]
tmp = pd.concat([y_validx, y_validy], axis=1)
 y_valid = [tmp, y_validf]
with timer("fit"):
model = create_model(X_train)
 if not INFERENCE_MODE:
if TRIAL_ROUND >= 1:
model.load_weights(f'.. /input/{MODEL_DATASET}/{MODEL_NAME}_{SEED}_{fold}.hdf5')
 model.fit(X_train, y_train,
validation_data=(X_valid, y_valid),
batch_size=64, epochs=MAX_EPOCHS + TRIAL_ROUND*MAX_EPOCHS, initial_epoch=TRIAL_ROUND*MAX_EPOCHS,
callbacks=[
ReduceLROnPlateau(monitor='val_xy_loss', factor=0.1, patience=6, verbose=1, min_delta=1e-4, mode='min')
 , ModelCheckpoint(f'{MODEL_NAME}_{SEED}_{fold}.hdf5', monitor = 'val_xy_loss', verbose = 0, save_best_only=True, save_weights_only=True, mode='min')
 , ModelCheckpoint(f'{MODEL_NAME}_{SEED}_{fold}_latest.hdf5', monitor = 'val_xy_loss', verbose = 0, save_best_only=False, save_weights_only=True, mode='min')
 , EarlyStopping(monitor='val_xy_loss', min_delta=1e-4, patience=10, mode='min', baseline = None, restore_best_weights = True)
 ])
 if INFERENCE_MODE:
model.load_weights(f'.. /input/{MODEL_DATASET}/{MODEL_NAME}_{SEED}_{fold}.hdf5')
 val_pred = model.predict(X_valid)
 oof_x[val_idx] = val_pred[0][:,0]
oof_y[val_idx] = val_pred[0][:,1]
_test_data = [test_data.loc[:,BSSID_FEATS], test_data.loc[:,'site_id'], test_data.loc[:,PCA_COLUMNS], test_data.loc[:,DELTA_FEATS], test_data.loc[:,'user_id']]
for r, g in zip(RSSI_FEATS, GAP_FEATS):
_test_data.append(pd.DataFrame(test_data.loc[:,r].values, test_data.loc[:,g].values ).reset_index(drop=False))
pred = model.predict(_test_data)
 preds_x += pred[0][:,0]
preds_y += pred[0][:,1]
sqn_pred =(mdl_sqn[:, 0] < mdl_sqn[:, 1] ).astype(int )<choose_model_class>
def metrics(output_way, output_floor, way, floor):
first_term = np.mean(np.sqrt(np.sum(( output_way - way)**2, axis = 1)))
 second_term = 15 * np.mean(np.abs(output_floor - floor))
return first_term, second_term
def compute_cv_score(oof_):
output_way = oof_[['pred_x', 'pred_y']].values
output_floor = oof_['pred_floor'].values
way = oof_[['true_x', 'true_y']].values
floor = oof_['true_floor'].values
loss_waypoints, loss_floor = metrics(output_way, output_floor, way, floor)
 return loss_waypoints, loss_floor
params = {
'degree' : list(range(1, 10)) ,
'gamma': ['scale', 'auto'],
'kernel':('linear', 'poly', 'rbf', 'sigmoid'),
'class_weight' :('balanced', None),
'C': list(range(1, 5))
}
gsearch = GridSearchCV(svm.SVC(random_state=42),
n_jobs=5,
cv = KFold(random_state=42, shuffle=True, n_splits=10),
param_grid=params, verbose=3,
scoring='accuracy' )<define_search_space>
if INFERENCE_MODE:
assess = pd.DataFrame()
assess['pred_x'] = oof_x
assess['pred_y'] = oof_y
assess['pred_floor'] = data['floor'].values
assess['true_x'] = data['x'].values
assess['true_y'] = data['y'].values
assess['true_floor'] = data['floor'].values
cv = compute_cv_score(assess)
 print(cv )
<save_to_csv><EOS> 
if INFERENCE_MODE:
preds_x /= N_SPLITS
preds_y /= N_SPLITS
sub = pd.read_csv('.. /input/indoor-location-navigation/sample_submission.csv')
 sub['x'] = preds_x
sub['y'] = preds_y
del sub['floor']
sub['path'] = sub['site_path_timestamp'].apply(lambda x: x.split('_')[1])
 floor = pd.read_csv('.. /input/indoor-floor-prediction/floor_pred_0507.csv' ).reset_index(drop=True)[['path', 'floor']]
sub = sub.merge(floor, on=['path'], how='left')
 sub[['site_path_timestamp', 'floor', 'x', 'y']].to_csv('submission.csv', index=False )