Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<create_dataframe>
import numpy as np
import pandas as pd
from fastai.vision import *
from fastai.tabular import *
import matplotlib.pyplot as plt
import re
final_output = []
test_dataset = TweetDataset(
 tweet=df_test.text.values,
sentiment=df_test.sentiment.values,
selected_text=df_test.selected_text.values 
)
 data_loader = torch.utils.data.DataLoader(
 test_dataset,
shuffle=False,
batch_size=config.VALID_BATCH_SIZE,
num_workers=1 
)
 with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
orig_selected = d["orig_selected"]
orig_tweet = d["orig_tweet"]
targets_start = d["targets_start"]
targets_end = d["targets_end"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 targets_start = targets_start.to(device, dtype=torch.long)
 targets_end = targets_end.to(device, dtype=torch.long)
 outputs_start1, outputs_end1 = model1(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start2, outputs_end2 = model2(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start3, outputs_end3 = model3(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start4, outputs_end4 = model4(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start5, outputs_end5 = model5(
 ids=ids,
mask=mask,
token_type_ids=token_type_ids 
)
 outputs_start =(
 outputs_start1
+ outputs_start2
+ outputs_start3
+ outputs_start4
+ outputs_start5 
)/ 5
outputs_end =(
 outputs_end1
+ outputs_end2
+ outputs_end3
+ outputs_end4
+ outputs_end5 
)/ 5
outputs_start = torch.softmax(outputs_start, dim=1 ).cpu().detach().numpy()
outputs_end = torch.softmax(outputs_end, dim=1 ).cpu().detach().numpy()
for px, tweet in enumerate(orig_tweet):
selected_tweet = orig_selected[px]
tweet_sentiment = sentiment[px]
_, output_sentence = calculate_jaccard_score(
 original_tweet=tweet,
target_string=selected_tweet,
sentiment_val=tweet_sentiment,
idx_start=np.argmax(outputs_start[px, :]),
idx_end=np.argmax(outputs_end[px, :]),
offsets=offsets[px] 
)
 final_output.append(output_sentence )<string_transform>
path=Path('/kaggle/working/' )
def post_process(selected):
return " ".join(set(selected.lower().split()))<save_to_csv>
train=pd.read_csv('/kaggle/input/train.csv')
 train['index'] = train.index
train.head()
sample = pd.read_csv(".. /input/tweet-sentiment-extraction/sample_submission.csv")
 sample.loc[:, 'selected_text'] = final_output
sample.selected_text = sample.selected_text.map(post_process)
 sample.to_csv("submission.csv", index=False )<define_variables>
tfms=get_transforms(rand_pad(padding=5,size=28,mode='zeros'))
Image(filename=".. /input/iln-dataset/mymodel-1.png",
format='png' )<install_modules>
data =(src.transform(tfms,size=128)
.databunch(num_workers=5,bs=48)
.normalize())
%pip install einops<import_modules>
data.show_batch(rows=3,figsize=(10,7))
import numpy as np
import pandas as pd
import os, random, pickle, gc, time, json, math, copy
from tqdm import tqdm
from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import LabelEncoder, StandardScaler
from matplotlib import pyplot as plt
import torch
from torch import nn, optim
from torch.nn.parameter import Parameter
from torch.nn.utils import rnn
from einops import rearrange<set_options>
learner=cnn_learner(data,models.resnet101,metrics=[FBeta() ,accuracy,error_rate] )
def set_seed(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed )<define_variables>
lr=9e-3
dat_ver = 'ILN_631dat'
model_ver = 'ILN_632'
subm_file = '.. /input/indoor-location-navigation/sample_submission.csv'
N_SPLITS = 10
NUM_FEATS = 40
SEED = 1605
batch_size = 128
inference_only = True
if inference_only:
n_epoch = 1
data_dir = '.. /input/iln-dataset/ILN_train_results/'
else:
n_epoch = 400
data_dir = ''
set_seed(SEED)
 ID_FEATS = [f'id_{i}' for i in range(NUM_FEATS)]
STRG_FEATS = [f'strength_{i}' for i in range(NUM_FEATS)]
IMU_FEATS = ['gyro_x_mean', 'gyro_y_mean', 'gyro_z_mean',
'gyro_x_std', 'gyro_y_std', 'gyro_z_std',
'gyro_x_max', 'gyro_y_max', 'gyro_z_max',
'gyro_x_min', 'gyro_y_min', 'gyro_z_min',
'gyro_x_skew', 'gyro_y_skew', 'gyro_z_skew',
'acce_x_mean', 'acce_y_mean', 'acce_z_mean',
'acce_x_std', 'acce_y_std', 'acce_z_std',
'acce_x_max', 'acce_y_max', 'acce_z_max',
'acce_x_min', 'acce_y_min', 'acce_z_min',
'acce_x_skew', 'acce_y_skew', 'acce_z_skew',
'ahrs_x_mean', 'ahrs_y_mean', 'ahrs_z_mean',
'ahrs_x_std', 'ahrs_y_std', 'ahrs_z_std',
'ahrs_x_max', 'ahrs_y_max', 'ahrs_z_max',
'ahrs_x_min', 'ahrs_y_min', 'ahrs_z_min',
'ahrs_x_skew', 'ahrs_y_skew', 'ahrs_z_skew',
'head_magn_x_mean', 'head_magn_y_mean',
'head_magn_x_std', 'head_magn_y_std',
'head_magn_x_max', 'head_magn_y_max',
'head_magn_x_min', 'head_magn_y_min',
'head_magn_x_skew', 'head_magn_y_skew',
'magn_z_mean', 'magn_z_std',
'magn_z_max', 'magn_z_min', 'magn_z_skew']<define_variables>
learner.fit_one_cycle(5,slice(lr))
floor_map = {"B2":-2, "B1":-1,
"F1":0, "F2":1, "F3":2, "F4":3, "F5":4,
"F6":5, "F7":6, "F8":7, "F9":8,
"1F":0, "2F":1, "3F":2, "4F":3, "5F":4,
"6F":5, "7F":6, "8F":7, "9F":8}
site_list = ['5d2709a003f801723c3251bf','5a0546857ecc773753327266',
'5c3c44b80379370013e0fd2b','5d2709b303f801723c327472',
'5d2709bb03f801723c32852c','5d2709c303f801723c3299ee',
'5d2709d403f801723c32bd39','5d2709e003f801723c32d896',
'5d27075f03f801723c2e360f','5d27096c03f801723c31e5e0',
'5d27097f03f801723c320d97','5d27099f03f801723c32511d',
'5da138b74db8ce0c98bd4774','5da958dd46f8266d0737457b',
'5da1382d4db8ce0c98bbe92e','5da1383b4db8ce0c98bc11ab',
'5da1389e4db8ce0c98bd0547','5da138274db8ce0c98bbd3d2',
'5da138314db8ce0c98bbf3a0','5da138364db8ce0c98bc00f1',
'5da138754db8ce0c98bca82f','5da138764db8ce0c98bcaa46',
'5dbc1d84c1eb61796cf7c010','5dc8cea7659e181adb076a3f']<load_from_csv>
learner.save('Resnet1' )
df_TestTimeLag = pd.read_csv('.. /input/iln-preprocess-time-lag-table-of-test-data/test_ts_lag.csv',index_col=0)
 with open(f'.. /input/iln-preprocess-create-dataset-public/{dat_ver}_df_wifi_all.pkl', 'rb')as f:
data = pickle.load(f )<feature_engineering>
learner.fit_one_cycle(4,slice(5e-6,lr/50))
min_seq_len = 10
max_seq_len = 200
tmp = []
for gid, grp in tqdm(data.groupby('path'), ncols=60):
if grp.shape[0] > min_seq_len:
thresh = np.sort(grp['strength_0'].values)[-min_seq_len]
thresh = min([-100, thresh])
 grp = grp[grp['strength_0']>=thresh].copy()
grp['seq_label'] = grp['path'].str.cat(( np.arange(grp.shape[0])
 //max_seq_len ).astype(str),
sep='_')
 tmp.append(grp)
 data = pd.concat(tmp)
 data.reset_index(drop=True, inplace=True)
 del tmp; gc.collect()
print(f'total num of path data: {data.shape[0]}' )<feature_engineering>
learner.save('Resnetfinal' )
tslabel ='sys_ts'
data['site_path_timestamp'] =(data['site'] 
.str.cat(data['path'], sep='_'))
mask = data['train/test']=='train'
data.loc[ mask,'timestamp'] = data.loc[mask,tslabel]
data.loc[~mask,'timestamp'] =(data.loc[~mask,tslabel]
-df_TestTimeLag.loc[data.loc[~mask,'path']] 
.values.reshape(-1))
data['site_path_timestamp'] =(data['site_path_timestamp'] 
.str.cat(data['timestamp'] 
.astype('int64' ).astype(str)
.str.zfill(13), sep='_'))<drop_column>
train=pd.read_csv('/kaggle/input/train.csv')
 data=TabularDataBunch.from_df(path,train,dep_var='label',valid_idx=range(4000,6000))
OTHER_FEATS = [itm for itm in data.columns
if(itm[:3]!='id_')&(itm[:9]!='strength_')& 
(itm[-5:]!='_mean')&(itm[-4:]!='_std')& 
(itm[-4:]!='_max')&(itm[-4:]!='_min')& 
(itm[-5:]!='_skew')]
data = data[OTHER_FEATS+ID_FEATS+STRG_FEATS+IMU_FEATS]<categorify>
tablearner=tabular_learner(data,layers=[200,100],ps=[0.001,0.01],emb_drop=0.004,metrics=accuracy )
le_id = LabelEncoder()
le_id.fit(ID_vocab)
 le_site = LabelEncoder()
le_site.fit(data['site'].unique())
 for i in ID_FEATS:
data.loc[:,i] = le_id.transform(data.loc[:,i])+1
data.loc[:, 'site'] = le_site.transform(data.loc[:, 'site'])
 data[ID_FEATS] *=(~(data[STRG_FEATS].isna() ).values)
 data[STRG_FEATS] = data[STRG_FEATS].fillna(-100)+100<feature_engineering>
tablearner.lr_find()
tablearner.recorder.plot(suggestion=True )
data.sort_index(inplace=True)
 tmp = []
for _, gdf in tqdm(data.groupby('path',sort=False),
ncols=60):
arr = gdf[STRG_FEATS].values.flatten()
gdf[STRG_FEATS] =(gdf[STRG_FEATS]-arr.min())/(arr.max() -arr.min())
 tmp.append(gdf[STRG_FEATS])
 tmp = pd.concat(tmp)
 tmp.sort_index(inplace=True)
 data[STRG_FEATS] = tmp.values
del arr,tmp; gc.collect()<define_variables>
lr=2.2e-2
def replace_outlier(series, thresh=0.25, bias=1.5):
q1 = series.quantile(thresh)
 q3 = series.quantile(1-thresh)
 iqr = q3 - q1
outlier_min = q1 -(iqr)* bias
outlier_max = q3 +(iqr)* bias
series = series.clip(outlier_min, outlier_max)
 return series<normalization>
tablearner.fit_one_cycle(10,slice(lr),wd=0.1 )
data[['acce_z_mean','acce_z_max','acce_z_min']] = \
data[['acce_z_mean','acce_z_max','acce_z_min']]-9.80665
for sens in['gyro','acce','ahrs','magn']:
ss = StandardScaler()
if sens=='magn':
tmpFEATS = [sens+'_z_mean']
ss.fit(data[tmpFEATS].values.reshape(-1,1))
else:
tmpFEATS = [sens+'_x_mean', sens+'_y_mean', sens+'_z_mean']
ss.fit(np.tile(data[tmpFEATS].values.reshape(-1,1),(1,3)))
 data[tmpFEATS] = ss.transform(data[tmpFEATS])
 for itm in tmpFEATS:
data[itm]=replace_outlier(data[itm])
 ss = StandardScaler()
if sens=='magn':
tmpFEATS = [sens+'_z_std']
ss.fit(data[tmpFEATS].values.reshape(-1,1))
else:
tmpFEATS = [sens+'_x_std', sens+'_y_std', sens+'_z_std']
ss.fit(np.tile(data[tmpFEATS].values.reshape(-1,1),(1,3)))
 data[tmpFEATS] = ss.transform(data[tmpFEATS])
 for itm in tmpFEATS:
data[itm]=replace_outlier(data[itm])
 ss = StandardScaler()
if sens=='magn':
tmpFEATS = [sens+'_z_skew']
ss.fit(data[tmpFEATS].values.reshape(-1,1))
else:
tmpFEATS = [sens+'_x_skew', sens+'_y_skew', sens+'_z_skew']
ss.fit(np.tile(data[tmpFEATS].values.reshape(-1,1),(1,3)))
 data[tmpFEATS] = ss.transform(data[tmpFEATS])
 for itm in tmpFEATS:
data[itm]=replace_outlier(data[itm])
 ss = StandardScaler()
if sens=='magn':
tmpFEATS = [sens+'_z_max', sens+'_z_min']
ss.fit(np.tile(data[tmpFEATS].values.reshape(-1,1),(1,2)))
 else:
tmpFEATS = [sens+'_x_max', sens+'_x_min',
sens+'_y_max', sens+'_y_min',
sens+'_z_max', sens+'_z_min']
ss.fit(np.tile(data[tmpFEATS].values.reshape(-1,1),(1,6)))
 data[tmpFEATS] = ss.transform(data[tmpFEATS])
 for itm in tmpFEATS:
data[itm]=replace_outlier(data[itm] )<create_dataframe>
tablearner.lr_find()
tablearner.recorder.plot(suggestion=True )
trvl_data = data[data['train/test']=='train'].copy()
test_data = data[data['train/test']=='test'].copy()
del data; gc.collect()<feature_engineering>
tablearner.fit_one_cycle(4,4e-6,wd=0.1 )
grb = trvl_data[['seq_label','site','floor']].groupby('seq_label')
 df_stra = grb.first()
df_stra['seq_len'] = grb['seq_label'].count().values
df_stra['seq_len_cut'] = pd.qcut(df_stra['seq_len'],4)
 df_stra['stratify'] = df_stra.apply(lambda x:
str(x['seq_len_cut'])+'_'+
str(x['site'])+'_'+
str(x['floor']),
axis=1)
 skf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True,
random_state=SEED)
 for i,(_, val_idx)in enumerate(skf.split(df_stra['stratify'],
df_stra['stratify'])) :
tmp = np.zeros(df_stra.shape[0] ).astype(bool)
 tmp[val_idx] = True
df_stra[f'fold_{i}'] = tmp
df_stra.shape<set_options>
df_test = pd.read_csv('.. /input/test.csv')
 df_test['label'] = 0
df_test['index'] = df_test.index
df_test.head()
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
 print('using device:', device )<categorify>
learner.load('Resnetfinal' )
tr_id, tr_strg, tr_dlt, tr_imu, tr_site, tr_pos, tr_xy, tr_mask = \
[],[],[],[],[],[],[],[]
for gid, grp in tqdm(trvl_data.groupby('seq_label'), ncols=60):
tr_id.append(torch.from_numpy(grp[ID_FEATS].values ).long())
 tr_strg.append(torch.from_numpy(grp[STRG_FEATS].values ).float())
 tr_dlt.append(torch.from_numpy(grp[DELTA_FEATS].values ).float())
 tr_imu.append(torch.from_numpy(grp[IMU_FEATS].values ).float())
 tr_site.append(torch.from_numpy(grp['site'].values ).long())
 tr_pos.append(torch.from_numpy(grp['len_pos'].values ).float())
 tr_xy.append(torch.from_numpy(grp[['x','y']].values ).float())

tr_mask.append(tr_id[-1].sum(dim=-1)==0)
 tr_id = rnn.pad_sequence(tr_id, batch_first=True ).to(device)
 tr_strg = rnn.pad_sequence(tr_strg, batch_first=True ).to(device)
 tr_dlt = rnn.pad_sequence(tr_dlt, batch_first=True ).to(device)
 tr_imu = rnn.pad_sequence(tr_imu, batch_first=True ).to(device)
 tr_site = rnn.pad_sequence(tr_site, batch_first=True ).to(device)
 tr_pos = rnn.pad_sequence(tr_pos, batch_first=True ).to(device)
 tr_xy = rnn.pad_sequence(tr_xy, batch_first=True ).to(device)
 tr_mask = rnn.pad_sequence(tr_mask, batch_first=True,
padding_value=True ).to(device )<train_model>
learner.data.add_test(PixelList.from_df(df_test,path='.',cols='index'))
class ILNdatasets(torch.utils.data.Dataset):
def __init__(self, x_id, x_strg, x_dlt, x_imu, x_site, x_pos, x_mask,
y, train=True):
self.train = train
self.x_id = x_id
self.x_strg = x_strg
self.x_dlt = x_dlt
self.x_imu = x_imu
self.x_site = x_site
self.x_pos = x_pos
self.x_mask = x_mask
self.y = y
self.datanum = len(y)
 def __len__(self):
return self.datanum
def __getitem__(self, idx):
return(self.x_id[idx], self.x_strg[idx], self.x_dlt[idx], self.x_imu[idx],
self.x_site[idx], self.x_pos[idx], self.x_mask[idx],
self.y[idx] )<categorify>
pred_test = learner.get_preds(ds_type=DatasetType.Test)
 test_result = torch.argmax(pred_test[0],dim=1)
 result = test_result.numpy()
<categorify><EOS> 
final = pd.Series(result,name='Label')
 submission = pd.concat([pd.Series(range(1,28001),name='ImageId'),final],axis=1)
 submission.to_csv('submit.csv',index=False )