Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<create_dataframe>
import numpy as np
import pandas as pd
import scipy
import sklearn
def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
<compute_test_metric>
from sklearn.preprocessing import OrdinalEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import OneHotEncoder
def map_at3(y_true, y_pred):
map3 = tf.keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=1)*0.5
map3 += tf.keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=2)*0.17
map3 += tf.keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=3)*0.33
return map3<define_variables>
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_validate
ext_data_path = '/kaggle/input/doodle-model/'<load_from_csv>
df_train = pd.read_csv('/kaggle/input/cat-in-the-dat/train.csv', index_col='id')
 df_test = pd.read_csv('/kaggle/input/cat-in-the-dat/test.csv', index_col='id' )
outputs = os.listdir(ext_data_path)
 hold_out_probs = [ext_data_path+ f for f in outputs if f.find('ho_prob')>= 0 ]
test_out_probs = [ext_data_path+ f for f in outputs if f.find('test_prob')>= 0 ]
hold_out_probs = sorted(hold_out_probs)
 test_out_probs = sorted(test_out_probs)
 ho_df = pd.read_csv(ext_data_path+hold_out_set)
 ho_s = []
for prob_path in hold_out_probs:
ho = np.load(prob_path)
 ho = ho[:len(ho_df)]
ho_s.append(ho)
 targets = ho_df.y.to_numpy()<set_options>
sample_submission = pd.read_csv('/kaggle/input/cat-in-the-dat/sample_submission.csv', index_col='id' )
del(ho_s)
 gc.collect()<choose_model_class>
binvar = ['bin_' + str(i)for i in range(1,5)]
ordvar = ['ord_' + str(i)for i in range(6)]
nomvar = ['nom_' + str(i)for i in range(10)]
dmvar = ['day', 'month']
xin = tf.keras.layers.Input(( len(label_names),ho_arr.shape[2]))
x = tf.keras.layers.Convolution1D(1,kernel_size=1,activation='linear',use_bias=False )(xin)
 x = tf.keras.layers.Reshape(target_shape=(len(label_names),))(x)
 wensemble_model = tf.keras.Model(inputs=xin, outputs=x)
 wensemble_model.summary()<train_on_grid>
df_work = df_train.copy()
y_train = df_work['target'].copy()
df_work = pd.concat([df_work.drop('target', axis = 1), df_test] )
check_point=tf.keras.callbacks.ModelCheckpoint(monitor='map_at3',verbose=1
,filepath='ensemble_w.h5',save_best_only=True,mode='max')
 wensemble_model.compile(optimizer=tf.keras.optimizers.Adam(1),loss='mse', metrics=[ map_at3])
 wensemble_model.fit(x=ho_arr,y=tf.keras.utils.to_categorical(targets)
 , epochs=20, batch_size=10000,verbose=1, callbacks=[check_point] )<load_pretrained>
df_work.drop('bin_0', inplace=True, axis=1 )
wensemble_model.load_weights('ensemble_w.h5')
 wensemble_model.get_weights()<set_options>
df_work['bin_3'] = df_work['bin_3'].map({'F':0, 'T':1})
 df_work['bin_4'] = df_work['bin_4'].map({'N':0, 'Y':1} )
del(ho_arr)
 gc.collect()<find_best_params>
df_work['ord_0'] = df_work['ord_0'] - 1
res = np.array(wensemble_model.get_weights() ).squeeze()<define_variables>
ord1dict = {'Novice':0, 'Contributor':1, 'Expert':2, 'Master':3, 'Grandmaster':4}
df_work['ord_1'] = df_work['ord_1'].map(ord1dict )
ens_prob = np.zeros(( len(sub_df),len(label_names)))
 for i, prob_path in enumerate(test_out_probs):
ens_prob +=(np.load(prob_path)* res[i] )<statistical_test>
ord2dict = {'Freezing':0, 'Cold':1, 'Warm':2, 'Hot':3, 'Boiling Hot':4, 'Lava Hot':5}
df_work['ord_2'] = df_work['ord_2'].map(ord2dict )
def balancing_predictions(test_prob, factor = 0.1, minfactor = 0.001, patient = 5, permit_cnt=332, max_search=10000, label_num=340):
maxk = float('inf')
 s_cnt = np.zeros(label_num)
 for i in range(max_search):
ctop1 = Counter(np.argmax(test_prob,axis=1))
ctop1 = sorted(ctop1.items() , key=itemgetter(1), reverse=True)
 if maxk > ctop1[0][1]:
maxk = ctop1[0][1]
else:
s_cnt[ctop1[0][0]]+=1
if np.max(s_cnt)>patient:
if factor< minfactor:
print('stop min factor')
 break
s_cnt=np.zeros(label_num)
 factor*=0.99
print('reduce factor: ', factor, ', current max category num: ', ctop1[0][1])
 if ctop1[0][1] <= permit_cnt:
print('idx: ',ctop1[0][0] ,', num: ', ctop1[0][1])
 break
test_prob[:,ctop1[0][0]] *=(1.0-factor)
 return test_prob<save_to_csv>
oe = OrdinalEncoder(categories='auto')
 df_work[ordvar[3:]] = oe.fit_transform(df_work[ordvar[3:]])
 for var, cl in zip(ordvar[3:], oe.categories_):
print(var)
 print(cl )
bal_test_prob = balancing_predictions(ens_prob)
 bal_top3 = preds2catids(bal_test_prob)
 id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(label_names)}
bal_top3cats = bal_top3.replace(id2cat)
 sub_df['word'] = bal_top3cats['a'] + ' ' + bal_top3cats['b'] + ' ' + bal_top3cats['c']
bal_submission = sub_df[['key_id', 'word']]
bal_submission.to_csv('submission_bal_ens.csv', index=False )<set_options>
df_work[ordvar] = StandardScaler().fit_transform(df_work[ordvar] )
%matplotlib inline<define_variables>
df_work[nomvar[5:]].nunique()
train_dir = '.. /input/train_simplified'
test_dir = '.. /input'
test_file = 'test_simplified.csv'
train_nrows = 2000
val_nrows = 100
train_files = os.listdir(train_dir)
 train_files = sorted(train_files)
 w2i_dict = {fn[:-4].replace(' ','_'): i for i, fn in enumerate(train_files)}
i2w_dict = {v: k for k, v in w2i_dict.items() }<load_from_csv>
df_work['nom_5'] = df_work['nom_5'].str[4:]
df_work['nom_6'] = df_work['nom_6'].str[3:]
df_work['nom_7'] = df_work['nom_7'].str[3:]
df_work['nom_8'] = df_work['nom_8'].str[3:]
df_work['nom_9'] = df_work['nom_9'].str[3:]
class DoodleDataset(Dataset):
def __init__(self, file, dir, mode='train', nrows = None, skiprows=None, transform = None):
self.file = file
self.dir = dir
self.mode = mode
self.nrows = nrows
self.transform = transform
self.pd_frame = pd.read_csv(os.path.join(self.dir, self.file), usecols=['drawing'], nrows=nrows, skiprows=skiprows)
 if self.mode == 'train':
self.label = w2i_dict[file[:-4].replace(' ','_')]
@staticmethod
def _point2line(strokes):
img = np.zeros(( 256,256), dtype = np.uint8)
 for stroke in strokes:
x_coords = stroke[0]
y_coords = stroke[1]
for i in range(len(stroke[0])-1):
cv2.line(img,(x_coords[i], y_coords[i]),(x_coords[i+1], y_coords[i+1]), color = 255, thickness = 2)
 return img
def __len__(self):
return len(self.pd_frame)
 def __getitem__(self, idx):
strokes = eval(self.pd_frame.drawing[idx])
 sample = self._point2line(strokes)
 if self.transform:
sample = self.transform(sample)
 if self.mode=='train':
sample = sample.numpy()
return sample.astype('float32'), self.label
else:
sample = sample.numpy()
return sample.astype('float32')
 data_transform = transforms.Compose([
transforms.ToPILImage() ,
transforms.RandomHorizontalFlip() ,
transforms.Resize(( 128,128)) ,
transforms.ToTensor()
])
 val_data_transform = transforms.Compose([
transforms.ToPILImage() ,
transforms.Resize(( 128,128)) ,
transforms.ToTensor()
])
 doodles = ConcatDataset([DoodleDataset(fn, train_dir, nrows=train_nrows, transform = data_transform)for fn in train_files])
 doodles_val = ConcatDataset([DoodleDataset(fn, train_dir, nrows=val_nrows, transform = val_data_transform, skiprows=range(1, train_nrows+1)) for fn in train_files] )<load_pretrained>
df_work[nomvar[5:]].nunique()
dataloader = DataLoader(doodles, batch_size = 16, shuffle = True, num_workers = 4)
 valloader = DataLoader(doodles_val, batch_size = 16, shuffle = True, num_workers = 4 )<compute_test_metric>
enc = OneHotEncoder(categories = 'auto', dtype = 'float64', drop = 'first')
 nom_matrix = enc.fit_transform(df_work[nomvar])
 df_work.drop(nomvar, inplace=True, axis=1 )
def mapk(output, target, k=3):
with torch.no_grad() :
batch_size = target.size(0)
 _, pred = output.topk(k, 1, True, True)
 pred = pred.t()
correct = pred.eq(target.view(1, -1 ).expand_as(pred))
for i in range(k):
correct[i] = correct[i]*(k-i)
 score = correct[:k].view(-1 ).float().sum(0, keepdim=True)
 score.mul_(1.0 /(k * batch_size))
return score<set_options>
enc = OneHotEncoder(categories='auto', dtype = 'float64', drop = 'first')
 dm_matrix = enc.fit_transform(df_work[dmvar])
 df_work.drop(dmvar, inplace=True, axis=1 )
class RAdam(Optimizer):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):
defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 super(RAdam, self ).__init__(params, defaults)
 self.step_buffer = [None, None, None]
self.N_sma_max = 2 /(1-betas[1])-1
def __setstate__(self, state):
super(RAdam, self ).__setstate__(state)
 def step(self, closure=None):
loss = None if closure is None else closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 1
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['step'] += 1
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
beta1_t, beta2_t = beta1 ** state['step'], beta2 ** state['step']
bias_correction1, bias_correction2 = 1 - beta1_t, 1 - beta2_t
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 if state['step'] == self.step_buffer[0]:
N_sma, step_size = self.step_buffer[1], self.step_buffer[2]
else:
N_sma = self.N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 if N_sma >= 5:
rectification = math.sqrt(((N_sma-4)*(N_sma-2)*self.N_sma_max)/(( self.N_sma_max-4)*(self.N_sma_max-2)*N_sma))
step_size = math.sqrt(bias_correction2)* rectification / bias_correction1
else:
step_size = 1.0 / bias_correction1
self.step_buffer[0] = state['step']
self.step_buffer[1] = N_sma
self.step_buffer[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma >= 5:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size * group['lr'], exp_avg, denom)
 else:
p_data_fp32.add_(-step_size * group['lr'], exp_avg)
 p.data.copy_(p_data_fp32)
 return loss 
<choose_model_class>
df_work_sprs =scipy.sparse.hstack([nom_matrix,
scipy.sparse.coo_matrix(df_work ).astype('float64'),
dm_matrix] ).tocsr()
display(df_work_sprs )
device = 'cuda'
def squeeze_weights(m):
m.weight.data = m.weight.data.sum(dim=1)[:,None]
m.in_channels = 1
model = torchvision.models.densenet121(pretrained=True)

num_ftrs = model.classifier.in_features
model.classifier = nn.Linear(num_ftrs, 340)
 model.features.conv0.apply(squeeze_weights)
 model = model.to(device)
 criterion = nn.CrossEntropyLoss()
optimizer = RAdam(model.parameters() , lr=0.005)
 scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[30000,60000,90000], gamma=0.3 )<train_on_grid>
X_train = df_work_sprs[:y_train.shape[0]]
X_test = df_work_sprs[y_train.shape[0]:]
epochs = 3
itr = 1
p_itr = 5000
total_loss, score = 0,0
val_score = 0.0
best_model_wts = copy.deepcopy(model.state_dict())
 best_score = 0.0
loss_list = []
score_list = []
for epoch in range(epochs):
model.train()
for samples, labels in dataloader:
samples, labels = samples.to(device), labels.to(device)
 optimizer.zero_grad()
output = model(samples)
 loss = criterion(output, labels)
 loss.backward()
optimizer.step()
total_loss += loss.item()
score += mapk(output, labels)[0].item()
scheduler.step()
if itr%p_itr==0:
print('[Epoch {}/{}] Iteration {} -> Train Loss: {:.4f}, MAP@3: {:.3f}'.format(epoch+1, epochs, itr, total_loss/p_itr, score/p_itr))
loss_list.append(total_loss/p_itr)
 score_list.append(score/p_itr)
 total_loss, score = 0,0
itr += 1
model.eval()
for samples, labels in valloader:
with torch.no_grad() :
samples = samples.to(device)
 labels = labels.to(device)
 output = model(samples)
 val_score += mapk(output, labels)[0].item()
print('[Epoch {}/{}] Validation Score: {:.4f}'.format(epoch+1, epochs, val_score/len(valloader)))
 if val_score > best_score:
best_score = val_score
best_model_wts = copy.deepcopy(model.state_dict())
 val_score = 0
model.load_state_dict(best_model_wts )<create_dataframe>
%%time
C = 0.12
clf = LogisticRegression(C=C, solver='lbfgs', max_iter=1000, verbose=0, n_jobs=-1)
 score = cross_validate(clf, X_train, y_train, cv=3, scoring="roc_auc")
 mean = score['test_score'].mean()
print(score['test_score'])
 print('C =', C, f'{mean:.8f}' )
filename_pth = 'ckpt_densenet121.pth'
torch.save(model.state_dict() , filename_pth)
 test_transform = transforms.Compose([
transforms.ToPILImage() ,
transforms.Resize(( 128,128)) ,
transforms.ToTensor()
])
 testset = DoodleDataset(test_file, test_dir, mode='test', transform = test_transform)
 testloader = DataLoader(testset, batch_size=16, shuffle=False, num_workers=4 )<save_to_csv>
%%time
clf = LogisticRegression(C=C, solver='lbfgs', max_iter=1000, verbose=0, n_jobs=-1)
 clf.fit(X_train, y_train )
model.eval()
labels = np.empty(( 0,3))
for x in tqdm.tqdm_notebook(testloader):
with torch.no_grad() :
x = x.to(device)
 output = model(x)
 _, pred = output.topk(3,1,True,True)
 labels = np.concatenate([labels, pred.cpu() ], axis=0)
 submission = pd.read_csv('.. /input/test_simplified.csv', index_col='key_id')
 submission.drop(['countrycode','drawing'], axis=1, inplace=True)
 submission['word']=''
for i, label in enumerate(labels):
submission.word.iloc[i] = " ".join([i2w_dict[l] for l in label])
 submission.to_csv('preds_densenet121.csv' )<import_modules>
y_preds = clf.predict_proba(X_test)[:,1]
<set_options><EOS> 
sample_submission['target'] = y_preds
sample_submission.to_csv('submission.csv' )