Kaggle competition: Santander Value Prediction Challenge
<SOS> metric: RMSLE Kaggle data source: santander-value-prediction-challenge<load_pretrained>
%matplotlib inline
def load_model(name, num_tasks):
base = torch.jit.load(name + '-base.trace')
 task_predictors = [torch.jit.load(name + '-task-predictor-{}.trace'.format(i)) for i in range(num_tasks)]
return MultiTaskNN(base, task_predictors )<load_pretrained>
train=pd.read_csv('.. /input/train.csv')
 test=pd.read_csv('.. /input/test.csv' )
model = load_model('/kaggle/input/bhgd-post-exp/r18-post-exp-1', 3 )<find_best_params>
constant = train.nunique().reset_index()
constant.columns = ["col", "count"]
constant = constant.loc[constant["count"]==1]
train = train.drop(columns=constant.col,axis = 1)
 test = test.drop(columns=constant.col,axis = 1 )
model = model.to(device)
 model.eval() ;<define_variables>
y = train["target"]
train = train.drop(["ID","target"],axis=1)
 test = test.drop("ID",axis=1)
 train["ID"] = train.index
test["ID"] = test.index
results = []<create_dataframe>
def maxabs(train,test):
scaler = MaxAbsScaler()
scaler.fit(train)
 train = scaler.transform(train)
 test = scaler.transform(test)
 return train,test
with torch.no_grad() :
for i in range(4):
te_ds = make_tensordataset_from_dfs(
 ['/kaggle/input/bengaliai-cv19' + '/test_image_data_{}.parquet'.format(i)] 
)
 te_ds = DatasetWithImageTransforms(te_ds, tfms)
 num_tta = 1
te_dl = DataLoader(te_ds, batch_size=256, num_workers=2, pin_memory=True)
 tta_results = [[] for i in range(num_tta)]
for i in range(num_tta):
for imgs, in te_dl:
imgs = imgs.to(device)
 g_pred, v_pred, c_pred, *rest = model(imgs)
 tta_results[i].append([g_pred, v_pred, c_pred])
 for i in range(len(te_dl)) :
g_preds = [results[i][0] for results in tta_results]
v_preds = [results[i][1] for results in tta_results]
c_preds = [results[i][2] for results in tta_results]
g_pred = torch.stack(g_preds ).mean(dim=0)
 v_pred = torch.stack(v_preds ).mean(dim=0)
 c_pred = torch.stack(c_preds ).mean(dim=0)
 g_pred = g_pred.argmax(1 ).tolist()
v_pred = v_pred.argmax(1 ).tolist()
c_pred = c_pred.argmax(1 ).tolist()
for g, v, c in zip(g_pred, v_pred, c_pred):
results.extend([c, 0, v])
 del imgs
del te_ds
del te_dl
del tta_results<drop_column>
y_train=np.log1p(y)
del model<load_from_csv>
trainSVD = train.copy()
testSVD = test.copy()
submission_df = pd.read_csv('/kaggle/input/bengaliai-cv19' + '/sample_submission.csv' )<feature_engineering>
def rowagg(train,test):
train["sum"] = train.sum(axis=1)
 test["sum"] = test.sum(axis=1)
 train["var"] = train.var(axis=1)
 test["var"] = test.var(axis=1)
 train["median"] = train.median(axis=1)
 test["median"] = test.median(axis=1)
 train["mean"] = train.mean(axis=1)
 test["mean"] = test.mean(axis=1)
 train["std"] = train.std(axis=1)
 test["std"] = test.std(axis=1)
 train["max"] = train.max(axis=1)
 test["max"] = test.max(axis=1)
 train["min"] =train.min(axis=1)
 test["min"] = test.min(axis=1)
 train["skew"] = train.skew(axis=1)
 test["skew"] = test.skew(axis=1)
 print("Null values in train: "+ str(np.sum(np.sum(pd.isnull(train)))))
 print("NAN values in train: "+ str(np.sum(np.isnan(train.values))))
print("Null values in test: "+ str(np.sum(np.sum(pd.isnull(test)))))
 print("NAN values in test: "+ str(np.sum(np.isnan(test.values))))
return train,test
submission_df['target'] = results<save_to_csv>
trainSVD,testSVD = rowagg(trainSVD,testSVD)
 svd = TruncatedSVD(n_components=2000)
 res = svd.fit(trainSVD)
 print(np.sum(res.explained_variance_ratio_))
submission_df.to_csv('./submission.csv', index=False )<install_modules>
trainSVD = res.transform(trainSVD)
 testSVD = res.transform(testSVD )
!pip install /kaggle/input/timm-model/timm-0.3.2-py3-none-any.whl<import_modules>
def run_lgb(X_train, Y_train, X_valid, Y_valid, test):
seed = 42
params = {
"objective" : "regression",
"metric" : "rmse",
"task": "train",
"boosting type":'dart',
"num_leaves" :500,
"learning_rate" : 0.005,
"bagging_fraction" : 0.8,
"feature_fraction" : 0.8,
"bagging_frequency" : 5,
"bagging_seed" : seed,
"verbosity" : -1,
"seed": seed
}
lgtrain = lgb.Dataset(X_train,label= Y_train)
 lgval = lgb.Dataset(X_valid,label =Y_valid)
 evals_result = {}
model = lgb.train(params, lgtrain, 5000,
valid_sets=[lgtrain, lgval],
early_stopping_rounds=300,
verbose_eval=100,
evals_result=evals_result)
 lgb_prediction = np.expm1(model.predict(test, num_iteration=model.best_iteration))
return lgb_prediction, model, evals_result
import pandas as pd
import cv2
import numpy as np
from tqdm.auto import tqdm
from matplotlib import pyplot as plt
import time
import pyarrow.parquet as pq<set_options>
trainSVD,testSVD = maxabs(trainSVD,testSVD)
 X_train, X_test, Y_train, Y_test = train_test_split(trainSVD, y_train, test_size=0.1, random_state=0)
 lgb_predSVD, model, evals_resultRF = run_lgb(X_train, Y_train, X_test, Y_test, testSVD )
class MultiHeadSimpleModel(nn.Module):
def __init__(self, input_size, backbone, pretrained=True, dropout=0.2):
super(MultiHeadSimpleModel, self ).__init__()
self.input_size = input_size
self.backbone = backbone
self.pretrained = pretrained
self.dropout=dropout
self.backbone = create_model(self.backbone, self.pretrained, num_classes=0)
 feature_size = self.backbone.state_dict() ['bn2.weight'].shape[0]
self.root_head = nn.Linear(feature_size, 168, bias=True)
 self.consonant_head = nn.Linear(feature_size, 168, bias=True)
 self.vowel_head = nn.Linear(feature_size, 168, bias=True)
 intermid = []
intermid.append(nn.BatchNorm1d(feature_size))
intermid.append(nn.Dropout(self.dropout))
intermid.append(nn.Linear(feature_size, 512))
intermid.append(nn.BatchNorm1d(512))
self.intermid_unique = nn.ModuleList(intermid)
 torch.nn.init.kaiming_normal_(self.intermid_unique[2].weight)
 self.arc_face_unique = Arcface(512, 1295)
 def multi_head(self, input, unique):
input = self.backbone(input)
 root = self.root_head(input)
 consonant = self.consonant_head(input)
 vowel = self.vowel_head(input)
 x = self.intermid_unique[0](input)
 for inter in self.intermid_unique[1:]:
x = inter(x)
 x = F.normalize(x, p=2, dim=1)
 unique_p = self.arc_face_unique(x, unique)
 return root, consonant, vowel, unique_p
def forward(self, input, unique):
multi_head_outputs = self.multi_head(input, unique)
 return multi_head_outputs
def l2_norm(input,axis=1):
norm = torch.norm(input,2,axis,True)
 output = torch.div(input, norm)
 return output
class Arcface(nn.Module):
def __init__(self, embedding_size=512, classnum=51332, s=30., m=0.5):
super(Arcface, self ).__init__()
self.classnum = classnum
self.weight = Parameter(torch.Tensor(embedding_size,classnum))
self.weight.data.uniform_(-1, 1 ).renorm_(2,1,1e-5 ).mul_(1e5)
 self.m = m
self.s = s
self.cos_m = math.cos(m)
 self.sin_m = math.sin(m)
 self.mm = self.sin_m * m
self.threshold = math.cos(math.pi - m)
 def forward(self, embbedings, label):
nB = len(embbedings)
 kernel_norm = l2_norm(self.weight,axis=0)
 cos_theta = torch.mm(embbedings,kernel_norm)
 cos_theta = cos_theta.clamp(-1,1)
 cos_theta_2 = torch.pow(cos_theta, 2)
 sin_theta_2 = 1 - cos_theta_2
sin_theta = torch.sqrt(sin_theta_2)
 cos_theta_m =(cos_theta * self.cos_m - sin_theta * self.sin_m)
 cond_v = cos_theta - self.threshold
cond_mask = cond_v <= 0
keep_val =(cos_theta - self.mm)
 cos_theta_m[cond_mask] = keep_val[cond_mask]
output = cos_theta * 1.0
idx_ = torch.arange(0, nB, dtype=torch.long)
 output[idx_, label] = cos_theta_m[idx_, label]
output *= self.s
return output
class BengaliDataset(Dataset):
def __init__(self, label_csv, unique_csv, train_folder, parquet_path, transforms, cache=True, test=True):
self.label_csv = label_csv
self.unique_csv = unique_csv
self.train_folder = train_folder
self.parquet_path = parquet_path
self.label = pd.read_csv(self.label_csv)
 self.label = self.label[self.label['component']=='grapheme_root']
self.label = self.label.reset_index(drop=True)
 unique_df = pd.read_csv(self.unique_csv)
 self.names = self.label['image_id'].values
self.uniques = unique_df.grapheme.unique()
self.transforms = transforms
self.img = [None] * self.label.shape[0]
self.test = test
if cache:
self.cache_images()
def cache_images(self):
count = 0
root = './test_graphemes/'
self.p = p = pd.read_parquet(self.parquet_path ).drop(['image_id'], axis=1)
 def load_image(self, idx):
img = resize(self.p.iloc[[idx]] ).values.reshape(-1 ).reshape(IMG_SIZE, IMG_SIZE ).astype(np.uint8)
 return self.transforms(img)
 img = self.img[idx]
if img is None:
img = resize(p.iloc[[i]] ).values.reshape(-1 ).reshape(IMG_SIZE, IMG_SIZE ).astype(np.uint8)
 return self.transforms(img)
 else:
return self.transforms(img)
 def __getitem__(self, idx):
if not self.test:
img = self.load_image(idx)
 root = self.label.loc[idx]['grapheme_root']
consonant = self.label.loc[idx]['consonant_diacritic']
vowel = self.label.loc[idx]['vowel_diacritic']
unique = np.where(self.uniques == self.label.grapheme[idx])[0][0]
return transforms.ToTensor()(img), root, consonant, vowel, unique
else:
img = self.load_image(idx)
 root = 0
consonant = 0
vowel = 0
unique = 0
return transforms.ToTensor()(img), root, consonant, vowel, unique
def __len__(self):
return self.p.shape[0]
def postprocess(preds, num_classes, EXP = -1.2):
p0 = np.argmax(preds,axis=1)
 s = pd.Series(p0)
 vc = s.value_counts().sort_index()
df = pd.DataFrame({'a':np.arange(num_classes),'b':np.ones(num_classes)})
 df.b = df.a.map(vc)
 df.fillna(df.b.min() ,inplace=True)
 mat1 = np.diag(df.b.astype('float32')**EXP)
 p0 = np.argmax(preds.dot(mat1), axis=1)
 return p0
class Tester:
def __init__(self,
dataset_path='./drive/MyDrive/datasets/car classification/train_data',
batch_size=1,
model_name='tf_efficientnet_b3_ns',
test_csv='./train_labels.csv',
unique_csv='./train_labels.csv',
output_dir='.. /drive/MyDrive/ckpt/grapheme/submission.csv',
ckpt='.. /drive/MyDrive/ckpt/grapheme/20.pth',
multihead_ckpt='.. /'):
self.dataset_path = dataset_path
self.batch_size = batch_size
self.model_name = model_name
self.test_csv = test_csv
self.unique_csv = unique_csv
self.output_dir = output_dir
self.ckpt = ckpt
self.multihead_ckpt = multihead_ckpt
if model_name == 'tf_efficientnet_b0_ns':
self.input_size =(224, 224)
 elif model_name == 'tf_efficientnet_b2_ns':
self.input_size =(260, 260)
 elif model_name == 'tf_efficientnet_b3_ns':
self.input_size =(300, 300)
 elif model_name == 'tf_efficientnet_b4_ns':
self.input_size =(380, 380)
 elif model_name == 'tf_efficientnet_b6_ns':
self.input_size =(528, 528)
 else:
raise Exception('non-valid model name')
 transform = []
transform += [transforms.ToPILImage() ]
transform += [transforms.Resize(self.input_size)]
self.transform = transforms.Compose(transform)
 self.device = torch.device('cuda')
 self.model_root = create_model(self.model_name, pretrained=False, num_classes=168 ).to(self.device)
 self.model_consonant = create_model(self.model_name, pretrained=False, num_classes=8 ).to(self.device)
 self.model_vowel = create_model(self.model_name, pretrained=False, num_classes=11 ).to(self.device)
 self.model_multihead = MultiHeadSimpleModel(self.input_size, self.model_name, pretrained=False, dropout=0 ).to(self.device)
 ckpt = torch.load(self.ckpt)
 multihead_ckpt = torch.load(self.multihead_ckpt)
 self.model_root.load_state_dict(ckpt['model_root_state_dict'])
 self.model_consonant.load_state_dict(ckpt['model_consonant_state_dict'])
 self.model_vowel.load_state_dict(ckpt['model_vowel_state_dict'])
 self.model_multihead.load_state_dict(multihead_ckpt['model_multihead_state_dict'])
 def test(self):
output_roots = []
output_consonants = []
output_vowels = []
count = 0
for ii in range(4):
self.test_dataset = BengaliDataset(self.test_csv, self.unique_csv, self.dataset_path, f'/kaggle/input/bengaliai-cv19/test_image_data_{ii}.parquet', self.transform, cache=True)
 self.names = self.test_dataset.names
self.test_dataloader = DataLoader(self.test_dataset, batch_size=self.batch_size, num_workers=0, shuffle=False)
 pbar = tqdm.tqdm(self.test_dataloader)
 pbar.set_description('testing process')
 self.model_root.eval()
self.model_consonant.eval()
self.model_vowel.eval()
self.model_multihead.eval()
with torch.no_grad() :
for it, data in enumerate(pbar):
inputs = data[0].to(self.device)
 inputs = inputs.repeat(1, 3, 1, 1)
 roots = data[1].to(self.device ).long()
consonants = data[2].to(self.device ).long()
vowels = data[3].to(self.device ).long()
uniques = data[4].to(self.device ).long()
root_preds = self.model_root(inputs)
 self.model_root.zero_grad()
consonant_preds = self.model_consonant(inputs)
 self.model_consonant.zero_grad()
vowel_preds = self.model_vowel(inputs)
 self.model_vowel.zero_grad()
root, consonant, vowel, unique = self.model_multihead(inputs, uniques)
 root_preds = postprocess(torch.nn.Softmax(1 )(root_preds ).cpu().numpy() , 168, -0.9)
 consonant_preds = postprocess(torch.nn.Softmax(1 )(consonant_preds ).cpu().numpy() , 8, -0.6)
 vowel_preds = postprocess(torch.nn.Softmax(1 )(vowel_preds ).cpu().numpy() , 11, -1.1)
 root = postprocess(torch.nn.Softmax(1 )(root ).cpu().numpy() , 168, -1.2)
 consonant = postprocess(torch.nn.Softmax(1 )(consonant[:,:8] ).cpu().numpy() , 8, -0.5)
 vowel = postprocess(torch.nn.Softmax(1 )(vowel[:,:11] ).cpu().numpy() , 11, -1.2)
 unique_prob = F.softmax(unique, dim=1)
 for index in range(inputs.shape[0]):
if unique_prob[index].max(-1 ).values < 0.5:
output_roots.append(root_preds[index].item())
 output_consonants.append(consonant_preds[index].item())
 output_vowels.append(vowel_preds[index].item())
 else:
output_roots.append(root[index].item())
 output_consonants.append(consonant[index].item())
 output_vowels.append(vowel[index].item())
 del self.test_dataset.p
del self.test_dataset
del self.test_dataloader
row_id, target = [], []
for iid, r, c, v in zip(self.names, output_roots, output_consonants, output_vowels):
row_id.append(iid + '_grapheme_root')
 target.append(int(r))
row_id.append(iid + '_vowel_diacritic')
 target.append(int(v))
row_id.append(iid + '_consonant_diacritic')
 target.append(int(c))
count += 1
sub_fn = self.output_dir
sub = pd.DataFrame({'row_id': row_id, 'target': target})
 sub.to_csv(sub_fn, index=False)
 print(f'Done wrote to {sub_fn}')
 tester = Tester(dataset_path='./test_graphemes',
batch_size=128,
model_name='tf_efficientnet_b2_ns',
test_csv='/kaggle/input/bengaliai-cv19/test.csv',
unique_csv='/kaggle/input/bengaliai-cv19/train.csv',
output_dir='./submission.csv',
ckpt='/kaggle/input/graphemesingle/27.pth',
multihead_ckpt='/kaggle/input/graphemesingle/10.pth' 
)
 tester.test()<import_modules>
trainRF = train.copy()
testRF = test.copy()
trainRF,testRF = rowagg(trainRF,testRF )
class APINet(nn.Module):

 def __init__(self, out_channel):
super(APINet, self ).__init__()
backbone = models.densenet121(pretrained=False)
 backbone.features.conv0 = nn.Conv2d(1, 64, kernel_size=(7, 7),
stride=(2, 2), padding=(3, 3))
num_feature = backbone.classifier.in_features
layers = list(backbone.children())[:-1]
self.conv = nn.Sequential(*layers)
 self.fc = nn.Linear(num_feature, out_channel)
 embedding = 512
self.map = nn.Sequential(
 nn.Linear(num_feature * 2, embedding),
nn.LeakyReLU(inplace=True),
nn.Linear(embedding, embedding),
nn.LeakyReLU(inplace=True),
nn.Linear(embedding, num_feature), 
)
 self.sigmoid = nn.Sigmoid()
self.criterion = nn.CrossEntropyLoss()
self.rank_criterion = nn.MarginRankingLoss(margin=0)
 self.softmax_layer = nn.Softmax(dim=1)
 def forward(self, images):
conv_out = self.conv(images)
 pool_out = F.adaptive_avg_pool2d(conv_out,(1, 1)).squeeze()
return self.fc(pool_out)
 mat1 = np.arange(137)
 mat2 = np.arange(236)
 def Allign(image):

 img1 = image.mean(axis=1)
 img2 = image.mean(axis=0)
 imgall = img1.mean()
mean_x = int(( img1 * mat1 ).mean() / imgall)
 mean_y = int(( img2 * mat2 ).mean() / imgall)
 _t = min(68, mean_x)
 _b = min(69, 137 - mean_x)
 _l = min(118, mean_y)
 _r = min(118, 236 - mean_y)
 zeros = np.zeros(( 137, 236))
zeros[68-_t:68 + _b, 118-_l:118 + _r] = \
image[mean_x-_t:mean_x + _b, mean_y-_l:mean_y + _r]
return zeros[68 - 64: 68 + 64, 118 - 96: 118 + 96]
class GraphemeDataset(Dataset):
def __init__(self, images):
self.images = 255 - images.reshape(-1, 137, 236)
 def __len__(self):
return len(self.images)
 def __getitem__(self, idx):
image = self.images[idx] / 255
image = Allign(image)* 2 - 1
return image
def test() :
model_root = APINet(168 ).cuda()
model_root = nn.DataParallel(model_root)
 model_root.load_state_dict(torch.load(
 '/kaggle/input/models/DenseNet121root_3.pth'))
model_root.eval()
model_vowel = APINet(11 ).cuda()
model_vowel = nn.DataParallel(model_vowel)
 model_vowel.load_state_dict(torch.load(
 '/kaggle/input/models/DenseNet121vowel_3.pth'))
model_vowel.eval()
model_consonant = APINet(7 ).cuda()
model_consonant = nn.DataParallel(model_consonant)
 model_consonant.load_state_dict(torch.load(
 '/kaggle/input/models/DenseNet121consonant.pth'))
model_consonant.eval()
predictions = []
for i in range(4):
print(i)
 test = pd.read_parquet(
 '/kaggle/input/bengaliai-cv19/test_image_data_%d.parquet' % i)
 test_data = test.iloc[:, 1:].values
del test
test_image = GraphemeDataset(test_data)
 test_loader = torch.utils.data.DataLoader(test_image,
batch_size=256,
shuffle=False,
drop_last=False)
 with torch.no_grad() :
for data in test_loader:
data = data.cuda()
data = data.unsqueeze(1 ).float()
labels = np.zeros(( data.shape[0] * 3,), dtype=np.int)
 vowel = model_vowel(data)
 labels[2::3] = vowel.argmax(1 ).cpu().detach().numpy()
del vowel
root = model_root(data)
 labels[1::3] = root.argmax(1 ).cpu().detach().numpy()
del root
consonant = model_consonant(data)
 labels[::3] = consonant.argmax(1 ).cpu().detach().numpy()
del consonant
predictions.append(labels)
 del data
del labels
del test_data
del test_image
del test_loader
submission = pd.read_csv(
 '/kaggle/input/bengaliai-cv19/sample_submission.csv')
 submission.target = np.hstack(predictions)
 submission.head(10)
 submission.to_csv('submission.csv', index=False)
 test() 
<install_modules>
rf_clf=RandomForestRegressor(random_state=42,n_jobs=-1)
 rf_clf.fit(trainRF,y_train)
 rank = pd.DataFrame()
rank["importance"] = np.array(rf_clf.feature_importances_)
 rank["feature"] = np.array(trainRF.columns ).T
rank = rank.sort_values(by=['importance'], ascending=False)
 col = rank[:2000]
!pip install /kaggle/input/efficientnet-pytorch
!pip install /kaggle/input/pytorchcv/pytorchcv-0.0.57-py2.py3-none-any.whl<import_modules>
trainRF=trainRF[col.feature]
testRF=testRF[col.feature]
trainRF,testRF = maxabs(trainRF,testRF)
 X_train, X_test, Y_train, Y_test = train_test_split(trainRF, y_train, test_size=0.1, random_state=0)
 lgb_predRF, model, evals_resultRF = run_lgb(X_train, Y_train, X_test, Y_test, testRF )
<define_variables><EOS> 
sub=pd.read_csv('.. /input/sample_submission.csv')
 sub["target"] = lgb_predRF
sub.to_csv('sub.csv', index=False )