Kaggle competition: Prostate cANcer graDe Assessment (PANDA) Challenge
<SOS> metric: QuadraticWeightedKappa Kaggle data source: prostate-cancer-grade-assessment-(panda)-challenge<define_variables>
MODEL_DICT = {
'reg':{
'model1':{'ckpt':f'.. /input/pandachallengemodel/tiles_eb0_netvlad_reg_36_256_best_kappa_f0.pth',
'backbone':'efficientnet-b0','threshold':[0.5433,1.5323,2.4829,3.484,4.2806],'num_tiles':36},
},
'reg_with_attention':{
'model3':{'atten_ckpt':'.. /input/pandachallengemodel/newcv_tiles_eb0_attention_reg_64_256_f1.pth','atten_output':1,
'ckpt':'.. /input/pandachallengemodel/newcv_tiles_eb4_netvlad_attention_reg_16_256_f1.pth',
'backbone':'efficientnet-b4','threshold':[0.5445,1.4968,2.4163,3.4254,4.3853],'num_tiles':16}
},
'ordinal_reg':{
'model1':{'atten_ckpt':'.. /input/pandachallengemodel/newcv_tiles_eb0_attention_reg_64_256_f1.pth','atten_output':1,
'ckpt':'.. /input/pandachallengemodel/bag_eb0_attention_ordinal_reg_16_256_f1.pth',
'backbone':'efficientnet-b0','num_tiles':16},
},
'stitch_reg':{
'model1':{'atten_ckpt':'.. /input/pandachallengemodel/newcv_tiles_eb0_attention_reg_64_256_f1.pth','atten_output':1,
'ckpt':'.. /input/pandachallengemodel/stitched_regnety_800m_attention_reg_16_256_f1.pth',
'threshold':[0.5124,1.5636,2.4927,3.4076,4.4082],'num_tiles':16}}
}
set2=set(i for i in train[(train['move']==0)&(train['kills']>20)].index.tolist() )<define_variables>
ATTENTION_MEMORIES = {'.. /input/pandachallengemodel/newcv_tiles_eb0_attention_reg_64_256_f1.pth':None,
'.. /input/pandachallengemodel/tiles_eb0_attention_cls_64_256_f1.pth':None}
set3=set(i for i in train[(train['damageDealt']>4000)&(train['heals']<2)].index.tolist() )<define_variables>
!pip install.. /input/pytorch-lightning/pytorch_lightning-0.8.5-py3-none-any.whl
set4=set(i for i in train[(train['rideDistance']>25000)&(train['roadkillsrate']>.4)].index.tolist() )<define_variables>
sys.path = [
'.. /input/efficientnet-package/EfficientNet-PyTorch/',
'.. /input/efficienet062020/EfficientNet-PyTorch-master/',
'.. /input/regnet/'
] + sys.path
set5=set(i for i in train[(train['killStreaks']>3)&(train['weaponsAcquired']> 30)].index.tolist() )<define_variables>
warnings.filterwarnings("ignore")
set6=set(i for i in train[(train['killStreaks']>3)&(train['weaponsAcquired']> 30)].index.tolist() )<drop_column>
DATA = '.. /input/prostate-cancer-grade-assessment/'
SAMPLE = '.. /input/prostate-cancer-grade-assessment/sample_submission.csv'
df_train = pd.read_csv(os.path.join(DATA,'train.csv'))
df_test = pd.read_csv(os.path.join(DATA,'test.csv'))
image_folder = os.path.join(DATA,'test_images')
 is_test = os.path.exists(image_folder)
 image_folder = image_folder if is_test else os.path.join(DATA,'train_images')
 df = df_test if is_test else df_train.loc[:5]
tile_size = 256
batch_size = 2
num_tiles = 16
num_attention_tiles = 64
nworkers = 4
sets=set1 | set2 | set3 | set4 | set5 | set6
train.drop(train.index[list(sets)],inplace=True )<drop_column>
class NetVLAD(nn.Module):
def __init__(self, feature_size, max_frames,cluster_size, add_bn=False, truncate=True):
super(NetVLAD, self ).__init__()
self.feature_size = feature_size / 2 if truncate else feature_size
self.max_frames = max_frames
self.cluster_size = cluster_size
self.batch_norm = nn.BatchNorm1d(cluster_size, eps=1e-3, momentum=0.01)
 self.linear = nn.Linear(self.feature_size, self.cluster_size)
 self.softmax = nn.Softmax(dim=1)
 self.cluster_weights2 = nn.Parameter(torch.FloatTensor(1, self.feature_size,
self.cluster_size))
self.add_bn = add_bn
self.truncate = truncate
self.first = True
self.init_parameters()
def init_parameters(self):
init.normal(self.cluster_weights2, std=1 / math.sqrt(self.feature_size))
def forward(self, reshaped_input):
random_idx = torch.bernoulli(torch.Tensor([0.5]))
if self.truncate:
if self.training == True:
reshaped_input = reshaped_input[:, :self.feature_size].contiguous() if random_idx[0]==0 else reshaped_input[:, self.feature_size:].contiguous()
else:
if self.first == True:
reshaped_input = reshaped_input[:, :self.feature_size].contiguous()
else:
reshaped_input = reshaped_input[:, self.feature_size:].contiguous()
activation = self.linear(reshaped_input)
 if self.add_bn:
activation = self.batch_norm(activation)
 activation = self.softmax(activation ).view([-1, self.max_frames, self.cluster_size])
 a_sum = activation.sum(-2 ).unsqueeze(1)
 a = torch.mul(a_sum, self.cluster_weights2)
 activation = activation.permute(0, 2, 1 ).contiguous()
reshaped_input = reshaped_input.view([-1, self.max_frames, self.feature_size])
 vlad = torch.matmul(activation, reshaped_input ).permute(0, 2, 1 ).contiguous()
vlad = vlad.sub(a ).view([-1, self.cluster_size * self.feature_size])
 if self.training == False:
self.first = 1 - self.first
return vlad
class Model(nn.Module):
def __init__(self, num_clusters,num_tiles,num_classes=1,arch='efficientnet-b0'):
super().__init__()
self.base = enet.EfficientNet.from_name(arch)
 self.nc = self.base._fc.in_features
self.tile = nn.Sequential(
 nn.BatchNorm2d(self.nc,eps=0.001,momentum=0.01),
nn.AdaptiveAvgPool2d(output_size=1),
nn.Flatten() ,
nn.Dropout(p=0.2), 
)
 self.netvlad = NetVLAD(cluster_size=num_clusters,max_frames=num_tiles,
feature_size=self.nc,truncate=False)
 self.fc = nn.Linear(num_clusters*self.nc,num_classes)
 def forward(self, x):

 batch = x.shape[0]
shape = x[0].shape
n = shape[0]
x = x.reshape(-1,shape[1],shape[2],shape[3])
 x = self.base.extract_features(x)
 x = self.tile(x)
 x = x.view(batch,n,self.nc)
 x = self.netvlad(x)
 x = self.fc(x)
 return x
train.drop(train.index[list(sets)],inplace=True )<drop_column>
class AdaptiveConcatPool2d(nn.Module):
def __init__(self):
super().__init__()
self.avg = nn.AdaptiveAvgPool2d(output_size=(1, 1))
self.max = nn.AdaptiveMaxPool2d(output_size=(1, 1))
def forward(self, x):
avg_x = self.avg(x)
 max_x = self.max(x)
 return torch.cat([avg_x, max_x], dim=1 ).squeeze(2 ).squeeze(2)
 class Flatten(nn.Module):
def forward(self, x):
return x.view(x.shape[0], -1)
 class EfficientModel(nn.Module):
def __init__(self, c_out=6, n_tiles=36, tile_size=224, name='efficientnet-b0', strategy='stitched', head='basic'):
super().__init__()
m = EfficientNet.from_name(name, num_classes=c_out, in_channels=3)
 c_feature = m._fc.in_features
m._fc = nn.Identity()
self.feature_extractor = m
self.n_tiles = n_tiles
self.tile_size = tile_size
if strategy == 'stitched':
if head == 'basic':
self.head = nn.Linear(c_feature, c_out)
 elif head == 'concat':
m._avg_pooling = AdaptiveConcatPool2d()
self.head = nn.Linear(c_feature * 2, c_out)
 elif head == 'gem':
m._avg_pooling = GeM()
self.head = nn.Linear(c_feature, c_out)
 elif strategy == 'bag':
if head == 'basic':
self.head = BasicHead(c_feature, c_out, n_tiles)
 elif head == 'attention':
self.head = AttentionHead(c_feature, c_out, n_tiles)
 self.strategy = strategy
def forward(self, x):
if self.strategy == 'bag':
x = x.view(-1, 3, self.tile_size, self.tile_size)
 h = self.feature_extractor(x)
 h = self.head(h)
 return h
class BasicHead(nn.Module):
def __init__(self, c_in, c_out, n_tiles):
self.n_tiles = n_tiles
super().__init__()
self.fc = nn.Sequential(AdaptiveConcatPool2d() ,
nn.Linear(c_in * 2, c_out))
def forward(self, x):
bn, c = x.shape
h = x.view(-1, self.n_tiles, c, 1, 1 ).permute(0, 2, 1, 3, 4)\ 
.contiguous().view(-1, c, 1 * self.n_tiles, 1)
 h = self.fc(h)
 return h
class AttentionHead(nn.Module):
def __init__(self, c_in, c_out, n_tiles):
self.n_tiles = n_tiles
super().__init__()
self.attention_pool = AttentionPool(c_in, c_in//2)
 self.fc = nn.Linear(c_in, c_out)
 def forward(self, x):
bn, c = x.shape
h = x.view(-1, self.n_tiles, c)
 h = self.attention_pool(h)
 h = self.fc(h)
 return h
class AttentionPool(nn.Module):
def __init__(self, c_in, d):
super().__init__()
self.lin_V = nn.Linear(c_in, d)
 self.lin_w = nn.Linear(d, 1)
 def compute_weights(self, x):
key = self.lin_V(x)
 weights = self.lin_w(torch.tanh(key))
weights = torch.softmax(weights, dim=1)
 return weights
def forward(self, x):
weights = self.compute_weights(x)
 pooled = torch.matmul(x.transpose(1, 2), weights ).squeeze(2)
 return pooled
train.drop(['roadkillsrate'],axis=1,inplace=True)
 train.columns.drop('winPlacePerc')==test.columns<feature_engineering>
class OptimizedRounder() :
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
elif pred >= coef[3] and pred < coef[4]:
X_p[i] = 4
else:
X_p[i] = 5
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5, 4.5]
self.coef_ = scipy.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
elif pred >= coef[3] and pred < coef[4]:
X_p[i] = 4
else:
X_p[i] = 5
return X_p
def coefficients(self):
return self.coef_['x']
optimized_rounder = OptimizedRounder()
def new_features(df):
df['items']=df['boosts']+df['heals']
df['headshotKills_over_kills'] = df['headshotKills'] / df['kills']
df['headshotKills_over_kills'].fillna(0, inplace=True)
 df['killPlace_over_maxPlace'] = df['killPlace'] / df['maxPlace']
df['killPlace_over_maxPlace'].fillna(0, inplace=True)
 df['killPlace_over_maxPlace'].replace(np.inf, 0, inplace=True)
 df['walkDistance_over_heals'] = df['walkDistance'] / df['heals']
df['walkDistance_over_heals'].fillna(0, inplace=True)
 df['walkDistance_over_heals'].replace(np.inf, 0, inplace=True)
 df['walkDistance_over_kills'] = df['walkDistance'] / df['kills']
df['walkDistance_over_kills'].fillna(0, inplace=True)
 df['walkDistance_over_kills'].replace(np.inf, 0, inplace=True)
 df['teamwork'] = df['assists'] + df['revives']
return df<feature_engineering>
def preprocess(img,sz,num_tiles):
shape = img.shape
pad0,pad1 =(sz - shape[0]%sz)%sz,(sz - shape[1]%sz)%sz
img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],
constant_values=255)
 img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)
 img = img.transpose(0,2,1,3,4 ).reshape(-1,sz,sz,3)
 idxs = np.argsort(img.reshape(img.shape[0],-1 ).sum(-1)) [:num_tiles]
img = img[idxs]
return img
class PandaDataset(Dataset):
def __init__(self,path,test_df,num_tiles,attention_model=None):
self.path = path
self.names = list(test_df.image_id)
 self.num_tiles = num_tiles
self.attention_model = attention_model
self.attention_memory = {}
def __len__(self):
return len(self.names)
 def __getitem__(self,idx):
name = self.names[idx]
image = skimage.io.MultiImage(os.path.join(self.path,name+'.tiff')) [1]
tiles = preprocess(image,tile_size,num_attention_tiles)
 if self.attention_model:
images = np.zeros(( num_attention_tiles,3,tile_size,tile_size),np.float32)
 else:
tiles = self.get_sum_tiles(tiles)
 images = np.zeros(( self.num_tiles,3,tile_size,tile_size),np.float32)
 for i,tile in enumerate(tiles):
tile = tile.astype(np.float32)
 tile /=255.
 tile = tile.transpose(2,0,1)
 images[i,:,:,:] = tile
if self.attention_model:
images = self.get_best_tiles(images, name)
 return torch.tensor(images ).float() ,name
def get_sum_tiles(self,tiles):
idxs = np.argsort(tiles.reshape(tiles.shape[0],-1 ).sum(-1)) [:self.num_tiles]
tiles = tiles[idxs]
return tiles
def get_best_tiles(self,tiles,name):
if name in self.attention_memory:
weights = self.attention_memory[name]
else:
tile_bags = torch.tensor(tiles.copy())
 with torch.no_grad() :
features = self.attention_model.feature_extractor(tile_bags.cuda())
 features = features.view(-1, num_attention_tiles, features.shape[-1])
 weights = self.attention_model.head.attention_pool.compute_weights(features)
 weights = weights.reshape(-1 ).argsort(0 ).cpu().numpy() [::-1]
self.attention_memory[name] = weights
best_tiles = [tiles[w] for w in weights[:num_tiles]]
return best_tiles
train=new_features(train)
 test=new_features(test )<prepare_x_and_y>
PREDS = []
def feature_engineering(is_train=True):
if is_train:
print("processing train.csv")
 df = train
else:
print("processing test.csv")
 df = test
target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove('matchType')
 y = None
if is_train:
y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = np.array(df_out, dtype=np.float64)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y
x_train, y = feature_engineering(True )<feature_engineering>
MODELS = []
THRES = []
for k in MODEL_DICT['reg']:
ckpt = MODEL_DICT['reg'][k]['ckpt']
n_tiles = MODEL_DICT['reg'][k]['num_tiles']
backbone = MODEL_DICT['reg'][k]['backbone']
threshold = MODEL_DICT['reg'][k]['threshold']
state_dict = torch.load(ckpt,map_location=lambda storage, loc: storage)
 model = Model(num_clusters=6,num_tiles=n_tiles,num_classes=1,arch=backbone)
 model.load_state_dict(state_dict,strict=True)
 model.float()
model.eval()
model.cuda()
del state_dict
MODELS.append(model)
 THRES.append(threshold )
test,z=feature_engineering(False )<normalization>
sub_df = pd.read_csv(SAMPLE)
 if os.path.exists(DATA):
ds = PandaDataset(image_folder,df,36,attention_model=None)
 for model,threshold in zip(MODELS,THRES):
preds = []
names = []
with torch.no_grad() :
for x,y in tqdm(ds):
x = x.cuda()
x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),
x.transpose(-1,-2),x.transpose(-1,-2 ).flip(-1),
x.transpose(-1,-2 ).flip(-2),x.transpose(-1,-2 ).flip(-1,-2)],0)
 p = model(x)
 p = p.view(-1 ).mean().cpu()
names.append(y)
 preds.append(p)
 final_preds = torch.tensor(preds ).numpy()
final_preds = optimized_rounder.predict(final_preds,threshold)
 final_preds = final_preds.astype(int)
 PREDS.append(final_preds)
 NAMES = names
scaler=MinMaxScaler(feature_range=(-1,1),copy=False ).fit(x_train)
 x_train=scaler.transform(x_train )<normalization>
for model in MODELS:
del model
x_test=scaler.transform(test )<import_modules>
ATTENTION_MODELS = []
ATTENTION_MODELS_NAMES = []
for k in MODEL_DICT['reg_with_attention']:
atten_ckpt = MODEL_DICT['reg_with_attention'][k]['atten_ckpt']
atten_output = MODEL_DICT['reg_with_attention'][k]['atten_output']
state_dict = torch.load(atten_ckpt,map_location=lambda storage, loc: storage)
 attention_model = EfficientModel(c_out=atten_output,
n_tiles=num_attention_tiles,
tile_size=tile_size,
name='efficientnet-b0',
strategy='bag',
head='attention')
 attention_model.load_state_dict(state_dict,strict=True)
 attention_model.float()
attention_model.eval()
attention_model.cuda()
del state_dict
ATTENTION_MODELS.append(attention_model)
 ATTENTION_MODELS_NAMES.append(atten_ckpt)
from xgboost.sklearn import XGBRegressor
from sklearn.model_selection import cross_val_score,KFold
import numpy as np
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import Lasso,LinearRegression
from sklearn import preprocessing
from sklearn.model_selection import RandomizedSearchCV,train_test_split
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone<compute_train_metric>
MODELS = []
THRES = []
for k in MODEL_DICT['reg_with_attention']:
ckpt = MODEL_DICT['reg_with_attention'][k]['ckpt']
n_tiles = MODEL_DICT['reg_with_attention'][k]['num_tiles']
backbone = MODEL_DICT['reg_with_attention'][k]['backbone']
threshold = MODEL_DICT['reg_with_attention'][k]['threshold']
state_dict = torch.load(ckpt,map_location=lambda storage, loc: storage)
 model = Model(num_clusters=6,num_tiles=n_tiles,num_classes=1,arch=backbone)
 model.load_state_dict(state_dict,strict=True)
 model.float()
model.eval()
model.cuda()
del state_dict
MODELS.append(model)
 THRES.append(threshold )
n_folds=2
def mbs(model):
kf=KFold(n_folds,shuffle=True,random_state=42 ).get_n_splits(x_train)
 score=-cross_val_score(model,x_train,y,scoring='neg_mean_absolute_error',cv=kf)
 return score<choose_model_class>
sub_df = pd.read_csv(SAMPLE)
 if os.path.exists(DATA):
for model, threshold, atten_model, atten_model_name in zip(MODELS, THRES, ATTENTION_MODELS, ATTENTION_MODELS_NAMES):
ds = PandaDataset(image_folder,df,16,attention_model=atten_model)
 if ATTENTION_MEMORIES[atten_model_name] is not None:
ds.attention_memory = ATTENTION_MEMORIES[atten_model_name]
preds = []
names = []
with torch.no_grad() :
for x,y in tqdm(ds):
x = x.cuda()
x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),
x.transpose(-1,-2),x.transpose(-1,-2 ).flip(-1),
x.transpose(-1,-2 ).flip(-2),x.transpose(-1,-2 ).flip(-1,-2)],0)
 p = model(x)
 p = p.view(-1 ).mean().cpu()
names.append(y)
 preds.append(p)
 final_preds = torch.tensor(preds ).numpy()
final_preds = optimized_rounder.predict(final_preds,threshold)
 final_preds = final_preds.astype(int)
 PREDS.append(final_preds)
 NAMES = names
ATTENTION_MEMORIES[atten_model_name] = ds.attention_memory.copy()
lasso=Lasso(alpha=.005,random_state=1 )<define_search_space>
for model in ATTENTION_MODELS:
del model
for model in MODELS:
del model
params={'learning_rate':[.02,.03,.1],'min_child_weight':[4,6,8],'max_depth':[8,10],"subsample":[.6,.4],"n_estimators":[300,500]}<choose_model_class>
atten_ckpt = MODEL_DICT['ordinal_reg']['model1']['atten_ckpt']
atten_output = MODEL_DICT['ordinal_reg']['model1']['atten_output']
state_dict = torch.load(atten_ckpt,map_location=lambda storage, loc: storage)
 attention_model = EfficientModel(c_out=atten_output,
n_tiles=num_attention_tiles,
tile_size=tile_size,
name='efficientnet-b0',
strategy='bag',
head='attention')
 attention_model.load_state_dict(state_dict,strict=True)
 attention_model.float()
attention_model.eval()
attention_model.cuda()
xgb=XGBRegressor(learning_rate=.03,min_child_weight=4,max_depth=10,subsample=.4,n_estimators=500,n_jobs=-1)
<choose_model_class>
MODELS = []
for k in MODEL_DICT['ordinal_reg']:
ckpt = MODEL_DICT['ordinal_reg'][k]['ckpt']
n_tiles = MODEL_DICT['ordinal_reg'][k]['num_tiles']
backbone = MODEL_DICT['ordinal_reg'][k]['backbone']
state_dict = torch.load(ckpt,map_location=lambda storage, loc: storage)
 model = Model(num_clusters=6,num_tiles=n_tiles,num_classes=5,arch=backbone)
 model.load_state_dict(state_dict,strict=True)
 model.float()
model.eval()
model.cuda()
del state_dict
MODELS.append(model )
GBoost = GradientBoostingRegressor(n_estimators=600, learning_rate=0.05,
max_depth=4, max_features='auto',
min_samples_leaf=15, min_samples_split=5,
loss='huber', random_state =5 )<train_model>
sub_df = pd.read_csv(SAMPLE)
 atten_model_name = atten_ckpt
if os.path.exists(DATA):
ds = PandaDataset(image_folder,df,16,attention_model)
 if ATTENTION_MEMORIES[atten_model_name] is not None:
ds.attention_memory = ATTENTION_MEMORIES[atten_model_name]
for model in MODELS:
names,preds = [],[]
with torch.no_grad() :
for x,y in tqdm(ds):
x = x.cuda()
x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),
x.transpose(-1,-2),x.transpose(-1,-2 ).flip(-1),
x.transpose(-1,-2 ).flip(-2),x.transpose(-1,-2 ).flip(-1,-2)],0)
 p = model(x)
 p = torch.sigmoid(p)
 p = p.view(8,5 ).mean(0 ).sum().round().cpu()
names.append(y)
 preds.append(p)
 final_preds = torch.tensor(preds ).numpy()
final_preds = final_preds.astype(int)
 PREDS.append(final_preds)
 NAMES = names
ATTENTION_MEMORIES[atten_model_name] = ds.attention_memory.copy()
class Average_models(BaseEstimator,RegressorMixin,TransformerMixin):
def __init__(self,models):
self.models=models
def fit(self,X,y):
self.models_=[clone(x)for x in self.models]
for model in self.models_:
model.fit(X,y)
 return self
def predict(self,X):
predictions= np.column_stack([model.predict(X)for model in self.models_])
 return np.mean(predictions,axis=1 )<find_best_model_class>
ATTENTION_MODELS = []
ATTENTION_MODELS_NAMES = []
for k in MODEL_DICT['stitch_reg']:
atten_ckpt = MODEL_DICT['stitch_reg'][k]['atten_ckpt']
atten_output = MODEL_DICT['stitch_reg'][k]['atten_output']
state_dict = torch.load(atten_ckpt,map_location=lambda storage, loc: storage)
 attention_model = EfficientModel(c_out=atten_output,
n_tiles=num_attention_tiles,
tile_size=tile_size,
name='efficientnet-b0',
strategy='bag',
head='attention')
 attention_model.load_state_dict(state_dict,strict=True)
 attention_model.float()
attention_model.eval()
attention_model.cuda()
del state_dict
ATTENTION_MODELS.append(attention_model)
 ATTENTION_MODELS_NAMES.append(atten_ckpt)
training,valid,ytraining,yvalid=train_test_split(x_train,y)
 xgb.fit(training,ytraining)
 lasso.fit(training,ytraining)
 pred1=xgb.predict(valid)
 pred2=lasso.predict(valid)
 test_pred1=xgb.predict(test)
 test_pred2=lasso.predict(test)
 stacked_predictions=np.column_stack(( pred1,pred2))
stacked_test_prediction=np.column_stack(( test_pred1,test_pred2))
meta_model=LinearRegression()
meta_model.fit(stacked_predictions,yvalid)
 final=meta_model.predict(stacked_test_prediction )<save_to_csv>
MODELS = []
for k in MODEL_DICT['stitch_reg']:
ckpt = MODEL_DICT['stitch_reg'][k]['ckpt']
n_tiles = MODEL_DICT['stitch_reg'][k]['num_tiles']
state_dict = torch.load(ckpt,map_location=lambda storage, loc: storage)
 model = regnet.Regnet(1)
 model.load_state_dict(state_dict,strict=True)
 model.float()
model.eval()
model.cuda()
del state_dict
MODELS.append(model )
df=pd.DataFrame({'id':test_id,'winPlacePerc':final},columns=['id','winPlacePerc'])
 csv=df[['id','winPlacePerc']].to_csv('submission.csv',index=False )<save_to_csv>
def preprocess(img,sz,num_tiles):
shape = img.shape
pad0,pad1 =(sz - shape[0]%sz)%sz,(sz - shape[1]%sz)%sz
img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],
constant_values=255)
 img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)
 img = img.transpose(0,2,1,3,4 ).reshape(-1,sz,sz,3)
 if len(img)< num_tiles:
img = np.pad(img,[[0,num_tiles-len(img)],[0,0],[0,0],[0,0]],constant_values=255)
 idxs = np.argsort(img.reshape(img.shape[0],-1 ).sum(-1)) [:num_tiles]
img = img[idxs]
return img
class StitchPandaDataset(Dataset):
def __init__(self,path,test_df,num_tiles,attention_model):
self.path = path
self.names = list(test_df.image_id)
 self.attention_model = attention_model
self.num_tiles = num_tiles
def __len__(self):
return len(self.names)
 def __getitem__(self,idx):
name = self.names[idx]
image = skimage.io.MultiImage(os.path.join(self.path,name+'.tiff')) [1]
best_tiles = self.get_best_tiles(image)
 images = self.make_square(best_tiles)
 return torch.tensor(images ).float() ,name
def make_square(self, tiles):
idxes = list(range(self.num_tiles))
image_size = tiles[0].shape[0]
n_rows = int(self.num_tiles**0.5)
 images = np.zeros(( image_size * n_rows, image_size * n_rows, 3))
for h in range(n_rows):
for w in range(n_rows):
i = h * n_rows + w
this_img = tiles[idxes[i]]
this_img = 255 - this_img
h1 = h * image_size
w1 = w * image_size
images[h1:h1 + image_size, w1:w1 + image_size] = this_img
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 images = torch.tensor(images)
 return images
def get_best_tiles(self,imgs):
tiles = preprocess(imgs,tile_size,num_attention_tiles)
 tiles_bag = self.make_bag(tiles)
 with torch.no_grad() :
features = self.attention_model.feature_extractor(tiles_bag.cuda())
 features = features.view(-1, num_attention_tiles, features.shape[-1])
 weights = self.attention_model.head.attention_pool.compute_weights(features)
 weights = weights.reshape(-1 ).argsort(0 ).cpu().numpy() [::-1]
best_tiles = [tiles[w] for w in weights[:num_tiles]]
return best_tiles
def make_bag(self,tiles):
images = []
for tile in tiles:
tile = tile.astype(np.float32)
 images.append(tile)
 images = np.stack(images,axis=0)
 images = images.astype(np.float32)
 images = images.transpose(0,3,1,2)
 images /= 255
images = torch.tensor(images)
 return images
df = pd.read_csv('.. /input/12282247/result(4 ).csv')
 df.to_csv('submission.csv', index=False )<import_modules>
sub_df = pd.read_csv(SAMPLE)
 if os.path.exists(DATA):
for model, threshold, atten_model, atten_model_name in zip(MODELS,THRES, ATTENTION_MODELS, ATTENTION_MODELS_NAMES):
ds = StitchPandaDataset(image_folder,df,16,attention_model=atten_model)
 if ATTENTION_MEMORIES[atten_model_name] is not None:
ds.attention_memory = ATTENTION_MEMORIES[atten_model_name]
preds = []
names = []
with torch.no_grad() :
for x,y in tqdm(ds):
x = x.cuda()
x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),
x.transpose(-1,-2),x.transpose(-1,-2 ).flip(-1),
x.transpose(-1,-2 ).flip(-2),x.transpose(-1,-2 ).flip(-1,-2)],0)
 p = model(x)
 p = p.view(-1 ).mean().cpu()
names.append(y)
 preds.append(p)
 final_preds = torch.tensor(preds ).numpy()
final_preds = optimized_rounder.predict(final_preds,threshold)
 final_preds = final_preds.astype(int)
 PREDS.append(final_preds)
 NAMES = names
ATTENTION_MEMORIES[atten_model_name] = ds.attention_memory.copy()
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt<load_from_csv>
import numpy as np
import cv2
import pandas as pd
import torch
import torch.nn as nn
import torch.utils.data as tdata
import os
import pickle
from hubconf import *
from mish_activation import Mish
import skimage.io
from scipy.stats import mode
from efficientnet_pytorch.model import EfficientNet
df = pd.read_csv('.. /input/pubg-finish-placement-prediction/train_V2.csv' )<count_missing_values>
ATTENTION_MODELS = [f'.. /input/panda-localtrain/attention_gcp/fold_1/epoch28-kappa0.8734.ckpt']
MODELS = [f'.. /input/panda-localtrain/efficient0-20200618-064502/efficient0-20200618-064502/fold_1/epoch23-kappa0.9014.ckpt',
f'.. /input/panda-localtrain/efficient0-20200618-064502/efficient0-20200618-064502/fold_4/epoch26-kappa0.9034.ckpt',
f'.. /input/panda-localtrain/efficient0-20200709-212941/efficient0-20200709-212941/fold_1/epoch33-kappa0.9021.ckpt',
f'.. /input/panda-localtrain/efficient0-20200628-213437/efficient0-20200628-213437/fold_1/epoch27-kappa0.8957.ckpt',
f'.. /input/panda-localtrain/efficient0-20200616-200111/efficient0-20200616-200111/fold_1/epoch26-kappa0.8911.ckpt',
f'.. /input/panda-localtrain/25tiles-gcp/25tiles-gcp/fold_1/epoch29-kappa0.9046.ckpt',
f'.. /input/panda-localtrain/efficient0-20200708-205659/efficient0-20200708-205659/fold_1/epoch29-kappa0.8975.ckpt',
f'.. /input/panda-localtrain/efficient0-20200708-205659/efficient0-20200708-205659/fold_1/epoch27-kappa0.8982.ckpt']
attention_tiles_num = 128
df.isnull().sum()<filter>
class AdaptiveConcatPool2d(nn.Module):
def __init__(self):
super().__init__()
self.avg = nn.AdaptiveAvgPool2d(output_size=(1, 1))
self.max = nn.AdaptiveMaxPool2d(output_size=(1, 1))
def forward(self, x):
avg_x = self.avg(x)
 max_x = self.max(x)
 return torch.cat([avg_x, max_x], dim=1 ).squeeze(2 ).squeeze(2)
 class Flatten(nn.Module):
def forward(self, x):
return x.view(x.shape[0], -1)
 class EfficientModel(nn.Module):
def __init__(self, c_out=5, n_tiles=36, tile_size=224, name='efficientnet-b0', strategy='stitched', head='basic'):
super().__init__()
m = EfficientNet.from_name(name, num_classes=c_out, in_channels=3)
 c_feature = m._fc.in_features
m._fc = nn.Identity()
self.feature_extractor = m
self.n_tiles = n_tiles
self.tile_size = tile_size
if strategy == 'stitched':
if head == 'basic':
self.head = nn.Linear(c_feature, c_out)
 elif head == 'concat':
m._avg_pooling = AdaptiveConcatPool2d()
self.head = nn.Linear(c_feature * 2, c_out)
 elif head == 'gem':
m._avg_pooling = GeM()
self.head = nn.Linear(c_feature, c_out)
 elif strategy == 'bag':
if head == 'basic':
self.head = BasicHead(c_feature, c_out, n_tiles)
 elif head == 'attention':
self.head = AttentionHead(c_feature, c_out, n_tiles)
 self.strategy = strategy
def forward(self, x):
if self.strategy == 'bag':
x = x.view(-1, 3, self.tile_size, self.tile_size)
 h = self.feature_extractor(x)
 h = self.head(h)
 return h
class BasicHead(nn.Module):
def __init__(self, c_in, c_out, n_tiles):
self.n_tiles = n_tiles
super().__init__()
self.fc = nn.Sequential(AdaptiveConcatPool2d() ,
nn.Linear(c_in * 2, c_out))
def forward(self, x):
bn, c = x.shape
h = x.view(-1, self.n_tiles, c, 1, 1 ).permute(0, 2, 1, 3, 4)\ 
.contiguous().view(-1, c, 1 * self.n_tiles, 1)
 h = self.fc(h)
 return h
class AttentionHead(nn.Module):
def __init__(self, c_in, c_out, n_tiles):
self.n_tiles = n_tiles
super().__init__()
self.attention_pool = AttentionPool(c_in, c_in//2)
 self.fc = nn.Linear(c_in, c_out)
 def forward(self, x):
bn, c = x.shape
h = x.view(-1, self.n_tiles, c)
 h = self.attention_pool(h)
 h = self.fc(h)
 return h
class AttentionPool(nn.Module):
def __init__(self, c_in, d):
super().__init__()
self.lin_V = nn.Linear(c_in, d)
 self.lin_w = nn.Linear(d, 1)
 def compute_weights(self, x):
key = self.lin_V(x)
 weights = self.lin_w(torch.tanh(key))
weights = torch.softmax(weights, dim=1)
 return weights
def forward(self, x):
weights = self.compute_weights(x)
 pooled = torch.matmul(x.transpose(1, 2), weights ).squeeze(2)
 return pooled
xy=df[df['winPlacePerc']==1]
xy=xy[xy['damageDealt']<1000]
xy<set_options>
class TileMaker:
def __init__(self, size, number, scale):
self.size = size
self.number = number
self.scale = scale
def __pad(self, image, constant_values=255):
h, w, c = image.shape
horizontal_pad = 0 if(w % self.size)== 0 else self.size -(w % self.size)
 vertical_pad = 0 if(h % self.size)== 0 else self.size -(h % self.size)
 image = np.pad(image, pad_width=(( vertical_pad // 2, vertical_pad - vertical_pad // 2), 
(horizontal_pad // 2, horizontal_pad - horizontal_pad // 2), 
(0, 0)) ,
mode='constant', constant_values=constant_values)
 return image
def __call__(self, image, constant_values=255):
image = self.__pad(image, constant_values=constant_values)
 h, w, c = image.shape
image = image.reshape(h // self.size, self.size, w // self.size, self.size, c)
 image = image.swapaxes(1, 2 ).reshape(-1, self.size, self.size, c)
 if image.shape[0] < self.number:
image = np.pad(image, pad_width=(( 0, self.number - image.shape[0]),(0, 0),(0, 0),(0, 0)) ,
mode='constant', constant_values=constant_values)
 sorted_tiles = np.argsort(np.sum(image, axis=(1, 2, 3)))
 sorted_tiles = sorted_tiles[:self.number]
image = image[sorted_tiles]
return image
warnings.filterwarnings('ignore' )<load_from_csv>
def dict_to_args(d):
args = argparse.Namespace()
def dict_to_args_recursive(args, d, prefix=''):
for k, v in d.items() :
if type(v)== dict:
dict_to_args_recursive(args, v, prefix=k)
 elif type(v)in [tuple, list]:
continue
else:
if prefix:
args.__setattr__(prefix + '_' + k, v)
 else:
args.__setattr__(k, v)
 dict_to_args_recursive(args, d)
 return args
def feature_engineering(is_train=True,debug=True):
test_idx = None
if is_train:
print("processing train.csv")
 if debug == True:
df = pd.read_csv('.. /input/pubg-finish-placement-prediction/train_V2.csv', nrows=10000)
 else:
df = pd.read_csv('.. /input/pubg-finish-placement-prediction/train_V2.csv')
 df = df[df['maxPlace'] > 1]
else:
print("processing test.csv")
 df = pd.read_csv('.. /input/pubg-finish-placement-prediction/test_V2.csv')
 test_idx = df.Id
df = reduce_mem_usage(df)
 print("remove some columns")
 target = 'winPlacePerc'
features = list(df.columns)
 features.remove("Id")
 features.remove("matchId")
 features.remove("groupId")
 features.remove("matchType")
 y = None
if is_train:
print("get target")
 y = np.array(df.groupby(['matchId','groupId'])[target].agg('mean'), dtype=np.float64)
 features.remove(target)
 print("get group mean feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('mean')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
if is_train: df_out = agg.reset_index() [['matchId','groupId']]
else: df_out = df[['matchId','groupId']]
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_mean", "_mean_rank"], how='left', on=['matchId', 'groupId'])
 print("get group max feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('max')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_max", "_max_rank"], how='left', on=['matchId', 'groupId'])
 print("get group min feature")
 agg = df.groupby(['matchId','groupId'])[features].agg('min')
 agg_rank = agg.groupby('matchId')[features].rank(pct=True ).reset_index()
df_out = df_out.merge(agg.reset_index() , suffixes=["", ""], how='left', on=['matchId', 'groupId'])
 df_out = df_out.merge(agg_rank, suffixes=["_min", "_min_rank"], how='left', on=['matchId', 'groupId'])
 print("get group size feature")
 agg = df.groupby(['matchId','groupId'] ).size().reset_index(name='group_size')
 df_out = df_out.merge(agg, how='left', on=['matchId', 'groupId'])
 print("get match mean feature")
 agg = df.groupby(['matchId'])[features].agg('mean' ).reset_index()
df_out = df_out.merge(agg, suffixes=["", "_match_mean"], how='left', on=['matchId'])
 print("get match size feature")
 agg = df.groupby(['matchId'] ).size().reset_index(name='match_size')
 df_out = df_out.merge(agg, how='left', on=['matchId'])
 df_out.drop(["matchId", "groupId"], axis=1, inplace=True)
 X = df_out
feature_names = list(df_out.columns)
 del df, df_out, agg, agg_rank
gc.collect()
return X, y, feature_names, test_idx<prepare_x_and_y>
class Model(nn.Module):
def __init__(self, model):
super().__init__()
self.model = model
def forward(self, x):
return model(x )
X_train, y_train, train_columns, _ = feature_engineering(True,False)
 X_test, y, _ , test_idx = feature_engineering(False,True )<train_model>
attention_models = []
for attention_model_path in ATTENTION_MODELS:
device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 ckpt = torch.load(attention_model_path)
 hparams = dict_to_args(ckpt['hparams'])
 attention_model = EfficientModel(c_out=5,
n_tiles=attention_tiles_num,
tile_size=hparams.tile_size,
name=hparams.backbone,
strategy=hparams.strategy,
head=hparams.head 
)
 attention_model = Model(attention_model)
 attention_model.load_state_dict(ckpt['state_dict'])
 attention_model = attention_model.model.to(device)
 attention_model.eval() ;
attention_models.append(attention_model )
LR_model = LinearRegression(n_jobs=4, normalize=True)
 LR_model.fit(X_train,y_train )<compute_test_metric>
models = []
hparams_list = []
for model_path in MODELS:
ckpt = torch.load(model_path)
 try:
hparams = dict_to_args(ckpt['hyper_parameters'])
 except:
hparams = dict_to_args(ckpt['hparams'])
 if hparams.loss == 'mse':
c_out = 1
elif hparams.loss == 'bce':
c_out = 5
model = EfficientModel(c_out=c_out,
n_tiles=hparams.n_tiles,
tile_size=hparams.tile_size,
name=hparams.backbone,
strategy=hparams.strategy,
head=hparams.head 
)
 model = Model(model)
 model.load_state_dict(ckpt['state_dict'])
 model = model.model.to(device)
 model.eval() ;
models.append(model)
 hparams_list.append(hparams)
 print(hparams )
LR_model.score(X_train,y_train )<predict_on_test>
tilemaker = TileMaker(hparams.tile_size, attention_tiles_num, hparams.scale )
y_pred_train = LR_model.predict(X_train)
 y_pred_test = LR_model.predict(X_test )<load_from_csv>
def preprocess(tile):
tile = transforms.ToTensor()(tile)
 tile = 1 - tile
return tile
df_test = reduce_mem_usage(pd.read_csv('.. /input/pubg-finish-placement-prediction/test_V2.csv'))
df_test.shape<save_to_csv>
class InferenceDataset(tdata.Dataset):
def __init__(self, attention_model, img_path, df, level, tile_size, num_tiles, scale, transform=None):
self.img_path = img_path
self.tile_maker = TileMaker(tile_size, attention_tiles_num, scale)
 self.imgs_names = list(df.image_id)
 self.df = df
self.level = level
self.num_tiles = num_tiles
self.transform = transform
self.attention_model = attention_model
self.attention_memory = {}
def __len__(self):
return len(self.attention_dataset.imgs_names)
 def __getitem__(self, idx):
name = self.imgs_names[idx]
img = skimage.io.MultiImage(os.path.join(self.img_path, name + '.tiff')) [-self.level]
best_tiles = self.get_best_tiles(img, name, transform_id=0)
 images = self.make_square(best_tiles)
 best_tiles = self.get_best_tiles(img[128:-128], name, transform_id=1)
 images2 = self.make_square(best_tiles)
 best_tiles = self.get_best_tiles(img[:, 128:-128], name, transform_id=2)
 images3 = self.make_square(best_tiles)
 best_tiles = self.get_best_tiles(img[128:-128, 128:-128], name, transform_id=3)
 images4 = self.make_square(best_tiles)
 return images, images2, images3, images4, name
def __len__(self):
return len(self.imgs_names)
 def make_square(self, tiles):
idxes = list(range(self.num_tiles))
image_size = tiles[0].shape[0]
n_rows = int(self.num_tiles**0.5)
 images = np.zeros(( image_size * n_rows, image_size * n_rows, 3))
for h in range(n_rows):
for w in range(n_rows):
i = h * n_rows + w
this_img = tiles[idxes[i]]
this_img = 255 - this_img
if self.transform is not None:
this_img = self.transform(image=this_img)['image']
h1 = h * image_size
w1 = w * image_size
images[h1:h1 + image_size, w1:w1 + image_size] = this_img
if self.transform is not None:
images = self.transform(image=images)['image']
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 images = torch.tensor(images)
 return images
def get_best_tiles(self, img, name, transform_id):
tiles = self.tile_maker(img)
 tiles_bag = self.make_bag(tiles)
 memory_name = name + '_' + str(transform_id)
 if memory_name in self.attention_memory:
weights = self.attention_memory[memory_name]
else:
weights = 0
for attention_model in self.attention_model:
with torch.no_grad() :
features = attention_model.feature_extractor(tiles_bag.to(device))
features = features.view(-1, attention_tiles_num, features.shape[-1])
 weights += attention_model.head.attention_pool.compute_weights(features)
 weights /= len(self.attention_model)
 weights = weights.reshape(-1 ).argsort(0 ).cpu().numpy() [::-1]
self.attention_memory[memory_name] = weights.astype(np.int32)
 best_tiles = [tiles[w] for w in weights[:self.num_tiles]]
return best_tiles
def make_bag(self, tiles):
images = []
for tile in tiles:
tile = 255 - tile
if self.transform is not None:
tile = self.transform(image=tile)['image']
tile = tile.astype(np.float32)
 images.append(tile)
 images = np.stack(images, axis=0)
 images = images.astype(np.float32)
 images = images.transpose(0, 3, 1, 2)
 images /= 255
images = torch.tensor(images)
 return images
y_pred_test[y_pred_test>1] = 1
y_pred_test[y_pred_test<0] = 0
df_test['winPlacePerc'] = y_pred_test
submission = df_test[['Id', 'winPlacePerc']]
submission.to_csv('submission.csv', index=False )<set_options>
dfs_preds = []
previous_memory = None
for model, hparams in zip(models, hparams_list):
dataset = InferenceDataset(attention_models, image_folder, df, hparams.level, hparams.tile_size, hparams.n_tiles, hparams.scale)
 if previous_memory is not None:
dataset.attention_memory = previous_memory
ids = []
predictions = []
with torch.no_grad() :
for idx in range(len(dataset)) :
imgs, imgs2, imgs3, imgs4, name = dataset[idx]
imgs, imgs2, imgs3, imgs4 = imgs.unsqueeze(0), imgs2.unsqueeze(0), imgs3.unsqueeze(0), imgs4.unsqueeze(0)
 logits = model(imgs.to(device))
logits = logits + model(imgs2.to(device))
logits = logits + model(imgs3.to(device))
logits = logits + model(imgs4.to(device))
logits = logits/4
if hparams.loss == 'bce':
pred = logits.sigmoid().sum(1 ).round()
elif hparams.loss == 'mse':
pred =(logits.sigmoid().squeeze(1)* 5 ).round()
pred = pred.detach().cpu().numpy()
ids.append(name)
 predictions.append(pred)
 ids = np.array(ids)
 predictions = np.concatenate(predictions, axis=0)
 pred_df = pd.DataFrame({'id': ids, 'preds': predictions})
 dfs_preds.append(pred_df)
 previous_memory = dataset.attention_memory.copy()
pd.set_option('display.max_columns', None )<define_variables>
for i in range(len(dfs_preds)) :
PREDS.append(dfs_preds[i]['preds'].values )
test_size = 0.05
batch_size = 20000
env = 'kaggle'
if env == 'kaggle':
test_size = 0.0
batch_size = 100000<train_model>
from scipy.stats import mode
<load_from_csv><EOS> 
arr = np.stack(PREDS, axis=1)
 majority_vote = arr.mean(axis=1 ).round().astype(int)
 sub_df = pd.DataFrame({'image_id': ids, 'isup_grade': majority_vote})
 sub_df.to_csv('submission.csv', index=False )