Kaggle competition: Predict Future Sales
def euler_to_Rot(yaw, pitch, roll):
Y = np.array([[cos(yaw), 0, sin(yaw)],
[0, 1, 0],
[-sin(yaw), 0, cos(yaw)]])
 P = np.array([[1, 0, 0],
[0, cos(pitch), -sin(pitch)],
[0, sin(pitch), cos(pitch)]])
 R = np.array([[cos(roll), -sin(roll), 0],
[sin(roll), cos(roll), 0],
[0, 0, 1]])
 return np.dot(Y, np.dot(P, R))<feature_engineering>
train = pd.read_csv('.. /input/sales_train.csv')
 items = pd.read_csv('.. /input/items.csv')
 categories = pd.read_csv('.. /input/item_categories.csv')
 shops = pd.read_csv('.. /input/shops.csv')
 test = pd.read_csv('.. /input/test.csv')
 submission = pd.read_csv('.. /input/sample_submission.csv' )
IMG_WIDTH = 1600
IMG_HEIGHT = 700
MODEL_SCALE = 8
def _regr_preprocess(regr_dict):
for name in ['x', 'y', 'z']:
regr_dict[name] = regr_dict[name] / 100
regr_dict['roll'] = rotate(regr_dict['roll'], np.pi)
 regr_dict['pitch_sin'] = sin(regr_dict['pitch'])
 regr_dict['pitch_cos'] = cos(regr_dict['pitch'])
 regr_dict.pop('pitch')
 regr_dict.pop('id')
 return regr_dict
def _regr_back(regr_dict):
for name in ['x', 'y', 'z']:
regr_dict[name] = regr_dict[name] * 100
regr_dict['roll'] = rotate(regr_dict['roll'], -np.pi)
 pitch_sin = regr_dict['pitch_sin'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)
 pitch_cos = regr_dict['pitch_cos'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)
 regr_dict['pitch'] = np.arccos(pitch_cos)* np.sign(pitch_sin)
 return regr_dict
def preprocess_image(img):
img = img[img.shape[0] // 2:]
bg = np.ones_like(img)* img.mean(1, keepdims=True ).astype(img.dtype)
 bg = bg[:, :img.shape[1] // 4]
img = np.concatenate([bg, img, bg], 1)
 img = cv2.resize(img,(IMG_WIDTH, IMG_HEIGHT))
return(img / 255 ).astype('float32')
 def get_mask_and_regr(img, labels):
mask = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE], dtype='float32')
 regr_names = ['x', 'y', 'z', 'yaw', 'pitch', 'roll']
regr = np.zeros([IMG_HEIGHT // MODEL_SCALE, IMG_WIDTH // MODEL_SCALE, 7], dtype='float32')
 coords = str2coords(labels)
 xs, ys = get_img_coords(labels)
 for x, y, regr_dict in zip(xs, ys, coords):
x, y = y, x
x =(x - img.shape[0] // 2)* IMG_HEIGHT /(img.shape[0] // 2)/ MODEL_SCALE
x = np.round(x ).astype('int')
 y =(y + img.shape[1] // 4)* IMG_WIDTH /(img.shape[1] * 1.5)/ MODEL_SCALE
y = np.round(y ).astype('int')
 if x >= 0 and x < IMG_HEIGHT // MODEL_SCALE and y >= 0 and y < IMG_WIDTH // MODEL_SCALE:
mask[x, y] = 1
regr_dict = _regr_preprocess(regr_dict)
 regr[x, y] = [regr_dict[n] for n in sorted(regr_dict)]
return mask, regr<categorify>
train['item_id'].value_counts(ascending=False)[:5]
class CarDataset(Dataset):

 def __init__(self, dataframe, root_dir, training=True, transform=None):
self.df = dataframe
self.root_dir = root_dir
self.transform = transform
self.training = training
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
idx, labels = self.df.values[idx]
img_name = self.root_dir.format(idx)
 img0 = imread(img_name, True)
 img = preprocess_image(img0)
 img = np.rollaxis(img, 2, 0)
 if self.training:
mask, regr = get_mask_and_regr(img0, labels)
 regr = np.rollaxis(regr, 2, 0)
 else:
mask, regr = 0, 0
return [img, mask, regr]<split>
items.loc[items['item_id']==20949]
train_images_dir = PATH + 'train_images/{}.jpg'
test_images_dir = PATH + 'test_images/{}.jpg'
df_train, df_dev = train_test_split(train, test_size=0.1, random_state=63)
 df_test = test
train_dataset = CarDataset(df_train, train_images_dir)
 dev_dataset = CarDataset(df_dev, train_images_dir)
 test_dataset = CarDataset(df_test, test_images_dir )<load_pretrained>
categories.loc[categories['item_category_id']==71]
BATCH_SIZE = 1
train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=0)
 dev_loader = DataLoader(dataset=dev_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)
 test_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4 )<choose_model_class>
train['item_cnt_day'].sort_values(ascending=False)[:5]
class double_conv(nn.Module):

 def __init__(self, in_ch, out_ch):
super(double_conv, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv2d(in_ch, out_ch, 3, padding=1),
nn.BatchNorm2d(out_ch),
nn.ReLU(inplace=True),
nn.Conv2d(out_ch, out_ch, 3, padding=1),
nn.BatchNorm2d(out_ch),
nn.ReLU(inplace=True)
)
 def forward(self, x):
x = self.conv(x)
 return x
class up(nn.Module):
def __init__(self, in_ch, out_ch, bilinear=True):
super(up, self ).__init__()
if bilinear:
self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)
 else:
self.up = nn.ConvTranspose2d(in_ch//2, in_ch//2, 2, stride=2)
 self.conv = double_conv(in_ch, out_ch)
 def forward(self, x1, x2=None):
x1 = self.up(x1)
 diffY = x2.size() [2] - x1.size() [2]
diffX = x2.size() [3] - x1.size() [3]
x1 = F.pad(x1,(diffX // 2, diffX - diffX//2,
diffY // 2, diffY - diffY//2))
if x2 is not None:
x = torch.cat([x2, x1], dim=1)
 else:
x = x1
x = self.conv(x)
 return x
def get_mesh(batch_size, shape_x, shape_y):
mg_x, mg_y = np.meshgrid(np.linspace(0, 1, shape_y), np.linspace(0, 1, shape_x))
mg_x = np.tile(mg_x[None, None, :, :], [batch_size, 1, 1, 1] ).astype('float32')
 mg_y = np.tile(mg_y[None, None, :, :], [batch_size, 1, 1, 1] ).astype('float32')
 mesh = torch.cat([torch.tensor(mg_x ).to(device), torch.tensor(mg_y ).to(device)], 1)
 return mesh<load_pretrained>
train[train['item_cnt_day'] == 2169]
model_urls = {
'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth',
'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
'densenet201': 'https://download.pytorch.org/models/densenet201-c1103571.pth',
'inceptionresnetv2': 'http://data.lip6.fr/cadene/pretrainedmodels/inceptionresnetv2-520b38e4.pth',
}
def conv3x3(in_planes, out_planes, stride=1):

 return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
padding=1, bias=False)
 def conv1x1(in_planes, out_planes, stride=1):

 return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)
 class BasicBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1):
super(BasicBlock, self ).__init__()
self.conv1 = conv3x3(inplanes, planes, stride)
 self.bn1 = nn.GroupNorm(16, planes)
 self.conv2 = conv3x3(planes, planes)
 self.bn2 = nn.GroupNorm(16, planes)
 if stride != 1 or inplanes != planes:
self.downsample = nn.Sequential(
 conv1x1(inplanes, planes, stride), nn.GroupNorm(16, planes))
else:
self.downsample = None
def forward(self, x):
identity = x
out = F.relu(self.bn1(self.conv1(x)) , inplace=True)
 out = self.bn2(self.conv2(out))
if self.downsample is not None:
identity = self.downsample(x)
 out += identity
out = F.relu(out, inplace=True)
 return out
class Bottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1):
super(Bottleneck, self ).__init__()
self.conv1 = conv1x1(inplanes, planes)
 self.bn1 = nn.GroupNorm(16, planes)
 self.conv2 = conv3x3(planes, planes, stride)
 self.bn2 = nn.GroupNorm(16, planes)
 self.conv3 = conv1x1(planes, planes * self.expansion)
 self.bn3 = nn.GroupNorm(16, planes * self.expansion)
 if stride != 1 or inplanes != planes * self.expansion:
self.downsample = nn.Sequential(
 conv1x1(inplanes, planes * self.expansion, stride),
nn.GroupNorm(16, planes * self.expansion))
else:
self.downsample = None
def forward(self, x):
identity = x
out = F.relu(self.bn1(self.conv1(x)) , inplace=True)
 out = F.relu(self.bn2(self.conv2(out)) , inplace=True)
 out = self.bn3(self.conv3(out))
if self.downsample is not None:
identity = self.downsample(x)
 out += identity
out = F.relu(out)
 return out
class ResNetFeatures(nn.Module):
def __init__(self, block, layers, num_classes=1000, zero_init_residual=False):
super(ResNetFeatures, self ).__init__()
self.inplanes = 64
self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
bias=False)
 self.bn1 = nn.GroupNorm(16, 64)
 self.layer1 = self._make_layer(block, 64, layers[0])
 self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
 self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
 self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
 for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.constant_(m.bias, 0)
 if zero_init_residual:
for m in self.modules() :
if isinstance(m, Bottleneck):
nn.init.constant_(m.bn3.weight, 0)
 elif isinstance(m, BasicBlock):
nn.init.constant_(m.bn2.weight, 0)
 def _make_layer(self, block, planes, blocks, stride=1):
layers = []
layers.append(block(self.inplanes, planes, stride))
self.inplanes = planes * block.expansion
for _ in range(1, blocks):
layers.append(block(self.inplanes, planes))
return nn.Sequential(*layers)
 def forward(self, x):
conv1 = F.relu(self.bn1(self.conv1(x)) , inplace=True)
 conv1 = F.max_pool2d(conv1, 3, stride=2, padding=1)
 feats4 = self.layer1(conv1)
 feats8 = self.layer2(feats4)
 feats16 = self.layer3(feats8)
 feats32 = self.layer4(feats16)
 return feats8, feats16, feats32
def resnet18(pretrained=False, **kwargs):

 model = ResNetFeatures(BasicBlock, [2, 2, 2, 2], **kwargs)
 if pretrained:
_load_pretrained(model, model_zoo.load_url(model_urls['resnet18']))
return model
def resnext50(pretrained=False, **kwargs):

 model = ResNetFeatures(BasicBlock, [2, 2, 2, 2], **kwargs)
 if pretrained:
_load_pretrained(model, model_zoo.load_url(model_urls['resnext50_32x4d']))
return model
def resnet34(pretrained=False, **kwargs):

 model = ResNetFeatures(BasicBlock, [3, 4, 6, 3], **kwargs)
 if pretrained:
_load_pretrained(model, model_zoo.load_url(model_urls['resnet34']))
return model
def inceptionresnetv2(pretrained=False, **kwargs):

 model = ResNetFeatures(BasicBlock, [3, 4, 6, 3], **kwargs)
 if pretrained:
_load_pretrained(model, model_zoo.load_url(model_urls['inceptionresnetv2']))
return model
def densenet201(pretrained=False, **kwargs):

 model = ResNetFeatures(BasicBlock, [3, 4, 6, 3], **kwargs)
 if pretrained:
_load_pretrained(model, model_zoo.load_url(model_urls['densenet201']))
return model
def _load_pretrained(model, pretrained):
model_dict = model.state_dict()
pretrained = {k : v for k, v in pretrained.items() if k in model_dict}
model_dict.update(pretrained)
 model.load_state_dict(model_dict )<choose_model_class>
items[items['item_id'] == 11373]
base_model = inceptionresnetv2(pretrained=False)
 base_model<define_search_model>
train = train[train['item_cnt_day'] < 2000]
class CentResnet(nn.Module):

 def __init__(self, n_classes):
super(CentResnet, self ).__init__()
self.base_model = base_model
self.lat8 = nn.Conv2d(128, 256, 1)
 self.lat16 = nn.Conv2d(256, 256, 1)
 self.lat32 = nn.Conv2d(512, 256, 1)
 self.bn8 = nn.GroupNorm(16, 256)
 self.bn16 = nn.GroupNorm(16, 256)
 self.bn32 = nn.GroupNorm(16, 256)
 self.conv0 = double_conv(5, 64)
 self.conv1 = double_conv(64, 128)
 self.conv2 = double_conv(128, 512)
 self.conv3 = double_conv(512, 1024)
 self.mp = nn.MaxPool2d(2)
 self.up1 = up(1282 , 512)
 self.up2 = up(512 + 512, 256)
 self.outc = nn.Conv2d(256, n_classes, 1)
 def forward(self, x):
batch_size = x.shape[0]
mesh1 = get_mesh(batch_size, x.shape[2], x.shape[3])
 x0 = torch.cat([x, mesh1], 1)
 x1 = self.mp(self.conv0(x0))
x2 = self.mp(self.conv1(x1))
x3 = self.mp(self.conv2(x2))
x4 = self.mp(self.conv3(x3))
feats8, feats16, feats32 = self.base_model(x)
 lat8 = F.relu(self.bn8(self.lat8(feats8)))
 lat16 = F.relu(self.bn16(self.lat16(feats16)))
 lat32 = F.relu(self.bn32(self.lat32(feats32)))
 mesh2 = get_mesh(batch_size, lat32.shape[2], lat32.shape[3])
 feats = torch.cat([lat32, mesh2], 1)
 x = self.up1(feats, x4)
 x = self.up2(x, x3)
 x = self.outc(x)
 return x<set_options>
train['item_price'].sort_values(ascending=False)[:5]
class RAdam(Optimizer):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):
defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 self.buffer = [[None, None, None] for ind in range(10)]
super(RAdam, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(RAdam, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
buffered = self.buffer[int(state['step'] % 10)]
if state['step'] == buffered[0]:
N_sma, step_size = buffered[1], buffered[2]
else:
buffered[0] = state['step']
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 buffered[1] = N_sma
if N_sma >= 5:
step_size = group['lr'] * math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 else:
step_size = group['lr'] /(1 - beta1 ** state['step'])
 buffered[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma >= 5:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 else:
p_data_fp32.add_(-step_size, exp_avg)
 p.data.copy_(p_data_fp32)
 return loss<choose_model_class>
train[train['item_price'] == 307980]
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(device)
 n_epochs = 9
model = CentResnet(8 ).to(device)
 optimizer = optim.AdamW(model.parameters() , lr=0.001)
 exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=max(n_epochs, 10)* 3828 // 3, gamma=0.1 )<define_variables>
items[items['item_id'] == 6066]
img_batch = torch.randn(( 1,3,700,1600))<train_model>
train[train['item_id'] == 6066]
test = model(img_batch.to(device))<compute_test_metric>
train = train[train['item_price'] < 300000]
def criterion(prediction, mask, regr,weight=0.4, size_average=True):
pred_mask = torch.sigmoid(prediction[:, 0])
 mask_loss = mask * torch.log(pred_mask + 1e-12)+(1 - mask)* torch.log(1 - pred_mask + 1e-12)
 mask_loss = -mask_loss.mean(0 ).sum()
pred_regr = prediction[:, 1:]
regr_loss =(torch.abs(pred_regr - regr ).sum(1)* mask ).sum(1 ).sum(1)/ mask.sum(1 ).sum(1)
 regr_loss = regr_loss.mean(0)
 loss = mask_loss +regr_loss
if not size_average:
loss *= prediction.shape[0]
return loss ,mask_loss , regr_loss<train_model>
train['item_price'].sort_values() [:5]
def train(epoch, history=None):
model.train()
t = tqdm(train_loader)
 for batch_idx,(img_batch, mask_batch, regr_batch)in enumerate(t):
img_batch = img_batch.to(device)
 mask_batch = mask_batch.to(device)
 regr_batch = regr_batch.to(device)
 optimizer.zero_grad()
output = model(img_batch)
 if epoch < SWITCH_LOSS_EPOCH :
loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,1)
 else:
loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,0.5)
 t.set_description(f'train_loss(l={loss:.3f} )(m={mask_loss:.2f} )(r={regr_loss:.4f}')
 if history is not None:
history.loc[epoch + batch_idx / len(train_loader), 'train_loss'] = loss.data.cpu().numpy()
loss.backward()
optimizer.step()
exp_lr_scheduler.step()
print('Train Epoch: {} \tLR: {:.6f}\tLoss: {:.6f}\tMaskLoss: {:.6f}\tRegLoss: {:.6f}'.format(
 epoch,
optimizer.state_dict() ['param_groups'][0]['lr'],
loss.data,
mask_loss.data,
regr_loss.data))
def evaluate(epoch, history=None):
model.eval()
loss = 0
valid_loss = 0
valid_mask_loss = 0
valid_regr_loss = 0
with torch.no_grad() :
for img_batch, mask_batch, regr_batch in dev_loader:
img_batch = img_batch.to(device)
 mask_batch = mask_batch.to(device)
 regr_batch = regr_batch.to(device)
 output = model(img_batch)
 if epoch < SWITCH_LOSS_EPOCH :
loss,mask_loss, regr_loss= criterion(output, mask_batch, regr_batch,1, size_average=False)
 valid_loss += loss.data
valid_mask_loss += mask_loss.data
valid_regr_loss += regr_loss.data
else :
loss,mask_loss, regr_loss = criterion(output, mask_batch, regr_batch,0.5, size_average=False)
 valid_loss += loss.data
valid_mask_loss += mask_loss.data
valid_regr_loss += regr_loss.data
valid_loss /= len(dev_loader.dataset)
 valid_mask_loss /= len(dev_loader.dataset)
 valid_regr_loss /= len(dev_loader.dataset)
 if history is not None:
history.loc[epoch, 'dev_loss'] = valid_loss.cpu().numpy()
history.loc[epoch, 'mask_loss'] = valid_mask_loss.cpu().numpy()
history.loc[epoch, 'regr_loss'] = valid_regr_loss.cpu().numpy()
print('Dev loss: {:.4f}'.format(valid_loss))<save_model>
train[train['item_price'] == -1]
torch.save(model.state_dict() , './resnext50.pth' )<statistical_test>
price_correction = train[(train['shop_id'] == 32)&(train['item_id'] == 2973)&(train['date_block_num'] == 4)&(train['item_price'] > 0)].item_price.median()
train.loc[train['item_price'] < 0, 'item_price'] = price_correction
act = torch.nn.Sigmoid()
logtens = torch.from_numpy(logits)
 probs = act(logtens)
 probs = probs[probs>0]
print(probs )<train_model>
shops_train = train['shop_id'].nunique()
shops_test = test['shop_id'].nunique()
print('Shops in Training Set: ', shops_train)
 print('Shops in Test Set: ', shops_test )
DISTANCE_THRESH_CLEAR = 2
def convert_3d_to_2d(x, y, z, fx = 2304.5479, fy = 2305.8757, cx = 1686.2379, cy = 1354.9849):
return x * fx / z + cx, y * fy / z + cy
def optimize_xy(r, c, x0, y0, z0):
def distance_fn(xyz):
x, y, z = xyz
x, y = convert_3d_to_2d(x, y, z0)
 y, x = x, y
x =(x - IMG_SHAPE[0] // 2)* IMG_HEIGHT /(IMG_SHAPE[0] // 2)/ MODEL_SCALE
x = np.round(x ).astype('int')
 y =(y + IMG_SHAPE[1] // 4)* IMG_WIDTH /(IMG_SHAPE[1] * 1.5)/ MODEL_SCALE
y = np.round(y ).astype('int')
 return(x-r)**2 +(y-c)**2
res = minimize(distance_fn, [x0, y0, z0], method='Powell')
 x_new, y_new, z_new = res.x
return x_new, y_new, z0
def clear_duplicates(coords):
for c1 in coords:
xyz1 = np.array([c1['x'], c1['y'], c1['z']])
 for c2 in coords:
xyz2 = np.array([c2['x'], c2['y'], c2['z']])
 distance = np.sqrt(((xyz1 - xyz2)**2 ).sum())
 if distance < DISTANCE_THRESH_CLEAR:
if c1['confidence'] < c2['confidence']:
c1['confidence'] = -1
return [c for c in coords if c['confidence'] > 0]
def extract_coords(prediction):
logits = prediction[0]
regr_output = prediction[1:]
points = np.argwhere(logits > 0)
 col_names = sorted(['x', 'y', 'z', 'yaw', 'pitch_sin', 'pitch_cos', 'roll'])
 coords = []
for r, c in points:
regr_dict = dict(zip(col_names, regr_output[:, r, c]))
coords.append(_regr_back(regr_dict))
coords[-1]['confidence'] = 1 /(1 + np.exp(-logits[r, c]))
coords[-1]['x'], coords[-1]['y'], coords[-1]['z'] = optimize_xy(r, c, coords[-1]['x'], coords[-1]['y'], coords[-1]['z'])
 coords = clear_duplicates(coords)
 return coords
def coords2str(coords, names=['yaw', 'pitch', 'roll', 'x', 'y', 'z', 'confidence']):
s = []
for c in coords:
for n in names:
s.append(str(c.get(n, 0)))
 return ' '.join(s )<load_pretrained>
train.loc[train['shop_id'] == 0, 'shop_id'] = 57
test.loc[test['shop_id'] == 0, 'shop_id'] = 57
train.loc[train['shop_id'] == 1, 'shop_id'] = 58
test.loc[test['shop_id'] == 1, 'shop_id'] = 58
train.loc[train['shop_id'] == 10, 'shop_id'] = 11
test.loc[test['shop_id'] == 10, 'shop_id'] = 11
val_preds = []
dev_loader = DataLoader(dataset=dev_dataset, batch_size=4, shuffle=False, num_workers=4)
 model.eval()
for img, _, _ in tqdm(dev_loader):
with torch.no_grad() :
output = model(img.to(device))
output = output.data.cpu().numpy()
for out in output:
coords = extract_coords(out)
 s = coords2str(coords)
 val_preds.append(s )<save_to_csv>
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda row: row[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
df_dev['PredictionString'] = val_preds
df_dev.head()
df_dev.to_csv('val_predictions.csv', index=False )<load_pretrained>
le = preprocessing.LabelEncoder()
le.fit_transform(shops['city'] )
predictions = []
test_loader = DataLoader(dataset=test_dataset, batch_size=2, shuffle=False, num_workers=4)
 model.eval()
for img, _, _ in tqdm(test_loader):
with torch.no_grad() :
output = model(img.to(device))
output = output.data.cpu().numpy()
for out in output:
coords = extract_coords(out)
 s = coords2str(coords)
 predictions.append(s )<save_to_csv>
shops['city_label'] = le.fit_transform(shops['city'])
 shops.drop(['shop_name', 'city'], axis = 1, inplace = True)
 shops.head()
test = pd.read_csv(PATH + 'sample_submission.csv')
 test['PredictionString'] = predictions
test.to_csv('predictions.csv', index=False)
 test.head()<data_type_conversions>
items_train = train['item_id'].nunique()
items_test = test['item_id'].nunique()
print('Items in Training Set: ', items_train)
 print('Items in Test Set: ', items_test )
def expand_df(df, PredictionStringCols):
df = df.dropna().copy()
df['NumCars'] = [int(( x.count(' ')+1)/7)for x in df['PredictionString']]
image_id_expanded = [item for item, count in zip(df['ImageId'], df['NumCars'])for i in range(count)]
prediction_strings_expanded = df['PredictionString'].str.split(' ',expand = True ).values.reshape(-1,7 ).astype(float)
 prediction_strings_expanded = prediction_strings_expanded[~np.isnan(prediction_strings_expanded ).all(axis=1)]
df = pd.DataFrame(
 {
'ImageId': image_id_expanded,
PredictionStringCols[0]:prediction_strings_expanded[:,0],
PredictionStringCols[1]:prediction_strings_expanded[:,1],
PredictionStringCols[2]:prediction_strings_expanded[:,2],
PredictionStringCols[3]:prediction_strings_expanded[:,3],
PredictionStringCols[4]:prediction_strings_expanded[:,4],
PredictionStringCols[5]:prediction_strings_expanded[:,5],
PredictionStringCols[6]:prediction_strings_expanded[:,6]
})
 return df
def str2coords(s, names):
coords = []
for l in np.array(s.split() ).reshape([-1, 7]):
coords.append(dict(zip(names, l.astype('float'))))
return coords
def TranslationDistance(p,g, abs_dist = False):
dx = p['x'] - g['x']
dy = p['y'] - g['y']
dz = p['z'] - g['z']
diff0 =(g['x']**2 + g['y']**2 + g['z']**2)**0.5
diff1 =(dx**2 + dy**2 + dz**2)**0.5
if abs_dist:
diff = diff1
else:
diff = diff1/diff0
return diff
def RotationDistance(p, g):
true=[ g['pitch'] ,g['yaw'] ,g['roll'] ]
pred=[ p['pitch'] ,p['yaw'] ,p['roll'] ]
q1 = R.from_euler('xyz', true)
 q2 = R.from_euler('xyz', pred)
 diff = R.inv(q2)* q1
W = np.clip(diff.as_quat() [-1], -1., 1.)
W =(acos(W)*360)/pi
if W > 180:
W = 360 - W
return W<define_search_space>
len(set(items_test_list ).difference(items_train_list))
thres_tr_list = [0.1, 0.09, 0.08, 0.07, 0.06, 0.05, 0.04, 0.03, 0.02, 0.01]
thres_ro_list = [50, 45, 40, 35, 30, 25, 20, 15, 10, 5]
def check_match(idx):
keep_gt=False
thre_tr_dist = thres_tr_list[idx]
thre_ro_dist = thres_ro_list[idx]
train_dict = {imgID:str2coords(s, names=['carid_or_score', 'pitch', 'yaw', 'roll', 'x', 'y', 'z'])for imgID,s in zip(train_df['ImageId'],train_df['PredictionString'])}
valid_dict = {imgID:str2coords(s, names=['pitch', 'yaw', 'roll', 'x', 'y', 'z', 'carid_or_score'])for imgID,s in zip(valid_df['ImageId'],valid_df['PredictionString'])}
result_flg = []
scores = []
MAX_VAL = 10**10
for img_id in valid_dict:
for pcar in sorted(valid_dict[img_id], key=lambda x: -x['carid_or_score']):
min_tr_dist = MAX_VAL
min_idx = -1
for idx, gcar in enumerate(train_dict[img_id]):
tr_dist = TranslationDistance(pcar,gcar)
 if tr_dist < min_tr_dist:
min_tr_dist = tr_dist
min_ro_dist = RotationDistance(pcar,gcar)
 min_idx = idx
if min_tr_dist < thre_tr_dist and min_ro_dist < thre_ro_dist:
if not keep_gt:
train_dict[img_id].pop(min_idx)
 result_flg.append(1)
 else:
result_flg.append(0)
 scores.append(pcar['carid_or_score'])
 return result_flg, scores<load_from_csv>
categories.loc[~categories['item_category_id'].isin(categories_in_test)].T
valid_df = pd.read_csv('val_predictions.csv')
 expanded_valid_df = expand_df(valid_df, ['pitch','yaw','roll','x','y','z','Score'])
 valid_df = valid_df.fillna('')
 train_df = pd.read_csv('.. /input/pku-autonomous-driving/train.csv')
 train_df = train_df[train_df.ImageId.isin(valid_df.ImageId.unique())]
expanded_train_df = expand_df(train_df, ['model_type','pitch','yaw','roll','x','y','z'])
 max_workers = 10
n_gt = len(expanded_train_df)
 ap_list = []
p = Pool(processes=max_workers)
 for result_flg, scores in p.imap(check_match, range(10)) :
if np.sum(result_flg)> 0:
n_tp = np.sum(result_flg)
 recall = n_tp/n_gt
ap = average_precision_score(result_flg, scores)*recall
else:
ap = 0
ap_list.append(ap)
 map = np.mean(ap_list)
 print('map:', map )<install_modules>
le = preprocessing.LabelEncoder()
main_categories = categories['item_category_name'].str.split('-')
 categories['main_category_id'] = main_categories.map(lambda row: row[0].strip())
 categories['main_category_id'] = le.fit_transform(categories['main_category_id'])
 categories['sub_category_id'] = main_categories.map(lambda row: row[1].strip() if len(row)> 1 else row[0].strip())
 categories['sub_category_id'] = le.fit_transform(categories['sub_category_id'] )
!pip -q install efficientnet-pytorch<import_modules>
train['date'] = pd.to_datetime(train['date'], format='%d.%m.%Y' )
RandomGamma, HueSaturationValue, RGBShift, MotionBlur, Blur,
GaussNoise, ChannelShuffle 
)
 osj = os.path.join
osdir = os.listdir
base_seed = 1289
def torch_seed_everything(seed=base_seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch_seed_everything()<set_options>
from itertools import product
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(device)
 if device == 'cuda':
torch.cuda.empty_cache()
model, optimizer, loss =0,0,0
gc.collect() 
<load_pretrained>
cartesian_test = []
cartesian_test.append(np.array(jan))
cartesian_test.append(np.array(feb))
platform = 'Kaggle'
if platform == 'Colab':
drive.mount("/content/drive", force_remount=False)
<define_variables>
cartesian_test = np.vstack(cartesian_test )
debug = False
effnet_ver = 'b0'
dropout_rate = 0.02
is_resume_train = True
n_samples = 100_000 if not debug else 20
img_h, img_w = 512, 2048
img_orig_h, img_orig_w = 2710, 3384
model_scale = 8
batch_size = 2
num_epochs = 6 if not debug else 2
num_epochs_resume = 3 if not debug else 2
start_epoch = 0
grad_accu_steps = 1
if debug:
grad_accu_steps = 1
num_splits = 8
fold_to_train = 3
date_time_str = time.strftime("%d_%m_time_%H_%M", time.localtime())
 path_dict = {'Kaggle': '.. /input/pku-autonomous-driving/',
'Colab': './drive/My Drive/Colab/Peking/Data/',
'gcp': 'gs://peking_12/data/raw_data',
'local': '/home/polk/Kaggle/Peking/data/raw_data'}
train_test_path_dict = {'Kaggle': '.. /input/pku-autonomous-driving/',
'Colab': './drive/My Drive/Colab/Peking/Data/',
'gcp': 'gs://peking_12/data/raw_data',
'local': '/home/polk/Kaggle/Peking/data'}
path_out_dict = {'Kaggle': './',
'Colab': f"./drive/My Drive/Colab/Peking/Output/Output_kaggle_eff_B0_{date_time_str}",
'gcp': f"/home/ik/peking/logs/output_{date_time_str}",
'local': f"/home/polk/Kaggle/Peking/Output_eff_{date_time_str}"}
path = path_dict[platform]
train_test_path = train_test_path_dict[platform]
out_path = path_out_dict[platform]
prev_model_fn_path = '.. /input/dropout-change-eff-b0-2-01-no-aug-epochs-6/model_fld_3_epoch_end_date_02_01_time_06_25.pth'
if platform != 'Kaggle':
if not os.path.exists(out_path): os.mkdir(out_path)
 print("
train and test images dir:", train_test_path)
 print("
train.csv, samples_submission.csv,...directory:", path)
 print("
logs, output dir:", out_path)
 print("
path directory list:", os.listdir(path))<load_from_csv>
cartesian_test_df = pd.DataFrame(cartesian_test, columns = ['shop_id', 'item_id', 'date_block_num'] )
train = pd.read_csv(osj(path, 'train.csv'), nrows = n_samples)
 drop_images = ['ID_1a5a10365', 'ID_4d238ae90.jpg', 'ID_408f58e9f', 'ID_bb1d991f6', 'ID_c44983aeb']
train = train[~train['ImageId'].isin(drop_images)]
test = pd.read_csv(osj(path, 'sample_submission.csv'), nrows = n_samples)
 camera_matrix = np.array([[2304.5479, 0, 1686.2379],
[0, 2305.8757, 1354.9849],
[0, 0, 1]], dtype=np.float32)
 camera_matrix_inv = np.linalg.inv(camera_matrix)
 print("train.shape:", train.shape, "\ttest.shape:", test.shape)
 train.head(2 )<categorify>
def downcast_dtypes(df):

 float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype == "int64"]
df[float_cols] = df[float_cols].astype(np.float16)
 df[int_cols] = df[int_cols].astype(np.int16)
 return df
def imread(path, fast_mode=False):
img = cv2.imread(path)
 if not fast_mode and img is not None and len(img.shape)== 3:
img = np.array(img[:, :, ::-1])
 return img
def str2coords(s, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']):

 coords = []
for l in np.array(s.split() ).reshape([-1, 7]):
coords.append(dict(zip(names, l.astype('float'))))
if 'id' in coords[-1]:
coords[-1]['id'] = int(coords[-1]['id'])
 return coords
def rotate(x, angle):
x = x + angle
x = x -(x + np.pi)//(2 * np.pi)* 2 * np.pi
return x
def add_number_of_cars(df):

 df['numcars'] = [int(( x.count(' ')+1)/7)for x in df['PredictionString']]
return df<train_model>
cartesian_df = pd.DataFrame(np.vstack(cartesian), columns = ['shop_id', 'item_id', 'date_block_num'], dtype=np.int32 )
DISTANCE_THRESH_CLEAR = 2
def convert_3d_to_2d(x, y, z, fx = 2304.5479, fy = 2305.8757, cx = 1686.2379, cy = 1354.9849):
return x * fx / z + cx, y * fy / z + cy
def optimize_xy(r, c, x0, y0, z0):
def distance_fn(xyz):
x, y, z = xyz
x, y = convert_3d_to_2d(x, y, z0)
 y, x = x, y
x =(x - img_orig_h // 2)* img_h /(img_orig_h // 2)/ model_scale
x = np.round(x ).astype('int')
 y =(y + img_orig_w // 4)* img_w /(img_orig_w * 1.5)/ model_scale
y = np.round(y ).astype('int')
 return(x-r)**2 +(y-c)**2
res = minimize(distance_fn, [x0, y0, z0], method='Powell')
 x_new, y_new, z_new = res.x
return x_new, y_new, z0
def clear_duplicates(coords):
for c1 in coords:
xyz1 = np.array([c1['x'], c1['y'], c1['z']])
 for c2 in coords:
xyz2 = np.array([c2['x'], c2['y'], c2['z']])
 distance = np.sqrt(((xyz1 - xyz2)**2 ).sum())
 if distance < DISTANCE_THRESH_CLEAR:
if c1['confidence'] < c2['confidence']:
c1['confidence'] = -1
return [c for c in coords if c['confidence'] > 0]
def extract_coords(prediction, threshold=0):
logits = prediction[0]
regr_output = prediction[1:]
points = np.argwhere(logits > threshold)
 col_names = sorted(['x', 'y', 'z', 'yaw', 'pitch_sin', 'pitch_cos', 'roll'])
 coords = []
for r, c in points:
regr_dict = dict(zip(col_names, regr_output[:, r, c]))
coords.append(_regr_back(regr_dict))
coords[-1]['confidence'] = 1 /(1 + np.exp(-logits[r, c]))
coords[-1]['x'], coords[-1]['y'], coords[-1]['z'] = optimize_xy(r, c, coords[-1]['x'], coords[-1]['y'], coords[-1]['z'])
 coords = clear_duplicates(coords)
 return coords
def coords2str(coords, names=['yaw', 'pitch', 'roll', 'x', 'y', 'z', 'confidence']):
s = []
for c in coords:
for n in names:
s.append(str(c.get(n, 0)))
 return ' '.join(s)
 def get_img_coords(s):

 coords = str2coords(s)
 xs = [c['x'] for c in coords]
ys = [c['y'] for c in coords]
zs = [c['z'] for c in coords]
P = np.array(list(zip(xs, ys, zs)) ).T
img_p = np.dot(camera_matrix, P ).T
img_p[:, 0] /= img_p[:, 2]
img_p[:, 1] /= img_p[:, 2]
img_xs = img_p[:, 0]
img_ys = img_p[:, 1]
img_zs = img_p[:, 2]
return img_xs, img_ys<feature_engineering>
x = train.groupby(['shop_id', 'item_id', 'date_block_num'])['item_cnt_day'].sum().rename('item_cnt_month' ).reset_index()
x.head()
def _regr_preprocess(regr_dict):
for name in ['x', 'y', 'z']:
regr_dict[name] = regr_dict[name] / 100
regr_dict['roll'] = rotate(regr_dict['roll'], np.pi)
 regr_dict['pitch_sin'] = sin(regr_dict['pitch'])
 regr_dict['pitch_cos'] = cos(regr_dict['pitch'])
 regr_dict.pop('pitch')
 regr_dict.pop('id')
 return regr_dict
def _regr_back(regr_dict):
for name in ['x', 'y', 'z']:
regr_dict[name] = regr_dict[name] * 100
regr_dict['roll'] = rotate(regr_dict['roll'], -np.pi)
 pitch_sin = regr_dict['pitch_sin'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)
 pitch_cos = regr_dict['pitch_cos'] / np.sqrt(regr_dict['pitch_sin']**2 + regr_dict['pitch_cos']**2)
 regr_dict['pitch'] = np.arccos(pitch_cos)* np.sign(pitch_sin)
 return regr_dict
def preprocess_image(img):
img = img[img.shape[0] // 2:]
bg = np.ones_like(img)* img.mean(1, keepdims=True ).astype(img.dtype)
 bg = bg[:, :img.shape[1] // 4]
img = np.concatenate([bg, img, bg], 1)
 img = cv2.resize(img,(img_w, img_h))
return img
def get_mask_and_regr(img_orig_height, img_orig_width, labels):
mask = np.zeros([img_h // model_scale, img_w // model_scale], dtype='float32')
 regr_names = ['x', 'y', 'z', 'yaw', 'pitch', 'roll']
regr = np.zeros([img_h // model_scale, img_w // model_scale, 7], dtype='float32')
 coords = str2coords(labels)
 xs, ys = get_img_coords(labels)
 for x, y, regr_dict in zip(xs, ys, coords):
x, y = y, x
x =(x - img_orig_height // 2)* img_h /(img_orig_height // 2)/ model_scale
x = np.round(x ).astype('int')
 y =(y + img_orig_width // 4)* img_w /(img_orig_width * 1.5)/ model_scale
y = np.round(y ).astype('int')
 if x >= 0 and x < img_h // model_scale and y >= 0 and y < img_w // model_scale:
mask[x, y] = 1
regr_dict = _regr_preprocess(regr_dict)
 regr[x, y] = [regr_dict[n] for n in sorted(regr_dict)]
return mask, regr<prepare_x_and_y>
new_train = pd.merge(cartesian_df, x, on=['shop_id', 'item_id', 'date_block_num'], how='left' ).fillna(0 )
def remove_out_image_cars(df):
def isnot_out(x,y):
return(x>=0)&(x<=img_orig_w)&(y>=0)&(y<=img_orig_h)
 df = add_number_of_cars(df)
 new_str_coords = []
counter_all_ls = []
for idx,str_coords in enumerate(df['PredictionString']):
coords = str2coords(str_coords, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z'])
 xs, ys = get_img_coords(str_coords)
 counter = 0
coords_new = []
for(item,x,y)in zip(coords, xs, ys):
if isnot_out(x, y):
coords_new.append(item)
 counter += 1
new_str_coords.append(coords2str(coords_new, names=['id', 'yaw', 'pitch', 'roll', 'x', 'y', 'z']))
counter_all_ls.append(counter)
 df['new_pred_string'] = new_str_coords
df['new_numcars'] = counter_all_ls
print("num of cars outside image bounds:", df['numcars'].sum() -df['new_numcars'].sum() ,
"out of all", df['numcars'].sum() , " cars in train")
 del df['PredictionString'], train['numcars']
df.rename(columns={'new_pred_string': 'PredictionString'}, inplace=True)
 return df
train = remove_out_image_cars(train )<prepare_x_and_y>
new_train['item_cnt_month'] = np.clip(new_train['item_cnt_month'], 0, 20 )
def euler_to_Rot(yaw, pitch, roll):
Y = np.array([[cos(yaw), 0, sin(yaw)],
[0, 1, 0],
[-sin(yaw), 0, cos(yaw)]])
 P = np.array([[1, 0, 0],
[0, cos(pitch), -sin(pitch)],
[0, sin(pitch), cos(pitch)]])
 R = np.array([[cos(roll), -sin(roll), 0],
[sin(roll), cos(roll), 0],
[0, 0, 1]])
 return np.dot(Y, np.dot(P, R))
def draw_line(image, points):
color =(255, 0, 0)
 cv2.line(image, tuple(points[0][:2]), tuple(points[3][:2]), color, 16)
 cv2.line(image, tuple(points[0][:2]), tuple(points[1][:2]), color, 16)
 cv2.line(image, tuple(points[1][:2]), tuple(points[2][:2]), color, 16)
 cv2.line(image, tuple(points[2][:2]), tuple(points[3][:2]), color, 16)
 return image
def draw_points(image, points):
for(p_x, p_y, p_z)in points:
cv2.circle(image,(p_x, p_y), int(1000 / p_z),(0, 255, 0), -1)
 return image
def visualize(img, coords):
x_l = 1.02
y_l = 0.80
z_l = 2.31
img = img.copy()
for point in coords:
x, y, z = point['x'], point['y'], point['z']
yaw, pitch, roll = -point['pitch'], -point['yaw'], -point['roll']
Rt = np.eye(4)
 t = np.array([x, y, z])
 Rt[:3, 3] = t
Rt[:3, :3] = euler_to_Rot(yaw, pitch, roll ).T
Rt = Rt[:3, :]
P = np.array([[x_l, -y_l, -z_l, 1],
[x_l, -y_l, z_l, 1],
[-x_l, -y_l, z_l, 1],
[-x_l, -y_l, -z_l, 1],
[0, 0, 0, 1]] ).T
img_cor_points = np.dot(camera_matrix, np.dot(Rt, P))
img_cor_points = img_cor_points.T
img_cor_points[:, 0] /= img_cor_points[:, 2]
img_cor_points[:, 1] /= img_cor_points[:, 2]
img_cor_points = img_cor_points.astype(int)
 img = draw_line(img, img_cor_points)
 img = draw_points(img, img_cor_points[-1:])
 return img<categorify>
del x
del cartesian_df
del cartesian
del cartesian_test
del cartesian_test_df
del feb
del jan
del items_test_list
del items_train_list
del train
means = np.array([80.302637, 73.818344, 69.698726] ).reshape(1,1,3)
 stds = np.array([52.455102, 52.848600, 50.285960] ).reshape(1,1,3)
 def normalize(img, means, stds, tensor=False):
return(img - means)/stds
def denormalize(img, means, stds, resize_to_original=False):

 img = np.moveaxis(img, 0, 2)
 img = img*stds + means
img = np.clip(img, 0, 255 ).astype('uint8')
 if resize_to_original:
img = img[:,(img_w//4):(img_w - img_w//4),:]
img = cv2.copyMakeBorder(img, img.shape[0], 0,0,0, cv2.BORDER_CONSTANT)
 img = cv2.resize(img,(img_orig_w, img_orig_h))
return img
class CarDataset(Dataset):

 def __init__(self, dataframe, root_dir, root_dir_dropmasks=osj(path,'test_masks/{}.jpg'), training=True, transform=None):
self.df = dataframe
self.root_dir = root_dir
self.root_dir_dropmasks = root_dir_dropmasks
self.transform = transform
self.training = training
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
idx, labels = self.df.values[idx]
img_name = self.root_dir.format(idx)
 img = imread(img_name, True)
 img = preprocess_image(img)
 if self.transform:
img = self.transform(image=img)['image']
img = normalize(img, means, stds)
 img = np.rollaxis(img, 2, 0)
 if self.training:
mask, regr = get_mask_and_regr(img_orig_h, img_orig_w, labels)
 regr = np.rollaxis(regr, 2, 0)
 dropmask = 0
else:
mask, regr = 0, 0
dropmask_name = self.root_dir_dropmasks.format(idx)
 if os.path.isfile(dropmask_name):
dropmask = imread(dropmask_name, True)
 dropmask = preprocess_image(dropmask)
 else:
dropmask = np.zeros(( img_h, img_w, 3))
img = torch.as_tensor(img, dtype=torch.float32)
 mask = torch.as_tensor(mask, dtype=torch.float32)
 regr = torch.as_tensor(regr, dtype=torch.float32)
 dropmask = torch.as_tensor(dropmask, dtype=torch.float32)
 return [img, mask, regr, dropmask]
class backup_CarDataset(Dataset):

 def __init__(self, dataframe, root_dir, training=True, transform=None, means = means, stds = stds):
self.df = dataframe
self.root_dir = root_dir
self.transform = transform
self.training = training
self.means = means
self.stds = stds
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
idx, labels = self.df.values[idx]
img_name = self.root_dir.format(idx)
 img = imread(img_name, True)
 img = preprocess_image(img)
 if self.transform:
img = self.transform(image=img)['image']
img = normalize(img, self.means, self.stds)
 img = np.rollaxis(img, 2, 0)
 if self.training:
mask, regr = get_mask_and_regr(labels)
 regr = np.rollaxis(regr, 2, 0)
 else:
mask, regr = 0, 0
img = torch.Tensor(img); mask = torch.Tensor(mask); regr = torch.Tensor(regr)
 return [img, mask, regr]<choose_model_class>
new_train.sort_values(['date_block_num','shop_id','item_id'], inplace = True)
 new_train.head()
class double_conv(nn.Module):

 def __init__(self, in_ch, out_ch):
super(double_conv, self ).__init__()
self.conv = nn.Sequential(
 nn.Conv2d(in_ch, out_ch, 3, padding=1),
nn.BatchNorm2d(out_ch),
nn.ReLU(inplace=True),
nn.Conv2d(out_ch, out_ch, 3, padding=1),
nn.BatchNorm2d(out_ch),
nn.ReLU(inplace=True)
)
 def forward(self, x):
x = self.conv(x)
 return x
class up(nn.Module):
def __init__(self, in_ch, out_ch, bilinear=True):
super(up, self ).__init__()
if bilinear:
self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)
 else:
self.up = nn.ConvTranspose2d(in_ch//2, in_ch//2, 2, stride=2)
 self.conv = double_conv(in_ch, out_ch)
 def forward(self, x1, x2=None):
x1 = self.up(x1)
 diffY = x2.size() [2] - x1.size() [2]
diffX = x2.size() [3] - x1.size() [3]
x1 = F.pad(x1,(diffX // 2, diffX - diffX//2,
diffY // 2, diffY - diffY//2))
if x2 is not None:
x = torch.cat([x2, x1], dim=1)
 else:
x = x1
x = self.conv(x)
 return x
def get_mesh(batch_size, shape_x, shape_y):
mg_x, mg_y = np.meshgrid(np.linspace(0, 1, shape_y), np.linspace(0, 1, shape_x))
mg_x = np.tile(mg_x[None, None, :, :], [batch_size, 1, 1, 1] ).astype('float32')
 mg_y = np.tile(mg_y[None, None, :, :], [batch_size, 1, 1, 1] ).astype('float32')
 mesh = torch.cat([torch.tensor(mg_x ).to(device), torch.tensor(mg_y ).to(device)], 1)
 return mesh<choose_model_class>
test.insert(loc=3, column='date_block_num', value=34 )
def set_dropout(model, drop_rate):
for name, child in model.named_children() :
if isinstance(child, torch.nn.Dropout):
child.p = drop_rate
print("name:", name)
 print("children:
", child)
 def effnet_dropout(drop_rate):
base_model0 = EfficientNet.from_pretrained(f"efficientnet-{effnet_ver}")
 set_dropout(base_model0, drop_rate)
 return base_model0
class MyUNet(nn.Module):

 def __init__(self, n_classes, drop_rate):
super(MyUNet, self ).__init__()
self.drop_rate = dropout_rate
self.base_model = effnet_dropout(drop_rate = self.drop_rate)
 self.conv0 = double_conv(5, 64)
 self.conv1 = double_conv(64, 128)
 self.conv2 = double_conv(128, 512)
 self.conv3 = double_conv(512, 1024)
 self.mp = nn.MaxPool2d(2)
 if effnet_ver == 'b0': self.up1 = up(1282 + 1024, 512)
 elif effnet_ver == 'b2': self.up1 = up(1410 + 1024, 512)
 elif effnet_ver == 'b4': self.up1 = up(1794 + 1024, 512)
 elif effnet_ver == 'b5': self.up1 = up(2050 + 1024, 512)
 self.up2 = up(512 + 512, 256)
 self.outc = nn.Conv2d(256, n_classes, 1)
 def forward(self, x):
batch_size = x.shape[0]
mesh1 = get_mesh(batch_size, x.shape[2], x.shape[3])
 x0 = torch.cat([x, mesh1], 1)
 x1 = self.mp(self.conv0(x0))
x2 = self.mp(self.conv1(x1))
x3 = self.mp(self.conv2(x2))
x4 = self.mp(self.conv3(x3))
feats = self.base_model.extract_features(x)
 mesh2 = get_mesh(batch_size, feats.shape[2], feats.shape[3])
 feats = torch.cat([feats, mesh2], 1)
 x = self.up1(feats, x4)
 x = self.up2(x, x3)
 x = self.outc(x)
 return x<categorify>
test['item_cnt_month'] = 0
def build_model(drop_rate):
print("device:", device)
 model = MyUNet(8, drop_rate ).to(device)
 return model<compute_train_metric>
new_train = new_train.append(test.drop('ID', axis = 1))
def criter_metric(prediction, mask, regr, size_average=True):
pred_mask = torch.sigmoid(prediction[:, 0])
 mask_loss = mask * torch.log(pred_mask + 1e-12)+(1 - mask)* torch.log(1 - pred_mask + 1e-12)
 mask_loss = -mask_loss.mean(0 ).sum()
pred_regr = prediction[:, 1:]
regr_loss =(torch.abs(pred_regr - regr ).sum(1)* mask ).sum(1 ).sum(1)/ mask.sum(1 ).sum(1)
 regr_loss = regr_loss.mean(0)
 loss = mask_loss + regr_loss
if not size_average:
loss *= prediction.shape[0]
return loss, mask_loss, regr_loss
w_mask = 0.1; w_regr = 0.9
w_mask = torch.tensor(w_mask, device=device); w_regr = torch.tensor(w_regr, device=device)
 gamma=torch.tensor(5, device=device)
 def criter_objective(prediction, mask, regr, size_average=True):
pred_mask = torch.sigmoid(prediction[:, 0])
 mask_loss = mask *(( 1-pred_mask)**gamma)* torch.log(pred_mask+1e-12)\
+(1 - mask)*torch.log(1-pred_mask+1e-12)
 mask_loss = -mask_loss.mean(0 ).sum()
pred_regr = prediction[:, 1:]
regr_loss =(torch.abs(pred_regr - regr ).sum(1)* mask ).sum(1 ).sum(1)/ mask.sum(1 ).sum(1)
 regr_loss = regr_loss.mean(0)
 loss = w_mask*mask_loss + w_regr*regr_loss
if not size_average:
loss *= prediction.shape[0]
return loss<train_model>
new_train = pd.merge(new_train, shops, on=['shop_id'], how='left')
 new_train.head()
def evaluate(model, epoch, history=None):
model.eval()
val_loss, mask_loss, regr_loss = torch.tensor(0., requires_grad=False), torch.tensor(0., requires_grad=False), torch.tensor(0., requires_grad=False)
 with torch.no_grad() :
for img_batch, mask_batch, regr_batch, _ in dev_loader:
img_batch = img_batch.to(device)
 mask_batch = mask_batch.to(device)
 regr_batch = regr_batch.to(device)
 output = model(img_batch)
 val_loss_batch, mask_loss_batch, regr_loss_batch = criter_metric(output, mask_batch, regr_batch, size_average=False)
 val_loss += val_loss_batch
mask_loss += mask_loss_batch
regr_loss += regr_loss_batch
val_loss /= len(dev_loader.dataset)
 mask_loss /= len(dev_loader.dataset)
 regr_loss /= len(dev_loader.dataset)
 return val_loss.cpu().numpy() , mask_loss.cpu().numpy() , regr_loss.cpu().numpy()
def save_checkpoint(state, model_name=None):

 if not model_name: model_name = f"model_date_{date_time_str}.pth"
torch.save(state, osj(out_path, model_name))
def train_eval_model(model, epoch, best_val_loss, history=None):
model.train()
freq_eval = len(train_loader.dataset)//4
for batch_idx,(img_batch, mask_batch, regr_batch, _)in enumerate(tqdm(train_loader)) :
img_batch = img_batch.to(device)
 mask_batch = mask_batch.to(device)
 regr_batch = regr_batch.to(device)
 output = model(img_batch)
 loss = criter_objective(output, mask_batch, regr_batch)
 train_loss = loss.data.cpu().numpy()
loss = loss / grad_accu_steps
loss.backward()
if(batch_idx+1)% grad_accu_steps == 0:
optimizer.step()
optimizer.zero_grad()
lr_value = optimizer.state_dict() ['param_groups'][0]['lr']
cond1 =(( batch_idx+1)% freq_eval)==0
cond2 = abs(batch_idx-len(train_loader)) *10 > len(train_loader)
 is_end_epoch =(( batch_idx+1)% len(train_loader)) == 0
if is_end_epoch |(cond1 & cond2):
t_eval0 = time.time()
val_loss, mask_val_loss, regr_val_loss = evaluate(model, dev_loader)
 print("
=========================")
 print(f"epoch={epoch}, batch={batch_idx} => lr={lr_value:.7f}, trn loss={loss:4f}")
 print(f"VAL LOSS: {val_loss:.2f}, \tBest Val Loss: {best_val_loss:.2f} \teval time={(time.time() -t_eval0)/60:.2f} min.")
 print("mask_val_loss:", mask_val_loss, "\tregr_val_loss", regr_val_loss)
 if is_end_epoch: print(f"
===END of EPOCH {epoch}================")
 history.loc[epoch*len(train_loader)+batch_idx, ['train_loss','val_loss', 'lr','mask_val_loss','regr_val_loss']] =\
[train_loss, val_loss, lr_value, mask_val_loss, regr_val_loss ]
history.to_csv(osj(out_path, f'history_{fld}.csv'))
is_best = val_loss < best_val_loss
best_val_loss = min(val_loss, best_val_loss)
 history.loc[epoch*len(train_loader)+batch_idx, ['train_loss', 'lr']] = [train_loss, lr_value ]
return model, optimizer, history, best_val_loss, val_loss<create_dataframe>
new_train = pd.merge(new_train, items.drop('item_name', axis = 1), on=['item_id'], how='left')
 new_train.head()
train_images_dir = osj(train_test_path,'train_images/{}.jpg')
 test_images_dir = osj(train_test_path,'test_images/{}.jpg')
 df_test = test
kfold = KFold(n_splits=num_splits, shuffle=True, random_state= base_seed)
 fld = fold_to_train
trn_idx, val_idx = list(kfold.split(train)) [fld]
train_dataset = CarDataset(train[['ImageId', 'PredictionString']].iloc[trn_idx], train_images_dir,
transform = albu_transform_train)
 dev_dataset = CarDataset(train[['ImageId', 'PredictionString']].iloc[val_idx], train_images_dir,
transform = albu_transform_valid)
 train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, num_workers=4)
 dev_loader = DataLoader(dataset=dev_dataset, batch_size=batch_size, shuffle=False, num_workers=4)
 model = build_model(drop_rate=dropout_rate)
 optimizer = optim.Adam(model.parameters() , lr=0.001)
 scheduler = lr_scheduler.MultiStepLR(optimizer, milestones=[7,10,14,17,20], gamma=0.5)
 if is_resume_train:
checkpoint = torch.load(prev_model_fn_path, map_location=device)
 model.load_state_dict(checkpoint['state_dict'])
 optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 epoch_start = checkpoint['epoch']
n_epochs = num_epochs_resume
best_val_loss = prev_epoch_val_loss = checkpoint['best_val_loss']
prev_epoch_val_loss = copy.copy(best_val_loss)
 scheduler.load_state_dict(checkpoint['scheduler'])
 del checkpoint; _ = gc.collect()
else:
epoch_start = 0
n_epochs = num_epochs
best_val_loss, prev_epoch_val_loss = 10e+6, 10e+6
history = pd.DataFrame(np.full(( len(train_loader)*n_epochs, 5), fill_value=-1, dtype=np.float32),
columns=['train_loss','val_loss','lr','mask_val_loss','regr_val_loss'])
 print(f"
======================
Train start.Output directory {out_path}")
 print("efficientnet-", effnet_ver, "n_epochs:", n_epochs, "len(train_loader):", len(train_loader))
for epoch in range(epoch_start, n_epochs+epoch_start):
torch.cuda.empty_cache()
gc.collect()
model, optimizer, history, best_val_loss, val_loss = train_eval_model(model, epoch, best_val_loss, history)
 scheduler.step()
is_last_epoch =(epoch==(n_epochs+epoch_start-1))
if(val_loss < prev_epoch_val_loss)|(is_last_epoch):
params_save_model = { 'epoch': start_epoch + epoch, 'loss': history['train_loss'].iat[-1],
'best_val_loss': best_val_loss,
'state_dict': model.state_dict() ,
'optimizer_state_dict': optimizer.state_dict() ,
'scheduler': scheduler.state_dict() }
print(f"Val_loss improved or end training => Saving at epoch {epoch} end w val_loss={val_loss:.2f}, best_val_loss = {best_val_loss:.2f}")
 save_checkpoint(params_save_model, model_name = f"model_fld_{fld}_islastepoch_{is_last_epoch:d}_date_{date_time_str}.pth")
 prev_epoch_val_loss = val_loss
history.to_csv(osj(out_path, f"history_final_fold_{fld}.csv"), index=False )<load_from_csv>
new_train = pd.merge(new_train, categories.drop('item_category_name', axis = 1), on=['item_category_id'], how='left')
 new_train.head()
history_old = pd.read_csv('.. /input/dropout-change-eff-b0-2-01-no-aug-epochs-6/history_final_fold_3.csv')
 history_old = history_old[history_old['val_loss'] > 0]
history_new = history[history['val_loss']>0]
history_new.index = history_new.index + history_old.index.max() +1
history_new = pd.concat([history_old, history_new], axis=0)
 history_new.tail()<categorify>
def generate_lag(train, months, lag_column):
for month in months:
train_shift = train[['date_block_num', 'shop_id', 'item_id', lag_column]].copy()
train_shift.columns = ['date_block_num', 'shop_id', 'item_id', lag_column+'_lag_'+ str(month)]
train_shift['date_block_num'] += month
train = pd.merge(train, train_shift, on=['date_block_num', 'shop_id', 'item_id'], how='left')
 return train
torch.cuda.empty_cache()
gc.collect()
threshold0 = -1.6
n_imgs = 6 if not debug else 2
for idx in random.sample(range(len(dev_dataset)) , n_imgs):
img, mask, regr, _ = dev_dataset[idx]
output = model(img[None].to(device)).data.cpu().numpy()
img = img.data.cpu().numpy()
mask = mask.data.cpu().numpy()
regr = regr.data.cpu().numpy()
coords_pred = extract_coords(output[0], threshold=threshold0)
 coords_true = extract_coords(np.concatenate([mask[None], regr], 0))
img = denormalize(img, means, stds, resize_to_original=True)
 fig, axes = plt.subplots(1, 2, figsize=(16,12))
axes[0].set_title('Ground truth')
 axes[0].imshow(visualize(img, coords_true)[(img.shape[0]//2):])
 axes[1].set_title('Prediction')
 axes[1].imshow(visualize(img, coords_pred)[(img.shape[0]//2):])
 plt.show()<load_pretrained>
del items
del categories
del shops
del test
is_last_epoch = False
model_name = f"model_fld_{fld}_islastepoch_{is_last_epoch:d}_date_{date_time_str}.pth"
try:
checkpoint = torch.load(osj(out_path, model_name), map_location=device)
 print("best model of the current session loaded.")
 except:
checkpoint = torch.load(prev_model_fn_path, map_location=device)
 print("NO best model in the current session.Previously trained model loaded")
 model.load_state_dict(checkpoint['state_dict'] )<predict_on_test>
new_train = downcast_dtypes(new_train )
def predict_test(model, threshold = 0):
preds, preds_before_drop_mask = [], []
test_loader = DataLoader(dataset=test_dataset, batch_size=4, shuffle=False, num_workers=4)
 model.eval()
for img_batch, _, _, test_mask_batch in tqdm(test_loader):
with torch.no_grad() :
output = model(img_batch.to(device)).data.cpu().numpy()
test_mask_batch = test_mask_batch.data.cpu().numpy()
for out, test_mask in zip(output, test_mask_batch):
coords = extract_coords(out, threshold= threshold)
 s = coords2str(coords)
 preds_before_drop_mask.append(s)
 if isinstance(test_mask, np.ndarray):
test_mask = cv2.resize(test_mask,(256,64)) [...,0]
test_mask = np.where(test_mask>255//2, 100, 0)
 out[0,...] = out[0,...] - test_mask
coords = extract_coords(out, threshold= threshold)
 s = coords2str(coords)
 preds.append(s)
 return preds, preds_before_drop_mask 
<load_from_csv>
gc.collect()
threshold0 = -1.5
test_thresh, test_thresh_before = {}, {}
test_batch_size = 4
test_images_dir = osj(train_test_path,'test_images/{}.jpg')
 test_masks_dir = osj(path,'test_masks/{}.jpg')
 test_dataset = CarDataset(test, test_images_dir, test_masks_dir, training=False)
 test_loader = DataLoader(dataset=test_dataset, batch_size=test_batch_size, shuffle=False, num_workers=4)
 for thresh in tqdm([threshold0]):
preds_test = pd.read_csv(osj(path,'sample_submission.csv'), nrows = n_samples)
 preds_test_before = preds_test.copy()
temp_preds, temp_preds_before_drop_mask = predict_test(model, thresh)
 preds_test['PredictionString'] = temp_preds[:n_samples]
preds_test_before['PredictionString'] = temp_preds_before_drop_mask[:n_samples]
test_thresh['thresh_'+str(thresh)] = preds_test
test_thresh_before['thresh_'+str(thresh)] = preds_test_before 
<create_dataframe>
%%time
new_train = generate_lag(new_train, [1,2,3,4,5,6,12], 'item_cnt_month' )
sub = test_thresh['thresh_'+str(thresh)]
sub_before = test_thresh_before['thresh_'+str(thresh)]
numcars = pd.DataFrame({'ImageId': sub['ImageId'].values, 'numcars': [int(( str(x ).count(' ')+1)/7)for x in sub['PredictionString']]})
 numcars_before = pd.DataFrame({'ImageId': sub_before['ImageId'].values, 'numcars': [int(( str(x ).count(' ')+1)/7)for x in sub_before['PredictionString']]})
 numcars_compare = pd.concat([numcars[numcars_before['numcars']!=numcars['numcars']],
numcars_before[numcars_before['numcars']!=numcars['numcars']]], axis=1)
 numcars_compare.columns = ['ImageId_new','numcars_new', 'ImageId_old','numcars_old']
numcars_compare = numcars_compare[['ImageId_old','ImageId_new','numcars_old','numcars_new']]
print("
Number of images affected by test_masks reduction", numcars_compare.shape[0])
 print()
numcars_compare.head() 
<save_to_csv>
%%time
group = new_train.groupby(['date_block_num', 'item_id'])['item_cnt_month'].mean().rename('item_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'item_id'], how='left')
 new_train = generate_lag(new_train, [1,2,3,6,12], 'item_month_mean')
 new_train.drop(['item_month_mean'], axis=1, inplace=True )
sub.to_csv(osj(out_path, f'sub_thresh_{threshold0:.2f}_cv_3045_eff_b0_dropout_change_resume_KGL.csv'), index=False)
 sub.head()<load_from_csv>
%%time
group = new_train.groupby(['date_block_num', 'shop_id'])['item_cnt_month'].mean().rename('shop_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'shop_id'], how='left')
 new_train = generate_lag(new_train, [1,2,3,6,12], 'shop_month_mean')
 new_train.drop(['shop_month_mean'], axis=1, inplace=True )
test=pd.read_csv('.. /input/digit-recognizer/test.csv')
 train=pd.read_csv('.. /input/digit-recognizer/train.csv')
 X_train =(train.iloc[:,1:].values ).astype('float32')
 y_train = train.iloc[:,0].values.astype('int32')
 y_train= to_categorical(y_train)
 X_test = test.values.astype('float32')
 X_train = X_train.reshape(X_train.shape[0], 28, 28,1)
 X_test = X_test.reshape(X_test.shape[0], 28, 28,1)
 mean_px = X_train.mean().astype(np.float32)
 std_px = X_train.std().astype(np.float32)
 def standardize(x):
return(x-mean_px)/std_px
model = Sequential([
Lambda(standardize, input_shape=(28,28,1)) ,
Convolution2D(32,(3,3), activation='relu'),
BatchNormalization() ,
Convolution2D(32,(3,3), activation='relu'),
BatchNormalization() ,
Convolution2D(32,(5,5), activation='relu',padding='SAME'),
BatchNormalization() ,
Dropout(0.4),
Convolution2D(64,(3,3), activation='relu'),
BatchNormalization() ,
Convolution2D(64,(3,3), activation='relu'),
BatchNormalization() ,
Convolution2D(64,(5,5), activation='relu',padding='SAME'),
BatchNormalization() ,
Dropout(0.4),
Flatten() ,
BatchNormalization() ,
Dense(1024, activation='relu'),
BatchNormalization() ,
Dense(512, activation='relu'),
BatchNormalization() ,
Dense(10, activation='softmax')
 ])
 model.compile(Adam(lr=1e-4), loss='categorical_crossentropy', metrics=['accuracy'])
 gen = image.ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.10,
width_shift_range=0.1,
height_shift_range=0.1)
 X = X_train
y = y_train
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.10, random_state=42)
 batches = gen.flow(X_train, y_train, batch_size=256)
 val_batches=gen.flow(X_val, y_val, batch_size=256)
 history=model.fit_generator(generator=batches, steps_per_epoch=len(batches), epochs=5,
validation_data=val_batches, validation_steps=len(val_batches))
predictions = model.predict_classes(X_test, verbose=0)
 submissions=pd.DataFrame({"ImageId": list(range(1,len(predictions)+1)) ,
"Label": predictions})
 submissions.to_csv("submission.csv", index=False, header=True )<choose_model_class>
%%time
group = new_train.groupby(['date_block_num', 'shop_id', 'item_category_id'])['item_cnt_month'].mean().rename('shop_category_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 new_train = generate_lag(new_train, [1, 2], 'shop_category_month_mean')
 new_train.drop(['shop_category_month_mean'], axis=1, inplace=True )
<set_options>
%%time
group = new_train.groupby(['date_block_num', 'main_category_id'])['item_cnt_month'].mean().rename('main_category_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'main_category_id'], how='left')
 new_train = generate_lag(new_train, [1], 'main_category_month_mean')
 new_train.drop(['main_category_month_mean'], axis=1, inplace=True )
!nvidia-smi<define_variables>
%%time
group = new_train.groupby(['date_block_num', 'sub_category_id'])['item_cnt_month'].mean().rename('sub_category_month_mean' ).reset_index()
new_train = pd.merge(new_train, group, on=['date_block_num', 'sub_category_id'], how='left')
 new_train = generate_lag(new_train, [1], 'sub_category_month_mean')
 new_train.drop(['sub_category_month_mean'], axis=1, inplace=True )
USE_COLAB, TRAIN_MODE, USE_TPU = 0, 1, 0
version_name = 'v16'
network_name = 'efficientnet_b4'<load_from_url>
new_train['month'] = new_train['date_block_num'] % 12
if USE_TPU:
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev 
<define_variables>
holiday_dict = {
0: 6,
1: 3,
2: 2,
3: 8,
4: 3,
5: 3,
6: 2,
7: 8,
8: 4,
9: 8,
10: 5,
11: 4,
}
if not USE_COLAB:
print(os.listdir('.. /input/'))
PATH = '.. /input/plant-pathology-image-processing/'
SAVE_PATH = './'
MODEL_PATH = '.. /input/plant-pathology-pytorch-gpu-model/'
!git clone -q https://github.com/welkin-feng/ComputerVision.git
sys.path.append('./ComputerVision/')
 !git clone -q https://github.com/rwightman/pytorch-image-models.git
sys.path.append('./pytorch-image-models/')
 else:
drive.mount('/content/drive', force_remount=True)
 PATH = './drive/My Drive/Competition/plant-pathology-2020/plant-pathology-2020/'
SAVE_PATH = f"./drive/My Drive/Competition/plant-pathology-2020/{version_name}_{network_name}/"
MODEL_PATH = SAVE_PATH
if not os.path.isdir(SAVE_PATH):
os.mkdir(SAVE_PATH)
 !git clone -q https://github.com/welkin-feng/ComputerVision.git
sys.path.append('./ComputerVision/')
 !git clone -q https://github.com/rwightman/pytorch-image-models.git
sys.path.append('./pytorch-image-models/')
 print("PATH: ", os.listdir(PATH))
print("SAVE_PATH: ", os.listdir(SAVE_PATH))
if os.path.isdir(MODEL_PATH):
print('MODEL_PATH: ', os.listdir(MODEL_PATH))
DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 print(DEVICE)
 random_seed = 644
print(f'random_state: {random_seed}' )<set_options>
new_train['holidays_in_month'] = new_train['month'].map(holiday_dict )
def seed_everything(seed):

 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
if USE_TPU:
seed_everything(random_seed )<define_variables>
moex = {
12: 659, 13: 640, 14: 1231,
15: 881, 16: 764, 17: 663,
18: 743, 19: 627, 20: 692,
21: 736, 22: 680, 23: 1092,
24: 657, 25: 863, 26: 720,
27: 819, 28: 574, 29: 568,
30: 633, 31: 658, 32: 611,
33: 770, 34: 723,
}
network_name = network_name
version_name = version_name
DEVICE = DEVICE
PATH = PATH
SAVE_PATH = SAVE_PATH
MODEL_PATH = MODEL_PATH
IMAGE_FILE_NAME = ['train_images_320x512.npy', 'test_images_320x512.npy']
IMG_SHAPE =(1365, 2048, 3)
 INPUT_IMG_SHAPE =(320, 512, 3)
 RGB_MEAN = np.array([0.40379888, 0.5128721, 0.31294255])
 RGB_STD = np.array([0.20503741, 0.18957737, 0.1883159])
 n_fold = 5
fold =(0, 1, 2, 3, 4)
 BATCH_SIZE = 16 if not USE_TPU else 64
TEST_BATCH_SIZE = 1
accumulation_steps = 1
loss_weights =(1, 1)
 learning_rate = 1e-3
lr_ratio = np.sqrt(0.1)
 reduce_lr_metric = ['loss', 'score', 'both'][0]
patience = 5
warm_up_steps = 1
warm_up_lr_factor = 0.1
num_classes = 4
n_epochs = 50
train_epochs = 50
resume = False
pretrained = not resume<load_from_csv>
new_train['moex_value'] = new_train.date_block_num.map(moex )
df_train = pd.read_csv(PATH + 'train.csv')
 df_train['class'] = np.argmax(df_train.iloc[:, 1:].values, axis=1)
 skf = StratifiedKFold(n_fold, shuffle = True, random_state = 644)
 for i_fold,(train_idx, val_idx)in enumerate(skf.split(df_train, df_train['class'].values)) :
df_train.loc[val_idx, 'fold'] = i_fold
df_train['fold'] = df_train['fold'].astype(int)
 df_test = pd.read_csv(PATH + 'test.csv')
 submission = pd.read_csv(PATH + 'sample_submission.csv' )<count_values>
new_train = downcast_dtypes(new_train )
for i_fold in range(5):
train_idx, valid_idx = np.where(( df_train['fold'] != i_fold)) [0], np.where(( df_train['fold'] == i_fold)) [0]
train_class = df_train['class'].iloc[train_idx].values.tolist()
val_class = df_train['class'].iloc[valid_idx].values.tolist()
train_class_count = {i:train_class.count(i)for i in range(4)}
val_class_count = {i:val_class.count(i)for i in range(4)}
print(f"i_fold {i_fold} train_class count {train_class_count} val_class_count {val_class_count}" )<save_to_csv>
import xgboost as xgb
class PlantPathologyDataset(Dataset):
def __init__(self, csv, idx, mode, transform = None, data = None):
self.csv = csv.reset_index(drop = True)
 self.data = data
self.filepath_format = PATH + 'images/{}.jpg'
self.idx = np.asarray(idx ).astype('int')
 self.mode = mode
self.transform = transform
def __len__(self):
return self.idx.shape[0]
def __getitem__(self, index):
index = self.idx[index]
if self.data is not None:
image = self.data[index]
else:
img_name = self.csv['image_id'].iloc[index]
image = cv2.imread(self.filepath_format.format(img_name))
if image.shape != IMG_SHAPE:
image = cv2.rotate(image, rotateCode = cv2.ROTATE_90_CLOCKWISE)
 if image.shape != INPUT_IMG_SHAPE:
image = cv2.resize(image, INPUT_IMG_SHAPE[1::-1], interpolation = cv2.INTER_AREA)
 image = image.astype('uint8')
 image_origin = image.copy().astype('float32')
 image = self.transform(image ).astype('float32')if self.transform is not None else image.astype('float32')
 image, image_origin = np.rollaxis(image, 2, 0)/ 255, np.rollaxis(image_origin, 2, 0)/ 255
if self.mode == 'test':
return torch.tensor(image)
 else:
label = self.csv.iloc[index, 1:5].values.astype('float32')
 return torch.tensor(image), torch.tensor(image_origin), torch.tensor(label)
 def get_train_val_dataloader(i_fold, transforms_train, transforms_val):
train_idx, valid_idx = np.where(( df_train['fold'] != i_fold)) [0], np.where(( df_train['fold'] == i_fold)) [0]
train_data = np.load(PATH + IMAGE_FILE_NAME[0])if os.path.isfile(PATH + IMAGE_FILE_NAME[0])else None
dataset_train = PlantPathologyDataset(df_train, train_idx, 'train', transform=transforms_train, data = train_data)
 dataset_valid = PlantPathologyDataset(df_train, valid_idx, 'val', transform=transforms_val, data = train_data)
 batch_size, train_sampler, drop_last = BATCH_SIZE, RandomSampler(dataset_train), False
if USE_TPU and xm.xrt_world_size() > 1:
batch_size, drop_last = BATCH_SIZE // xm.xrt_world_size() , True
train_sampler = torch.utils.data.distributed.DistributedSampler(
 dataset_train, num_replicas=xm.xrt_world_size() , rank=xm.get_ordinal() , shuffle=True 
)
 train_loader = DataLoader(dataset_train, batch_size, sampler = train_sampler, drop_last = drop_last, num_workers=4)
 val_loader = DataLoader(dataset_valid, TEST_BATCH_SIZE, num_workers=4, pin_memory = True)
 return train_loader, val_loader
def get_test_dataloader() :
test_data = np.load(PATH + IMAGE_FILE_NAME[1])if os.path.isfile(PATH + IMAGE_FILE_NAME[1])else None
dataset_test = PlantPathologyDataset(df_test, np.arange(len(df_test)) , 'test', data = test_data)
 test_loader = DataLoader(dataset_test, TEST_BATCH_SIZE, sampler=None, num_workers=4)
 return test_loader<prepare_x_and_y>
new_train = new_train[new_train.date_block_num > 11]
def rand_bbox(img_shape, lam):
H, W = img_shape
cut_rat = np.sqrt(1.- lam)
 cut_w, cut_h = int(W * cut_rat), int(H * cut_rat)
 cx, cy = np.random.randint(cut_w // 4, W - cut_w // 4), np.random.randint(cut_h // 4, H - cut_h // 4)
 bbx1 = np.clip(cx - cut_w // 2, 0, W)
 bby1 = np.clip(cy - cut_h // 2, 0, H)
 bbx2 = np.clip(cx + cut_w // 2, 0, W)
 bby2 = np.clip(cy + cut_h // 2, 0, H)
 return bbx1, bby1, bbx2, bby2
class MixupBatch(object):
def __init__(self, mixup_prob, mixup_alpha, **kwargs):
self.prob = mixup_prob
self.alpha = mixup_alpha
def set_prob(self, epoch, max_epoch):
self.prob = min(1., epoch / max_epoch)
 def __call__(self, ori_img_batch, img_batch, label_batch):

 label_batch_mix, lam = None, 1
if self.alpha <= 0 or np.random.rand() > self.prob:
return img_batch, label_batch, label_batch_mix, lam
batch_size = ori_img_batch.shape[0]
lam = np.random.beta(self.alpha, self.alpha, batch_size)
 lam = np.maximum(1 - lam, lam)
 lam = torch.from_numpy(lam ).view(batch_size, 1, 1, 1 ).to(ori_img_batch)
 shuffled_idx = torch.randperm(batch_size)
 ori_img_batch = lam * ori_img_batch +(1 - lam)* ori_img_batch[shuffled_idx]
label_batch_mix = [label[shuffled_idx] for label in label_batch]
return ori_img_batch, label_batch, label_batch_mix, lam
class CutmixBatch(object):
def __init__(self, cutmix_prob, cutmix_alpha, **kwargs):
self.prob = cutmix_prob
self.alpha = cutmix_alpha
def set_prob(self, epoch, max_epoch):
self.prob = min(1., epoch / max_epoch)
 def __call__(self, ori_img_batch, img_batch, label_batch):

 label_batch_mix, lam = None, 1
if self.alpha <= 0 or np.random.rand() > self.prob:
return img_batch, label_batch, label_batch_mix, lam
batch_size = ori_img_batch.shape[0]
lam = np.random.beta(self.alpha, self.alpha, batch_size)
 lam = np.maximum(1 - lam, lam)
 lam = torch.from_numpy(lam ).view(batch_size, 1, 1, 1 ).to(ori_img_batch)
 shuffled_idx = torch.randperm(batch_size)
 y = ori_img_batch[shuffled_idx].clone().detach()
label_batch_mix = [label[shuffled_idx] for label in label_batch]
for i in range(batch_size):
bbx1, bby1, bbx2, bby2 = rand_bbox(ori_img_batch.shape[-2:], lam[i])
 ori_img_batch[i, :, bby1:bby2, bbx1:bbx2] = y[i, :, bby1:bby2, bbx1:bbx2]
lam[i] = 1 -(bbx2 - bbx1)*(bby2 - bby1)/ np.prod(ori_img_batch.shape[-2:])
 return ori_img_batch, label_batch, label_batch_mix, lam
class FMixBatch(object):
def __init__(self, fmix_prob, fmix_alpha=1, decay_power=3, max_soft=0.0, reformulate=False):
self.prob = fmix_prob
self.alpha = fmix_alpha
self.decay_power = decay_power
self.max_soft = max_soft
self.reformulate = reformulate
def set_prob(self, epoch, max_epoch):
self.prob = min(1., epoch / max_epoch)
 def __call__(self, ori_img_batch, img_batch, label_batch):

 label_batch_mix, lam = None, 1
if np.random.rand() > self.prob:
return img_batch, label_batch, label_batch_mix, lam
size = ori_img_batch.shape[-2:]
lam, mask = sample_mask(self.alpha, self.decay_power, size, self.max_soft, self.reformulate)
 mask = torch.from_numpy(mask ).to(ori_img_batch)
 shuffled_idx = torch.randperm(ori_img_batch.size(0))
ori_img_batch = mask * ori_img_batch +(1 - mask)* ori_img_batch[shuffled_idx]
label_batch_mix = [label[shuffled_idx] for label in label_batch]
return ori_img_batch, label_batch, label_batch_mix, lam
class MixBatch(object):
def __init__(self, transforms_dict = None, img_mean = np.zeros(1), img_std = np.ones(1), mixup_alpha = 0.4, cutmix_alpha = 1.0, **kwargs):
transforms_dict = transforms_dict or {}
self.mix_prob = transforms_dict.get('mix_prob', 0)
 probs = [transforms_dict.get('mixup_prob', 0), transforms_dict.get('cutmix_prob', 0),
transforms_dict.get('fmix_prob', 0), transforms_dict.get('grid_prob', 0)]
self.probs = np.cumsum(probs)
 cutout_ratio = transforms_dict.get('cutout_ratio',(0.05, 0.25))
self.mixup = MixupBatch(mixup_prob = 1, mixup_alpha = mixup_alpha)
 self.cutmix = CutmixBatch(cutmix_prob = 1, cutmix_alpha = cutmix_alpha)
 self.fmix = FMixBatch(fmix_prob = 1, fmix_alpha=1, decay_power=3, max_soft=0.0, reformulate=False)
 self.gridmask = [GridMaskBatch(num_grid =(3, 6), rotate = 15, mode = 0, prob = 1.) ,
GridMaskBatch(num_grid =(3, 6), rotate = 15, mode = 1, prob = 1.) ,
GridMaskBatch(num_grid =(3, 6), rotate = 15, mode = 2, prob = 1.) ,]
def set_prob(self, epoch, max_epoch = 15):
self.mixup.set_prob(epoch, max_epoch)
 self.cutmix.set_prob(epoch, max_epoch)
 self.fmix.set_prob(epoch, max_epoch)
 for gridmask in self.gridmask:
gridmask.set_prob(epoch, max_epoch)
 def __call__(self, ori_img_batch, img_batch, label_batch):
label_batch_mix, lam = None, 1
if np.random.rand() < self.mix_prob:
r = np.random.rand()
if r < self.probs[0]:
return self.mixup(ori_img_batch, img_batch, label_batch)
 elif r < self.probs[1]:
return self.cutmix(ori_img_batch, img_batch, label_batch)
 elif r < self.probs[2]:
return self.fmix(ori_img_batch, img_batch, label_batch)
 elif r < self.probs[3]:
img_batch = np.random.choice(self.gridmask )(ori_img_batch, img_batch)
 return img_batch, label_batch, label_batch_mix, lam<set_options>
gc.collect()
transforms_train, transforms_val = None, None
if TRAIN_MODE:
!pip install --upgrade -q albumentations
!pip install --upgrade -q imgaug
transforms_train = transforms.Compose([
lambda image: iaa.Sequential([
iaa.Fliplr(0.5),
iaa.Flipud(0.5),
] )(image=image),
lambda image: albumentations.OneOf([
albumentations.ShiftScaleRotate(shift_limit=0.2, scale_limit=0.2, rotate_limit=30, p=1),
] )(image=image)['image'],
lambda image: iaa.Sequential([
iaa.Sometimes(0.7, iaa.Cutout(nb_iterations=1, size=(0.6, 0.7), fill_mode="constant", cval=0))
] )(image=image),
])
 mix = None<init_hyperparams>
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
df[col].fillna(0, inplace=True)
 return df
new_train = fill_na(new_train )
class RGBNorm(nn.Module):
IMAGENET_MEAN = torch.tensor(( 0.485, 0.456, 0.406)).view(1, 3, 1, 1)
 IMAGENET_STD = torch.tensor(( 0.229, 0.224, 0.225)).view(1, 3, 1, 1)
 def __init__(self, **kwargs):
super().__init__()
self.register_buffer('rgb_mean', torch.tensor(RGB_MEAN ).view(1,3,1,1))
self.register_buffer('rgb_std', torch.tensor(RGB_STD ).view(1,3,1,1))
def forward(self, x):
_, C, _, _ = x.shape
if C == 1:
x = x.repeat(( 1, 3, 1, 1))
x =(x - self.rgb_mean.to(x)) / self.rgb_std.to(x)
 return x
class Mish(nn.Module):
def __init__(self, *args, **kwagrs):
super(Mish, self ).__init__()
def forward(self,x):
return x.mul(F.softplus(x ).tanh())
 def gem(x, kernel_size, stride = None, p = 3, eps = 1e-6):
return F.avg_pool2d(x.clamp(min=eps ).pow(p), kernel_size, stride ).pow(1./p)
 class ArcMarginProduct(nn.Module):
def __init__(self, in_features, out_features):
super().__init__()
self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))
self.reset_parameters()
def reset_parameters(self):
stdv = 1./ math.sqrt(self.weight.size(1))
self.weight.data.uniform_(-stdv, stdv)
 def forward(self, features):
cosine = F.linear(F.normalize(features), F.normalize(self.weight))
return cosine
class Classifier(nn.Module):

 def __init__(self, backbone, num_classes, **kwargs):
super().__init__()
self.norm = RGBNorm()
self.backbone = backbone
self.gfc = nn.Sequential(*[nn.AdaptiveAvgPool2d(1),
nn.Flatten() ,
nn.Linear(backbone.out_channel, 2048),
Mish() ])
 self.dropouts = nn.ModuleList([nn.Dropout(0.5)for _ in range(5)])
 self.metric_classify = ArcMarginProduct(2048, num_classes)
 self.cls_head = nn.Linear(2048, num_classes)
 def forward(self, x):
x = self.norm(x)
 x = self.backbone(x)
 x = self.gfc(x)
 for i, dropout in enumerate(self.dropouts):
if i == 0:
out = self.cls_head(dropout(x))
else:
out += self.cls_head(dropout(x))
out /= len(self.dropouts)
 metric_output = self.metric_classify(x)
 return out, metric_output
def load_pretrained_model(model, model_path = '', url = '', skip =() , conversion =()):
if os.path.isfile(model_path):
state_dict = torch.load(model_path, map_location = 'cpu')
 if not USE_TPU or xm.is_master_ordinal() :
print('=> loading pretrained model {}'.format(model_path))
elif url != '':
state_dict = model_zoo.load_url(url, progress = False, map_location = 'cpu')
 if not USE_TPU or xm.is_master_ordinal() :
print('=> loading pretrained model {}'.format(url))
else:
return
conversion = np.array(conversion ).reshape(-1, 2)if len(conversion)else []
model_dict = model.state_dict()
pretrained_state_dict = {}
for ks in state_dict.keys() :
if ks in model_dict.keys() and all(s not in ks for s in skip):
km = ks
for _km, _ks in conversion:
if ks == _ks:
km = _km
break
pretrained_state_dict[km] = state_dict[ks]
if not USE_TPU or xm.is_master_ordinal() :
print(f"=> loading pretrained model weight length {len(pretrained_state_dict)} / total_state_dict {len(state_dict)} / total_model_dict {len(model_dict)}")
 model_dict.update(pretrained_state_dict)
 model.load_state_dict(model_dict, strict = False )<choose_model_class>
def xgtrain() :
regressor = xgb.XGBRegressor(n_estimators = 5000,
learning_rate = 0.01,
max_depth = 10,
subsample = 0.5,
colsample_bytree = 0.5)
 regressor_ = regressor.fit(new_train[new_train.date_block_num < 33].drop(['item_cnt_month'], axis=1 ).values,
new_train[new_train.date_block_num < 33]['item_cnt_month'].values,
eval_metric = 'rmse',
eval_set = [(new_train[new_train.date_block_num < 33].drop(['item_cnt_month'], axis=1 ).values,
new_train[new_train.date_block_num < 33]['item_cnt_month'].values), 
(new_train[new_train.date_block_num == 33].drop(['item_cnt_month'], axis=1 ).values,
new_train[new_train.date_block_num == 33]['item_cnt_month'].values)
 ],
verbose=True,
early_stopping_rounds = 50, 
)
 return regressor_
def efficientnet_b4(pretrained = False, **kwargs):
if not USE_TPU or xm.is_master_ordinal() :
print('=> create `efficientnet_b4`')
 backbone = EfficientNet.from_name('efficientnet-b4')
 backbone._fc = None
backbone.out_channel = backbone.out_channels
backbone.forward = backbone.forward_features
if pretrained:
pretrained_file_name = ''
url = 'https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b4-6ed6700e.pth'
load_pretrained_model(backbone, model_path = MODEL_PATH + pretrained_file_name, url = url,
skip =('.num_batches_tracked', '_fc.',))
model = Classifier(backbone, num_classes)
 return model<choose_model_class>
%%time
regressor_ = xgtrain()
def seresnext50_32x4d(pretrained = False, **kwargs):
if not USE_TPU or xm.is_master_ordinal() :
print('=> create `seresnext50_32x4d`')
 backbone_kwargs = dict(cardinality = 32, base_width = 4, act_layer = Mish, block_args = dict(attn_layer = 'se'))
backbone = ResNet(Bottleneck, [3, 4, 6, 3], in_chans = 3, **backbone_kwargs)
 backbone.fc = None
backbone.out_channel, backbone.forward = backbone.num_features, backbone.forward_features
if pretrained:
pretrained_file_name = 'gluon_seresnext50_32x4d-90cf2d6e.pth'
url = 'https://github.com/rwightman/pytorch-pretrained-gluonresnet/releases/download/v0.1/gluon_seresnext50_32x4d-90cf2d6e.pth'
load_pretrained_model(backbone, model_path = MODEL_PATH + pretrained_file_name, url = url,
skip =('.num_batches_tracked',))
model = Classifier(backbone, num_classes)
 return model<compute_train_metric>
predictions = regressor_.predict(new_train[new_train.date_block_num == 34].drop(['item_cnt_month'], axis = 1 ).values )
def cross_entropy(preds, trues, class_weights = 1.0, reduction = 'mean', **kwargs):
class_weights = torch.tensor(class_weights ).to(preds)
 ce_loss = -torch.sum(class_weights * trues * F.log_softmax(preds, dim = 1), dim = 1)
 if reduction == 'mean':
return ce_loss.mean()
elif reduction == 'sum':
return ce_loss.sum()
elif reduction == 'none':
return ce_loss
def focol_loss(preds, trues, gamma = 2, class_weights = 1, reduction = 'mean', **kwargs):
ce_loss = cross_entropy(preds, trues, class_weights=class_weights, reduction = 'none')
 alpha = torch.sum(trues *(1 - F.softmax(preds, dim = 1)) ** gamma, dim = 1)
 pos_loss = alpha * ce_loss
if reduction == 'mean':
return pos_loss.mean()
elif reduction == 'sum':
return pos_loss.sum()
elif reduction == 'none':
return pos_loss
def ohem_loss(preds, trues, ohem_rate = 1, class_weights = 1, reduction = 'mean', **kwargs):
ohem_rate = max(0, min(1, ohem_rate))
ce_loss = cross_entropy(preds, trues, class_weights, reduction = 'none')
 loss = torch.topk(ce_loss, int(ohem_rate * ce_loss.size(0)))[0] if 0 <= ohem_rate < 1 else ce_loss
if reduction == 'mean':
return torch.mean(loss)
 elif reduction == 'sum':
return torch.sum(loss)
 elif reduction == 'none':
return loss
class ArcFaceLoss(nn.Module):
def __init__(self, s=30.0, m=0.5, reduction='mean'):
super().__init__()
self.reduction = reduction
self.s = s
self.cos_m = math.cos(m)
 self.sin_m = math.sin(m)
 self.th = math.cos(math.pi - m)
 self.mm = math.sin(math.pi - m)* m
def forward(self, logits, labels):
logits = logits.float()
cosine = logits
sine = torch.sqrt(1.0 - torch.pow(cosine, 2))
phi = cosine * self.cos_m - sine * self.sin_m
phi = torch.where(cosine > self.th, phi, cosine - self.mm)
 output =(labels * phi)+(( 1.0 - labels)* cosine)
 output *= self.s
loss = cross_entropy(output, labels, reduction = self.reduction)
 return loss / 2
def criterion(logits, metric_logits, trues, is_val = False, lam = 1, **kwargs):
assert len(trues)== len(logits)== len(metric_logits)
 weights = loss_weights
class_weights =(1, 4, 1, 1)
 if not is_val:
loss_0 = cross_entropy(logits, trues.float() , class_weights=class_weights, reduction='mean')* lam
loss_metric = ArcFaceLoss(reduction='mean' )(metric_logits, trues.float())* lam
loss =(loss_0 * weights[0] + loss_metric * weights[1])/ sum(weights)
 else:
loss_0 = cross_entropy(logits, trues.float() , class_weights=class_weights, reduction='sum')* lam
loss_metric = ArcFaceLoss(reduction='sum' )(metric_logits, trues.float())* lam
loss =(loss_0 * weights[0] + loss_metric * weights[1])/ sum(weights)
 return loss,(loss_0.detach() , loss_metric.detach() )<compute_test_metric>
submission['item_cnt_month'] = predictions
def get_score(submission, solution):
roc_score = roc_auc_score(solution, submission)
 predictions = np.argmax(submission, axis = 1)
 trues = np.argmax(solution, axis = 1)
 hard_idx = np.where(np.max(submission, axis = 1)<= 0.9)[0]
easy_idx = np.where(np.max(submission, axis = 1)> 0.9)[0]
score = np.mean(predictions == trues)
 hard_score = np.mean(predictions[hard_idx] == trues[hard_idx])if len(hard_idx)else 0
easy_score = np.mean(predictions[easy_idx] == trues[easy_idx])if len(easy_idx)else 0
return roc_score, score, hard_score, easy_score
def get_each_score(submission, solution):
predictions = np.argmax(submission, axis = 1)
 trues = np.argmax(solution, axis = 1)
 roc_scores, scores, hard_scores, easy_scores = [0] * num_classes, [0] * num_classes, [0] * num_classes, [0] * num_classes
for i in range(num_classes):
roc_scores[i] = roc_auc_score(solution.T[i], submission.T[i])
 pred, true = predictions[trues==i], trues[trues==i]
hard_idx = np.where(submission[trues==i][:, i] <= 0.9)[0]
easy_idx = np.where(submission[trues==i][:, i] > 0.9)[0]
scores[i] = np.mean(pred == true)
 hard_scores[i] = np.mean(pred[hard_idx] == true[hard_idx])if len(hard_idx)else 0
easy_scores[i] = np.mean(pred[easy_idx] == true[easy_idx])if len(easy_idx)else 0
roc_score, score, hard_score, easy_score = get_score(submission, solution)
 roc_scores.insert(0, roc_score)
 scores.insert(0, score)
 hard_scores.insert(0, hard_score)
 easy_scores.insert(0, easy_score)
 return roc_scores, scores, hard_scores, easy_scores
def show_running_time(text):
def decorator(func):
@functools.wraps(func)
 def wrapper(*args, **kw):
t0 = time.time()
res = func(*args, **kw)
 print(f"{text} time: {time.time() -t0:.1f} s")
 return res
return wrapper
return decorator
def clear_cache(func):
@functools.wraps(func)
 def wrapper(*args, **kw):
torch.cuda.empty_cache()
gc.collect()
return func(*args, **kw)
 return wrapper
@show_running_time('train')
 @clear_cache
def train_epoch(model, loader, device, optimizer, verbose = False):
model.train()
train_loss = []
optimizer.zero_grad()
acc_steps = accumulation_steps
steps = len(loader)// acc_steps * acc_steps
progress_bar = tqdmauto(loader)if(not USE_TPU or xm.is_master_ordinal())and verbose else None
for batch_idx,(img_batch, origin_img_batch, label_batch)in enumerate(loader):
label_batch_mix, lam = None, 1
if mix is not None:
img_batch, label_batch, label_batch_mix, lam = mix(img_batch, origin_img_batch, label_batch)
 img_batch, label_batch = img_batch.to(device), label_batch.to(device)
 label_batch_mix = None if label_batch_mix is None else label_batch_mix.to(device)
 lam = lam.to(device)if isinstance(lam, torch.Tensor)else lam
if steps is not None and batch_idx >= steps:
acc_steps = 1
logits, metric_logits = model(img_batch)
 loss, _ = criterion(logits, metric_logits, label_batch, is_val = False, lam = lam)
 if label_batch_mix is not None:
loss_mix = criterion(logits, metric_logits, label_batch_mix, is_val = False, lam = 1 - lam)
 loss = loss + loss_mix
loss = loss / acc_steps
loss.backward()
if(batch_idx + 1)% acc_steps == 0:
if USE_TPU:
xm.optimizer_step(optimizer, barrier = True)
 else:
optimizer.step()
optimizer.zero_grad()
loss_np = loss.detach().cpu().item() * acc_steps
train_loss.append(loss_np)
 if(not USE_TPU or xm.is_master_ordinal())and verbose and(batch_idx <= 10 or(batch_idx - 10)% verbose_step == 0):
progress_bar.set_postfix_str(f"loss: {loss_np:.4f}, smooth_loss: {np.mean(train_loss[-20:]):.4f}")
 progress_bar.update(1 if batch_idx <= 10 else 30)
 return np.asarray(train_loss ).mean()
@clear_cache
def val_epoch(model, loader, device):
model.eval()
val_loss, val_loss1, val_loss2 = 0, 0, 0
preds, metric_preds, trues = [], [], []
with torch.no_grad() :
for img_batch, origin_img_batch, label_batch in loader:
img_batch, label_batch = img_batch.to(device), label_batch.to(device)
 logits, metric_logits = model(img_batch)
 loss,(loss1, loss2)= criterion(logits, metric_logits, label_batch, is_val = True)
 val_loss += loss.detach().cpu().item()
val_loss1 += loss1.detach().cpu().item()
val_loss2 += loss2.detach().cpu().item()
preds.append(F.softmax(logits, dim = 1 ).cpu().numpy())
 metric_preds.append(metric_logits.cpu().numpy())
 trues.append(label_batch.cpu().numpy())
 preds = np.concatenate(preds)
 metric_preds = np.concatenate(metric_preds)
 trues = np.concatenate(trues)
 scores1 = get_each_score(preds, trues)
 scores2 = get_each_score(metric_preds, trues)
 val_result =(val_loss / len(trues), val_loss1 / len(trues), val_loss2 / len(trues), preds, metric_preds, trues, scores1, scores2)
 return val_result<load_pretrained>
submission.to_csv('sales_faster_learn.csv', index=False )
def resume_model(model, optimizer = None, lr_scheduler = None, resume_file_path = '', device = 'cpu'):
other_param = {}
history, last_epoch, best_val_loss, best_val_score = None, None, None, None
if os.path.isfile(resume_file_path):
checkpoint = torch.load(resume_file_path, map_location = device)
 model.load_state_dict(checkpoint.pop('state_dict'), strict = False)
 print(f"load model success! last_epoch: {checkpoint['last_epoch']}, file_path: {resume_file_path}")
 if optimizer is not None and lr_scheduler is not None:
optimizer.load_state_dict(checkpoint['optimizer'])
 lr_scheduler.load_state_dict(checkpoint['lr_scheduler'])
 print(f"lr: {optimizer.state_dict() ['param_groups'][0]['lr']:.2e}")
 checkpoint.pop('optimizer', None)
 checkpoint.pop('lr_scheduler', None)
 other_param.update(checkpoint)
 print(f"cur_val_score: {other_param['cur_val_score']:.4f}, cur_val_loss: {other_param['cur_val_loss']:.4f}
" +
f"best_val_score: {other_param['best_val_score']:.4f}, best_val_loss: {other_param['best_val_loss']:.4f}
" +
f"best_epoch: {other_param['best_epoch']}, best_score_epoch: {other_param['best_score_epoch']}
")
 return model, optimizer, lr_scheduler, other_param<choose_model_class>
FileLinks('.' )
model_class = efficientnet_b4 
<train_model>
vega_url = 'https://cdn.jsdelivr.net/npm/vega@' + alt.SCHEMA_VERSION
vega_lib_url = 'https://cdn.jsdelivr.net/npm/vega-lib'
vega_lite_url = 'https://cdn.jsdelivr.net/npm/vega-lite@' + alt.SCHEMA_VERSION
vega_embed_url = 'https://cdn.jsdelivr.net/npm/vega-embed@3'
noext = "?noext"
altair_paths = {
'vega': vega_url + noext,
'vega-lib': vega_lib_url + noext,
'vega-lite': vega_lite_url + noext,
'vega-embed': vega_embed_url + noext
}
workaround =
def add_autoincrement(render_func):
cache = {}
def wrapped(chart, id="vega-chart", autoincrement=True):

 if autoincrement:
if id in cache:
counter = 1 + cache[id]
cache[id] = counter
else:
cache[id] = 0
actual_id = id if cache[id] == 0 else id + '-' + str(cache[id])
 else:
if id not in cache:
cache[id] = 0
actual_id = id
return render_func(chart, id=actual_id)
 return wrapped
@add_autoincrement
def render_alt(chart, id="vega-chart"):
chart_str =
return HTML(
 chart_str.format(
 id=id,
chart=json.dumps(chart)if isinstance(chart, dict)else chart.to_json(indent=None)
)
)
 HTML("".join((
"<script>",
workaround.format(paths=json.dumps(altair_paths)) ,
"</script>" 
)) )
@clear_cache
def train(i_fold, verbose = False):
device = DEVICE if not USE_TPU else xm.xla_device()
devices = [device] if not USE_TPU else [device]
lr = learning_rate if not USE_TPU else learning_rate * max(len(devices), xm.xrt_world_size())
 batch_size = BATCH_SIZE if not USE_TPU else BATCH_SIZE // max(len(devices), xm.xrt_world_size())
 filename = f"{version_name}_{network_name}_fold_{i_fold}.pth"
log_name = f"log_{version_name}_{network_name}_fold_{i_fold}.txt"
train_loader, val_loader = get_train_val_dataloader(i_fold, transforms_train, transforms_val)
 model = model_class(pretrained ).to(device)
 optimizer = torch.optim.AdamW(model.parameters() , lr)
 lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, n_epochs, eta_min = 1e-6)
 train_param = {}
if resume:
if SAVE_PATH != MODEL_PATH and os.path.isfile(MODEL_PATH + log_name):
shutil.copy(MODEL_PATH + log_name, SAVE_PATH + log_name)
 model, optimizer, lr_scheduler, train_param = \
resume_model(model, optimizer, lr_scheduler, MODEL_PATH + filename, device)
 last_epoch = train_param.get('last_epoch', -1)
 best_val_loss, best_val_score = train_param.get('best_val_loss', 2**20), train_param.get('best_val_score', 0)
 best_epoch, best_score_epoch = train_param.get('best_epoch', -1), train_param.get('best_score_epoch', -1)
 history = train_param.get('history', pd.DataFrame())
 def save_model(model, optimizer, lr_scheduler, is_best = False, is_best_score = False):
model_state = {'state_dict': model.state_dict() ,
'last_epoch': last_epoch, 'history': history, 'i_fold': i_fold,
'cur_val_loss': val_loss, 'cur_val_score': val_score,
'best_val_loss': best_val_loss, 'best_val_score': best_val_score,
'best_epoch': best_epoch, 'best_score_epoch': best_score_epoch,
'optimizer': optimizer.state_dict() , 'lr_scheduler': lr_scheduler.state_dict() }
model_path = SAVE_PATH + f"{version_name}_{network_name}_fold_{i_fold}.pth"
if not USE_TPU:
torch.save(model_state, model_path)
 elif xm.is_master_ordinal() :
xm.save(model_state, model_path)
 else:
return
if is_best:
best_model_path = SAVE_PATH + f"{version_name}_{network_name}_fold_{i_fold}_best.pth"
shutil.copy(model_path, best_model_path)
 if is_best_score:
best_model_path = SAVE_PATH + f"{version_name}_{network_name}_fold_{i_fold}_best_score.pth"
shutil.copy(model_path, best_model_path)
 def show_attributes() :
attr =(f"train_set count: {len(train_loader.dataset)} val_set count: {len(val_loader.dataset)}
" +
f"save_path: {SAVE_PATH}
" + f"batch_size: {batch_size}, accumulation_steps: {accumulation_steps}
" +
f"current lr: {optimizer.param_groups[0]['lr']:.2e}, reduce_lr_metric: {reduce_lr_metric}
" +
f"best_val_score: {best_val_score:.5f}, best_val_loss: {best_val_loss:.5f}
" +
f"best_epoch: {best_epoch}, best_score_epoch: {best_score_epoch}
" +
f"last_epoch: {last_epoch}, train_epochs: {train_epochs}, n_epochs: {n_epochs}
")
 if hasattr(lr_scheduler, 'num_bad_epochs'):
attr += f"num_bad_epochs: {lr_scheduler.num_bad_epochs}, min_lr: {lr_scheduler.min_lrs[0]:.2e}, patience: {lr_scheduler.patience}
"
print('
' + attr + '
')
 return attr
try:
if not USE_TPU or xm.is_master_ordinal() :
attr = show_attributes()
if(not USE_TPU or xm.is_master_ordinal())and(not resume or not os.path.isfile(SAVE_PATH + log_name)) :
content =(f"
Training fold {i_fold}
filename: {filename}
train_data: {IMAGE_FILE_NAME[0]}
" +
f"train_img_size: {INPUT_IMG_SHAPE}
batch_size: {batch_size}
accumulation_steps: {accumulation_steps}
n_epochs: {n_epochs}
"+
f"model_class: {model_class.__name__}
optimizer: {optimizer}
lr_scheduler: {lr_scheduler}
")
 content += '
' + f"{time.ctime() } Training start
"
print(content)
 with open(SAVE_PATH + log_name, 'a')as appender:
appender.write(content)
 for epoch in range(last_epoch + 1, train_epochs):
cur_lr = optimizer.param_groups[0]['lr']
train_loss = train_epoch(model, train_loader, device, optimizer, verbose)
 val_result = val_epoch(model, val_loader, device)
 val_loss, val_loss1, val_loss2, preds, metric_preds, trues, scores1, scores2 = val_result
last_epoch, val_score = epoch, scores1[1][0]
is_best_loss, is_best_score = val_loss < best_val_loss, val_score > best_val_score
is_best =(is_best_loss or reduce_lr_metric is 'score')and(is_best_score or reduce_lr_metric is 'loss')
 best_val_loss, best_val_score = min(val_loss, best_val_loss), max(val_score, best_val_score)
 lr_scheduler.step()
content =(f"{time.ctime() } Epoch {epoch}, lr: {cur_lr:.2e}, " +
f"train loss: {train_loss:.5f}, val loss: {val_loss:.5f}, val loss1: {val_loss1:.4f}, val loss2: {val_loss2:.4f}, " +
f"roc_score1: {scores1[0][0]:.4f}, roc_score2: {scores2[0][0]:.4f}, " +
f"score1: {scores1[1][0]:.4f}, hard_score1: {scores1[2][0]:.4f}, easy_score1: {scores1[3][0]:.4f}, " +
f"score2: {scores2[1][0]:.4f}, hard_score2: {scores2[2][0]:.4f}, easy_score2: {scores2[3][0]:.4f}")
 if is_best:
best_epoch, content = epoch, content + " => best metric"
if is_best_score:
best_score_epoch, content = epoch, content + " => best score"
if not USE_TPU or xm.is_master_ordinal() :
print(content)
 with open(SAVE_PATH + log_name, 'a')as appender:
appender.write(content + '
')
 _h = pd.DataFrame({'train_loss': [train_loss], 'val_loss': [val_loss],
'roc_score1': [scores1[0][0]], 'roc_score2': [scores2[0][0]],
'score1': [scores1[1][0]], 'score2': [scores2[1][0]]})
 history = history.append(_h, ignore_index = True)
 save_model(model, optimizer, lr_scheduler, is_best, is_best_score)
 history.to_csv(SAVE_PATH + f"history_{version_name}_fold_{i_fold}.csv")
 content =(f"
best_val_score: {best_val_score:.5f}, best_val_loss: {best_val_loss:.5f}
" +
f"best_epoch: {best_epoch}, best_score_epoch: {best_score_epoch}
")
 with open(SAVE_PATH + log_name, 'a')as appender:
appender.write(content + '
')
 finally:
torch.cuda.empty_cache()
gc.collect()
if not USE_TPU or xm.is_master_ordinal() :
show_attributes()<train_model>
alt.data_transformers.enable('default', max_rows=None )
if TRAIN_MODE:
if DEVICE == torch.device('cuda'):
torch.backends.cudnn.benchmark = True
print('cudnn.benchmark = True')
 if USE_TPU:
torch.set_default_tensor_type('torch.FloatTensor')
 for i_fold in fold:
train(i_fold, verbose = False )<load_from_csv>
train = pd.read_csv('.. /input/sales_train.csv' )
for i_fold in range(n_fold):
if os.path.isfile(SAVE_PATH + f"history_{version_name}_fold_{i_fold}.csv"):
history_file = SAVE_PATH + f"history_{version_name}_fold_{i_fold}.csv"
elif os.path.isfile(MODEL_PATH + f"history_{version_name}_fold_{i_fold}.csv"):
history_file = MODEL_PATH + f"history_{version_name}_fold_{i_fold}.csv"
else:
continue
print(f"show {history_file}")
 history = pd.read_csv(history_file)
 visualize_training_history(history, start_loc = 2 )<load_pretrained>
alt_df = train.groupby('date_block_num')['item_cnt_day'].sum().rename('sum' ).reset_index()
alt_df.head(2 )
def get_models(model_files, model_class):
device = DEVICE if not USE_TPU else xm.xla_device()
models = []
params = []
for model_f in model_files:
if os.path.isfile(SAVE_PATH + model_f):
resume_file_path = SAVE_PATH + model_f
elif os.path.isfile(MODEL_PATH + model_f):
resume_file_path = MODEL_PATH + model_f
else:
continue
model = model_class()
model, _, _, other_params = resume_model(model, resume_file_path = resume_file_path, device = 'cpu')
 model = model.to(device)
 model.eval()
other_params['sub_mode'] = ''
if 'best.' in model_f:
other_params['sub_mode'] = '_best'
elif 'best_score.' in model_f:
other_params['sub_mode'] = '_best_score'
models.append(model)
 params.append(other_params)
 return models, params
@show_running_time('test')
 def predict(model, loader):
device = DEVICE if not USE_TPU else xm.xla_device()
model.eval()
preds, metric_preds = [], []
with torch.no_grad() :
for img_batch in loader:
img_batch = img_batch.to(device)
 logits, metric_logits = model(img_batch)
 preds.append(F.softmax(logits, dim = 1 ).cpu().numpy())
 metric_preds.append(metric_logits.cpu().numpy())
 preds = np.concatenate(preds)
 metric_preds = np.concatenate(metric_preds)
 return preds, metric_preds<train_model>
dict_date_block_num = {0: 'January 2013', 1: 'February 2013', 2: 'March 2013', 3: 'April 2013',
4: 'May 2013', 5: 'June 2013', 6: 'July 2013', 7: 'August 2013',
8: 'September 2013', 9: 'October 2013', 10: 'November 2013', 11: 'December 2013',
12: 'January 2014', 13: 'February 2014', 14: 'March 2014', 15: 'April 2014',
16: 'May 2014', 17: 'June 2014', 18: 'July 2014', 19: 'August 2014',
20: 'September 2014', 21: 'October 2014', 22: 'November 2014', 23: 'December 2014',
24: 'January 2015', 25: 'February 2015', 26: 'March 2015', 27: 'April 2015',
28: 'May 2015', 29: 'June 2015', 30: 'July 2015', 31: 'August 2015',
32: 'September 2015', 33: 'October 2015'}
def show_each_score(folds, model_files):
device = DEVICE if not USE_TPU else xm.xla_device()
models, params = get_models(model_files, model_class)
 cv = np.mean([p['cur_val_score'] for p in params])
 for i_fold, model, p in zip(folds, models, params):
_, val_loader = get_train_val_dataloader(i_fold, transforms_train, transforms_val)
 val_result = val_epoch(model, val_loader, device)
 val_loss, val_loss1, val_loss2, preds, metric_preds, trues, scores1, scores2 = val_result
print(f"
Fold {i_fold}, last_epoch: {p['last_epoch']}
val loss: {val_loss:.5f}, val loss1: {val_loss1:.4f}, val loss2: {val_loss2:.4f}")
 for i in range(5):
print(f"{i}, roc_score1: {scores1[0][i]:.4f}, roc_score2: {scores2[0][i]:.4f}, " +
f"score1: {scores1[1][i]:.4f}, hard_score1: {scores1[2][i]:.4f}, easy_score1: {scores1[3][i]:.4f}, " +
f"score2: {scores2[1][i]:.4f}, hard_score2: {scores2[2][i]:.4f}, easy_score2: {scores2[3][i]:.4f}" )<find_best_params>
split = alt_df['date_block_num'].str.split(" ", n = 1, expand = True )
model_files = [
f"{version_name}_{network_name}_fold_{0}_best.pth",
f"{version_name}_{network_name}_fold_{1}_best.pth",
f"{version_name}_{network_name}_fold_{2}_best.pth",
f"{version_name}_{network_name}_fold_{3}_best.pth",
f"{version_name}_{network_name}_fold_{4}_best.pth",
]
show_each_score(( 0,1,2,3,4), model_files )<predict_on_test>
alt_df_two = alt_df.copy()
alt_df_two['month'] = split[0]
alt_df_two['year'] = split[1]
alt_df_two.head()
@clear_cache
def predict_and_submission(test_loader, model_files):
submission1 = submission.copy()
submission2 = submission.copy()
n_fold_preds, n_fold_metric_preds = [], []
models, params = get_models(model_files, model_class)
 cv = np.mean([p['cur_val_score'] for p in params])
 sub_name = '_'.join([f"{p['i_fold']}_{p['last_epoch']+1}ep{p['sub_mode']}" for p in params])
 submission_name1 = f"{version_name}_{network_name}_{len(model_files)}_fold_submission1_cv{cv:.4f}_ifold_{sub_name}.csv"
submission_name2 = f"{version_name}_{network_name}_{len(model_files)}_fold_submission2_ifold_{sub_name}.csv"
with open(SAVE_PATH + f'log_{version_name}_{network_name}_submission.txt', 'a')as appender:
content =(f"
{time.ctime() }, submission file: {submission_name1}
" +
f"test_data: {IMAGE_FILE_NAME[0]}
test_img_size: {INPUT_IMG_SHAPE}
from models:
")
 for s, p in zip(model_files, params):
content += s + f", i_fold: {p['i_fold']}, train_epochs: {p['last_epoch']+1}ep, cur_val_score: {p['cur_val_score']:.5f}, cur_val_loss: {p['cur_val_loss']:.4f}, best_val_score: {p['best_val_score']:.5f}, best_val_loss: {p['best_val_loss']:.4f}
"
print(content)
 appender.write(content + '
')
 for model in models:
preds, metric_preds = predict(model, test_loader)
 n_fold_preds.append(preds)
 n_fold_metric_preds.append(metric_preds)
 preds = np.stack(n_fold_preds ).mean(axis = 0)
 submission1.iloc[:, 1:] = preds
submission1.to_csv(SAVE_PATH + submission_name1, index=False)
 display(submission1.tail(5))<load_from_csv>
alt.themes.enable('opaque' )
<predict_on_test><EOS> 
alt_df_three = train.groupby(['date_block_num', 'shop_id'])['item_cnt_day'].sum().rename('sum' ).reset_index()
alt_df_three['date_block_num'].replace(dict_date_block_num, inplace=True)
 split = alt_df_three['date_block_num'].str.split(" ", n = 1, expand = True)
 alt_df_three['month'] = split[0]
alt_df_three['year'] = split[1]
alt_df_three.head(2 )