Kaggle competition: iMet Collection 2019 - FGVC6
<SOS> metric: MeanFScoreBeta Kaggle data source: imet-2019-fgvc6<save_to_csv>
!nvidia-smi
test_probs = gbm.predict(xgb.DMatrix(test[features]), ntree_limit=gbm.best_ntree_limit)
 indices = test_probs < 0
test_probs[indices] = 0
submission = pd.DataFrame({"Id": test["Id"], "Sales": np.expm1(test_probs)})
 submission.to_csv("xgboost.csv", index=False )<set_options>
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
 print(device )
%matplotlib inline
%reload_ext autoreload
%autoreload 2
np.random.seed(23)
 np.set_printoptions(threshold=50, edgeitems=20 )<load_from_disk>
train = pd.read_csv(".. /input/imet-2019-fgvc6/train.csv")
 lable = pd.read_csv(".. /input/imet-2019-fgvc6/labels.csv")
 test = pd.read_csv(".. /input/imet-2019-fgvc6/sample_submission.csv" )
OUTPUT = '/kaggle/working/'
PATH='/kaggle/input/rossmann-time-series-data-engineering/'
df = pd.read_feather(f'{PATH}df')
 train_df = pd.read_feather(f'{PATH}joined2')
 test_df = pd.read_feather(f'{PATH}joined2_test')
 train_df.shape, test_df.shape<define_variables>
def creatData(train,lable_length):
train = np.array(train)
 trainA_data = []
lab_data = []
for t in range(train_length):
v = np.zeros(lable_length)
 lab = []
for s in train[t,1].split(" "):
v[int(s)] = 1
lab.append(int(s))
trainA_data.append([train[t,0],v])
 lab_data.append([train[t,0],np.array(lab)])
 return np.array(trainA_data),np.array(lab_data)
cat_vars = ['Store', 'DayOfWeek', 'Promo',
'StateHoliday', 'SchoolHoliday', 'Year', 'Month', 'Week', 'Day',
'Is_year_end', 'Is_year_start', 'StoreType', 'Assortment',
'Promo2', 'PromoInterval', 'State',
'Events', 'CompetitionMonthsOpen',
'Promo2Weeks',
'SchoolHoliday_bw','StateHoliday_bw', 'Promo_bw', 'SchoolHoliday_fw', 'StateHoliday_fw','Promo_fw',
'SchoolHoliday_DaySum', 'StateHoliday_DaySum', 'Promo_DaySum',
'SchoolHoliday_DayCount', 'StateHoliday_DayCount', 'Promo_DayCount']
cont_vars = ['CompetitionDistance', 'Max_TemperatureC', 'Mean_TemperatureC', 'Min_TemperatureC',
'Max_Humidity','Mean_Humidity', 'Min_Humidity',
'Max_Wind_SpeedKm_h', 'Mean_Wind_SpeedKm_h','Precipitationmm','CloudCover',
'trend', 'trend_DE', 'CompetitionDaysOpen', 'Promo2Days',
'AfterSchoolHoliday', 'BeforeSchoolHoliday', 'AfterStateHoliday',
'BeforeStateHoliday', 'AfterPromo', 'BeforePromo']
dep_var = 'Sales'
df = train_df[cat_vars + cont_vars + [dep_var,'Date']].copy()<filter>
train_a,train_lab = creatData(train,lable_length)
cut = train_df['Date'][(train_df['Date'] == train_df['Date'][len(test_df)])].index.max()
valid_idx = range(cut); valid_idx<categorify>
def dfilter(data,th):
dfilter = []
for i in range(len(data)) :
if train_a[i][1].sum() >th:
dfilter.append(train_a[i])
 return np.array(dfilter )
procs=[FillMissing, Categorify, Normalize]
datalist =(TabularList.from_df(df, path=OUTPUT, cat_names=cat_vars, cont_names=cont_vars, procs=procs,)
.split_by_idx(valid_idx=valid_idx)
.label_from_df(cols=dep_var, label_cls=FloatList, log=True)
.add_test(TabularList.from_df(test_df, path=PATH, cat_names=cat_vars, cont_names=cont_vars)))
 data = datalist.databunch(bs=512 )<not_enough_vertices>
datafilter = dfilter(train_a,1 )
defaults.device<choose_model_class>
datafilter = train_a
max_log_y = np.log(np.max(train_df['Sales'])*1.2)
 y_range = torch.tensor([0, max_log_y], device=defaults.device)
 learn = tabular_learner(data, layers=[1000,500], ps=[0.001,0.01], emb_drop=0.04,
y_range=y_range, metrics=exp_rmspe )<categorify>
image_resize = 200
data_transforms2 = transforms.Compose([
transforms.Resize(( image_resize,image_resize)) ,
transforms.ToTensor() ,
transforms.Normalize(
 [0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
 ])
 data_transforms = transforms.Compose([
transforms.Resize(( image_resize,image_resize)) ,
transforms.ToTensor() ,
])
 train_transformer = transforms.Compose([
transforms.Resize(( 128,128)) ,
transforms.ToTensor() ,
] )
learn.to_fp16<define_variables>
class trainDataset(Dataset):
def __init__(self, train_lib, transform,transform2):
self.filenames = train_lib[:,0]
self.labels = train_lib[:,1]
self.transform = transform
self.transform2 = transform2
def __len__(self):
return len(self.filenames)
 def __getitem__(self, idx):
img = Image.open(".. /input/imet-2019-fgvc6/train/"+format(self.filenames[idx])+'.png')
 image = self.transform(img)
 image2 = self.transform2(img)
 return image,image2, self.labels[idx]
learn.data.batch_size 
<import_modules>
class testDataset(Dataset):
def __init__(self, test_lib, transform,transform2):
test_lib = np.array(test_lib)
 self.filenames = test_lib[:,0]
self.transform = transform
self.transform2 = transform2
def __len__(self):
return len(self.filenames)
 def __getitem__(self, idx):
img = Image.open(".. /input/imet-2019-fgvc6/test/"+format(self.filenames[idx])+'.png')
 image = self.transform(img)
 image2 = self.transform2(img)
 return image,image2,self.filenames[idx]
fastai.__version__<find_best_params>
trainA_dataloader = DataLoader(trainDataset(datafilter, data_transforms,data_transforms2),batch_size=32, shuffle=True,num_workers=2, pin_memory=True)
learn.lr_find(end_lr=100, wd=0.3)
 learn.recorder.plot()<train_model>
score_dataloader = DataLoader(trainDataset(datafilter[:1000], data_transforms,data_transforms2),batch_size=32, shuffle=False,num_workers=2, pin_memory=True )
learn.fit_one_cycle(5, 3e-3, wd=0.3 )<save_model>
test_dataloader = DataLoader(testDataset(test, data_transforms,data_transforms2),batch_size=32,shuffle=False,num_workers=2, pin_memory=True )
learn.save('bs512_5ep_2e-3_wd0.3' )<train_model>

 __all__ = ['SENet', 'senet154', 'se_resnet50', 'se_resnet101', 'se_resnet152',
'se_resnext50_32x4d', 'se_resnext101_32x4d']
pretrained_settings = {
'senet154': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/senet154-c7b49a05.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet50': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet50-ce0d4300.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet101': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet101-7e38fcc6.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnet152': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet152-d17c99b7.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnext50_32x4d': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext50_32x4d-a260b3a4.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
'se_resnext101_32x4d': {
'imagenet': {
'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth',
'input_space': 'RGB',
'input_size': [3, 224, 224],
'input_range': [0, 1],
'mean': [0.485, 0.456, 0.406],
'std': [0.229, 0.224, 0.225],
'num_classes': 1000
}
},
}
class SEModule(nn.Module):
def __init__(self, channels, reduction):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 x = self.sigmoid(x)
 return module_input * x
class Bottleneck(nn.Module):

 def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = self.se_module(out)+ residual
out = self.relu(out)
 return out
class SEBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm2d(planes * 2)
 self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
stride=stride, padding=1, groups=groups,
bias=False)
 self.bn2 = nn.BatchNorm2d(planes * 4)
 self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNetBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEResNetBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,
stride=stride)
 self.bn1 = nn.BatchNorm2d(planes)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,
groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(planes)
 self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNeXtBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None, base_width=4):
super(SEResNeXtBottleneck, self ).__init__()
width = math.floor(planes *(base_width / 64)) * groups
self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
stride=1)
 self.bn1 = nn.BatchNorm2d(width)
 self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
padding=1, groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(width)
 self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SENet(nn.Module):
def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
inplanes=128, input_3x3=True, downsample_kernel_size=3,
downsample_padding=1, num_classes=1000):

 super(SENet, self ).__init__()
self.inplanes = inplanes
if input_3x3:
layer0_modules = [ 
('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,
bias=False)) , 
('bn1', nn.BatchNorm2d(64)) , 
('relu1', nn.ReLU(inplace=True)) , 
('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
bias=False)) , 
('bn2', nn.BatchNorm2d(64)) , 
('relu2', nn.ReLU(inplace=True)) , 
('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
bias=False)) , 
('bn3', nn.BatchNorm2d(inplanes)) , 
('relu3', nn.ReLU(inplace=True)) ,
]
else:
layer0_modules = [ 
('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
padding=3, bias=False)) , 
('bn1', nn.BatchNorm2d(inplanes)) , 
('relu1', nn.ReLU(inplace=True)) ,
]
layer0_modules.append(( 'pool', nn.MaxPool2d(3, stride=2,
ceil_mode=True)))
 self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
self.layer1 = self._make_layer(
 block,
planes=64,
blocks=layers[0],
groups=groups,
reduction=reduction,
downsample_kernel_size=1,
downsample_padding=0 
)
 self.layer2 = self._make_layer(
 block,
planes=128,
blocks=layers[1],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer3 = self._make_layer(
 block,
planes=256,
blocks=layers[2],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer4 = self._make_layer(
 block,
planes=512,
blocks=layers[3],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.avg_pool = nn.AvgPool2d(7, stride=1)
 self.dropout = nn.Dropout(dropout_p)if dropout_p is not None else None
self.last_linear = nn.Linear(512 * block.expansion, num_classes)
 def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,
downsample_kernel_size=1, downsample_padding=0):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=downsample_kernel_size, stride=stride,
padding=downsample_padding, bias=False),
nn.BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, groups, reduction, stride,
downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes, groups, reduction))
return nn.Sequential(*layers)
 def features(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
def logits(self, x):
x = self.avg_pool(x)
 if self.dropout is not None:
x = self.dropout(x)
 x = x.view(x.size(0), -1)
 x = self.last_linear(x)
 return x
def forward(self, x):
x = self.features(x)
 x = self.logits(x)
 return x
def initialize_pretrained_model(model, num_classes, settings):
assert num_classes == settings['num_classes'], \
'num_classes should be {}, but is {}'.format(
 settings['num_classes'], num_classes)
 model.load_state_dict(model_zoo.load_url(settings['url']))
model.input_space = settings['input_space']
model.input_size = settings['input_size']
model.input_range = settings['input_range']
model.mean = settings['mean']
model.std = settings['std']
def senet154(num_classes=1000, pretrained='imagenet'):
model = SENet(SEBottleneck, [3, 8, 36, 3], groups=64, reduction=16,
dropout_p=0.2, num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['senet154'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnet50(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNetBottleneck, [3, 4, 6, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnet50'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnet101(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNetBottleneck, [3, 4, 23, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnet101'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnet152(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNetBottleneck, [3, 8, 36, 3], groups=1, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnet152'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnext50_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnext50_32x4d'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
def se_resnext101_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 if pretrained is not None:
settings = pretrained_settings['se_resnext101_32x4d'][pretrained]
initialize_pretrained_model(model, num_classes, settings)
 return model
learn.fit_one_cycle(5, 1e-3, wd=0.3 )<save_model>
cache_dir = expanduser(join('~', '.torch'))
if not exists(cache_dir):
makedirs(cache_dir)
 models_dir = join(cache_dir, 'models/')
 if not exists(models_dir):
makedirs(models_dir)
 copyfile(".. /input/pretrained-pytorch/densenet201-c1103571.pth", models_dir+"densenet201-c1103571.pth")
 copyfile(".. /input/pretrained-pytorch/resnet50-19c8e357.pth", models_dir+"resnet50-19c8e357.pth")
 copyfile(".. /input/pretrained-se-resnet/se_resnext101_32x4d-3b2fe3d8.pth", models_dir+"se_resnext101_32x4d-3b2fe3d8.pth" )
learn.save('bs512_2_5ep_1e-3_wd0.3' )<train_model>
class Ensemble(nn.Module):
def __init__(self, modelA, modelB,input_length,output_length):
super(Ensemble, self ).__init__()
self.modelA = modelA
self.modelB = modelB
self.classifier = nn.Linear(input_length, output_length)
 def forward(self, xin,xin2):
x1 = self.modelA(xin)
 x2 = self.modelB(xin2)
 x = torch.cat(( x1, x2), dim=1)
 x = self.classifier(F.relu(x))
return x
seresnext_model = se_resnext101_32x4d(pretrained='imagenet')
 seresnext_model.last_linear = nn.Linear(in_features=2048, out_features=lable_length, bias=True)
 resnet_model = models.resnet50(pretrained=True)
 resnet_model.fc= nn.Linear(in_features=2048, out_features=lable_length)
 model = seresnext_model
model.to(device )
learn.fit_one_cycle(5, 3e-4, wd=0.3)
 learn.recorder.plot_losses()<save_model>
def train(epoch,fnum,dloader):
model.train()
for step,(x,x2,y)in enumerate(dloader):
data = Variable(x ).cuda()
target = Variable(y ).cuda()
output = model(data)
 loss = loss_func(output, target.float())
 optimizer.zero_grad()
loss.backward()
optimizer.step()
if step==0:
start = time.time()
ti = 0
elif step==100:
ti = time.time() -start
ti = ti*(len(dloader)/100)
 if step % 100 == 0:
second = ti*(((len(dloader)-step)/len(dloader)))
 print('Train Fold:{}/4 Ep: {}/3 [{}/{}({:.0f}%)]\t Loss: {:.6f}\t Remain : {} '.
 format(fnum+1,
epoch+1,
step * len(data),
len(dloader.dataset),
100.*step/len(dloader),
loss.data.item() ,
datetime.timedelta(seconds = int(second))))
data.cpu()
target.cpu()
torch.cuda.empty_cache()
print("Finish" )
learn.save('bs512_3_5ep_3e-4_wd0.3' )<load_pretrained>
def val(dloader):
model.eval()
los = []
for step,(x,x2, y)in enumerate(dloader):
data = Variable(x ).cuda()
target = Variable(y ).cuda()
output = model(data)
 loss = loss_func(output, target.float())
 los.append(loss.item())
 if step %100 == 0:
print('[{}/{}({:.1f}%)]'.format(step * len(data),
len(dloader.dataset),
100.*step/len(dloader)))
 data.cpu()
target.cpu()
torch.cuda.empty_cache()
los = np.array(los)
 avg_val_loss = los.sum() /len(los)
 print(f"Avg val loss: {avg_val_loss:.8f}")
data = datalist.databunch(bs=128)
 learn.data = data
learn.data.batch_size<train_model>
class FocalLoss(nn.Module):
def __init__(self, gamma):
super().__init__()
self.gamma = gamma
def forward(self, input, target):
if not(target.size() == input.size()):
raise ValueError("Target size({})must be the same as input size({})".format(target.size() , input.size()))
max_val =(-input ).clamp(min=0)
 loss = input - input * target + max_val +(( -max_val ).exp() +(-input - max_val ).exp() ).log()
invprobs = F.logsigmoid(-input *(target * 2 - 1))
loss =(invprobs * self.gamma ).exp() * loss
return loss.mean()
learn.fit_one_cycle(5, 1e-3, wd=0.2)
 learn.recorder.plot_losses()<save_model>
fold = KFold(n_splits = 4, random_state = 10)
 for fold_num,(trn_idx, val_idx)in enumerate(fold.split(datafilter)) :
Ftrain_dataloader = datafilter[trn_idx, :]
Fval_dataloader = datafilter[val_idx, :]
val_dataloader = DataLoader(trainDataset(Fval_dataloader, data_transforms,data_transforms2),batch_size=32, shuffle=False,num_workers=2, pin_memory=True)
 train_dataloader = DataLoader(trainDataset(Ftrain_dataloader, data_transforms,data_transforms2),batch_size=32, shuffle=True,num_workers=2, pin_memory=True)
 for epoch in range(3):
if epoch==0:
optimizer = torch.optim.Adam(model.parameters() , lr=0.0001/(2**epoch))
else:
optimizer = torch.optim.SGD(model.parameters() , lr=0.0001,momentum=0.9, weight_decay=1e-4)
 loss_func = FocalLoss(0.4)
 train(epoch,fold_num,train_dataloader)
 val(val_dataloader )
learn.save('bs128_5ep_1e-3_wd0.2' )<train_model>
torch.cuda.empty_cache()
!nvidia-smi
learn.fit_one_cycle(5, 1e-3, wd=0.2)
 learn.recorder.plot_losses()<train_model>
torch.save(model, 'net.pkl')
learn.fit_one_cycle(5, 1e-3, wd=0.1)
 learn.recorder.plot_losses()<train_model>
def findPre(output,gate):
a = ''
output = np.array(output)
 m = np.max(output)
 for i in range(len(output)) :
if output[i]>gate:
a = a + format(i)+' '
return a
learn.fit_one_cycle(20, 1e-3, wd=0.1)
 learn.recorder.plot_losses()<save_model>
def makeScore(pre,ans):
pre = np.array(pre)
 va = fbeta_score(y_pred=pre, y_true=ans, beta=2, average="samples")
 print("Score : {:.5f}".format(va))
return va
learn.save('last' )<save_to_csv>
def findThreshold() :
score = []
candidates = np.arange(0, 1.0, 0.01)
 for th in candidates:
print("Threshold : {:.2f}".format(th))
_,pre = test(model = model,dloader = score_dataloader,threshold = th)
 score.append(makeScore(np.array(pre),np.array(train_a[:1000,1].tolist())))
 print("=============================")
 pm = np.array(score ).argmax()
best_th, best_score = candidates[pm], score[pm]
return best_th, best_score
test_preds=learn.get_preds(DatasetType.Test)
 test_df["Sales"]=np.exp(test_preds[0].data ).numpy().T[0]
test_df[["Id","Sales"]]=test_df[["Id","Sales"]].astype("int")
 test_df[["Id","Sales"]].to_csv("rossmann_submission.csv",index=False )<load_from_csv>
bt, bs = findThreshold()
print("Best Threshold : {:.2f}".format(bt))
print("Best Score : {:.5f}".format(bs))
sales = pd.read_csv(".. /input/train.csv")
 sales.head()<load_from_csv>
torch.cuda.empty_cache()
!nvidia-smi
stores = pd.read_csv(".. /input/store.csv")
 stores.head()<merge>
sub,_ = test(model = model,dloader = test_dataloader,threshold = bt )
test = pd.read_csv('.. /input/test.csv')
 test = pd.merge(test, stores, 'left', 'Store')
 test.head()<feature_engineering>
sub = pd.DataFrame(sub )
data = pd.merge(sales, stores, 'left', 'Store' ).sample(frac= 1)
 data['Date'] = pd.to_datetime(data['Date'])
 data['Month'] = data['Date'].dt.month
data['Year'] = data['Date'].dt.year
data['Day'] = data['Date'].dt.day
data['WeekOfYear'] = data.Date.dt.weekofyear
data['Quarter'] = data.Date.dt.quarter
data['StateHoliday'] = data['StateHoliday'].replace(0, '0')
 data.head()<count_values>
sub = sub.rename(index=str, columns={0: "id", 1: "attribute_ids"} )
print(len(set(sales['Store'])- set(stores['Store'])) )<count_values>
sub.to_csv('submission.csv', index=False )
<count_values><EOS> 
sub.to_csv('submission.csv', index=False )