Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<import_modules>
!pip install lofo-importance
import pdb
import os
import cv2
import torch
import pandas as pd
import numpy as np
from tqdm import tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(Normalize, Compose)
 from albumentations.torch import ToTensor
import torch.utils.data as data
from model import Unet<categorify>
!pip install pandarallel
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<categorify>
mean_absolute_error)
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples<categorify>
pandarallel.initialize()
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num<define_variables>
train_df = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 test_df = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 item_df = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 item_cat_df = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv')
 shop_df = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv' )
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"<load_from_csv>
exch_rate = pd.read_csv('.. /input/1c-params/exchange_rate.csv', sep=',', parse_dates=['<DATE>'])
 exch_rate = exch_rate.drop(['<TICKER>', '<PER>', '<TIME>'], axis=1)
 exch_rate = exch_rate.rename(columns={'<DATE>':'date', '<CLOSE>':'rate'} )
best_threshold = 0.5
num_workers = 2
batch_size = 4
print('best_threshold', best_threshold)
 min_size = 3500
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 df = pd.read_csv(sample_submission_path)
 testset = DataLoader(
 TestDataset(test_data_folder, df, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True
)<set_options>
train_df['date'] = train_df['date'].parallel_apply(dateutil.parser.parse, dayfirst=True )
torch.cuda.is_available()<load_pretrained>
def name_correction(x):
x = x.lower()
x = x.partition('[')[0]
x = x.partition('(')[0]
x = re.sub('[^A-Za-z0-9А-Яа-я]+', ' ', x)
 x = x.replace(' ', ' ')
 x = x.strip()
return x
ckpt_path = ".. /input/unetstartermodelfile/model.pth"
device = torch.device("cuda")
 model = Unet("resnet18", encoder_weights=None, classes=4, activation=None)
 model.to(device)
 model.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 model.load_state_dict(state["state_dict"] )<save_to_csv>
item_df['item_name'] = item_df['item_name'].apply(name_correction )
predictions = []
for i, batch in enumerate(tqdm(testset)) :
fnames, images = batch
batch_preds = torch.sigmoid(model(images.to(device)))
 batch_preds = batch_preds.detach().cpu().numpy()
for fname, preds in zip(fnames, batch_preds):
for cls, pred in enumerate(preds):
pred, num = post_process(pred, best_threshold, min_size)
 rle = mask2rle(pred)
 name = fname + f"_{cls+1}"
predictions.append([name, rle])
 df = pd.DataFrame(predictions, columns=['ImageId_ClassId', 'EncodedPixels'])
 df.to_csv("submission.csv", index=False )<install_modules>
item_cat_df['new_cats'] =(item_cat_df['item_category_name'] 
.str.split(' - ' ).apply(lambda x: x[0]))
package_path = '.. /input/underscripts/'
sys.path.append(package_path )<import_modules>
item_cat_df['new_subcats'] =(item_cat_df['item_category_name'] 
.str.split(' - ' ).apply(lambda x: x[-1]))
import pdb
import os
import cv2
import torch
import pandas as pd
import numpy as np
from tqdm import tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(HorizontalFlip, ShiftScaleRotate, Normalize, Resize, Compose, GaussNoise)
 from albumentations.torch import ToTensor
import torch.utils.data as data
from model import Unet<categorify>
shop_df['city'] =(shop_df['shop_name'].apply(name_correction)
.str.split(' ' ).apply(lambda x: x[0]))
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<feature_engineering>
shop_df.loc[shop_df['city']=='н', 'city'] = 'нижний_новгород'
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std,TTA=False):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 if TTA==True:
self.transform = Compose(
 [
HorizontalFlip() ,
Normalize(mean=mean,std=std,p=1),
ToTensor() ,
] 
)
 else:
self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples<categorify>
shop_df['shop_type'] =(shop_df['shop_name'].apply(name_correction)
.str.split(' ' ).apply(lambda x: x[1] if(len(x)>1)else 'other'))
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num<define_variables>
shop_df.loc[(shop_df['shop_type']=='орджоникидзе')| 
(shop_df['shop_type']=='ул')| 
(shop_df['shop_type']=='распродажа')| 
(shop_df['shop_type']=='торговля')| 
(shop_df['shop_type']=='магазин'), 'shop_type'] = 'other'
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"<load_from_csv>
shop_df.loc[(shop_df['shop_type']=='новгород')| 
(shop_df['shop_type']=='мтрц'), 'shop_type'] = 'трц'
best_threshold = 0.5
num_workers = 2
batch_size = 1
print('best_threshold', best_threshold)
 min_size = 3500
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 df = pd.read_csv(sample_submission_path)
 testset = DataLoader(
 TestDataset(test_data_folder, df, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True 
)
 testset_TTA = DataLoader(
 TestDataset(test_data_folder,df,mean,std,TTA=True),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True
)<load_pretrained>
shop_df.loc[(shop_df['shop_type']=='посад'),
'shop_type'] = 'тц'
predictions = []
model_name = "resnet18"
models = []
for i in range(4):
ckpt_path = ".. /input/ures184fold/u-resnet18_fold_{}.pth".format(i)
 device = torch.device("cuda")
 model = Unet(encoder_type="resnet",encoder_name="resnet18",classes=4,activation=None)
 model.to(device)
 model.eval()
state = torch.load(ckpt_path,map_location=lambda storage,loc:storage)
 model.load_state_dict(state["state_dict"])
 models.append(model )<predict_on_test>
item_df = pd.merge(item_df, item_cat_df[['item_category_id', 'new_cats', 'new_subcats']] )
predictions = []
for i,(batch,batch_TTA)in enumerate(tqdm(zip(testset,testset_TTA))):
fnames, images = batch
fnames, images_TTA = batch
batch_preds = 0
for model in models:
pred = torch.sigmoid(model(images.to(device)) ).detach().cpu().numpy()
pred_TTA = torch.sigmoid(model(images_TTA.to(device)) ).detach().cpu().numpy()
batch_preds +=(pred+pred_TTA[:,:,:,::-1])/2
batch_preds+=pred
batch_preds/= len(models)
 for fname, preds in zip(fnames, batch_preds):
for cls, pred in enumerate(preds):
pred, num = post_process(pred, best_threshold, min_size)
 rle = mask2rle(pred)
 name = fname + f"_{cls+1}"
predictions.append([name, rle])
 df = pd.DataFrame(predictions, columns=['ImageId_ClassId', 'EncodedPixels'])
 df.to_csv("submission.csv", index=False )<set_options>
item_df = item_df.rename(columns={'new_cats': 'item_category',
'new_subcats': 'item_subcategory'} )
%reload_ext autoreload
%autoreload 2
%matplotlib inline
warnings.filterwarnings("ignore")
 fastai.__version__<define_variables>
train_df = pd.merge(train_df, item_df[['item_name', 'item_id',
'item_category', 'item_subcategory']] )
nfolds = 1
bs = 4
n_cls = 4
noise_th = 2000
TEST = '.. /input/severstal-steel-defect-detection/test_images/'
BASE = '.. /input/severstal-fast-ai-256x256-crops/'
torch.backends.cudnn.benchmark = True<import_modules>
train_df = pd.merge(train_df, shop_df[['city', 'shop_type', 'shop_id']] )
class Hcolumns(nn.Module):
def __init__(self, hooks:Collection[Hook], nc:Collection[int]=None):
super(Hcolumns,self ).__init__()
self.hooks = hooks
self.n = len(self.hooks)
 self.factorization = None
if nc is not None:
self.factorization = nn.ModuleList()
for i in range(self.n):
self.factorization.append(nn.Sequential(
 conv2d(nc[i],nc[-1],3,padding=1,bias=True),
conv2d(nc[-1],nc[-1],3,padding=1,bias=True)))
 def forward(self, x:Tensor):
n = len(self.hooks)
 out = [F.interpolate(self.hooks[i].stored if self.factorization is None
else self.factorization[i](self.hooks[i].stored), scale_factor=2**(self.n-i),
mode='bilinear',align_corners=False)for i in range(self.n)] + [x]
return torch.cat(out, dim=1)
 class DynamicUnet_Hcolumns(SequentialEx):
"Create a U-Net from a given architecture."
def __init__(self, encoder:nn.Module, n_classes:int, blur:bool=False, blur_final=True,
self_attention:bool=False,
y_range:Optional[Tuple[float,float]]=None,
last_cross:bool=True, bottle:bool=False, **kwargs):
imsize =(256,256)
 sfs_szs = model_sizes(encoder, size=imsize)
 sfs_idxs = list(reversed(_get_sfs_idxs(sfs_szs)))
 self.sfs = hook_outputs([encoder[i] for i in sfs_idxs])
 x = dummy_eval(encoder, imsize ).detach()
ni = sfs_szs[-1][1]
middle_conv = nn.Sequential(conv_layer(ni, ni*2, **kwargs),
conv_layer(ni*2, ni, **kwargs)).eval()
x = middle_conv(x)
 layers = [encoder, batchnorm_2d(ni), nn.ReLU() , middle_conv]
self.hc_hooks = [Hook(layers[-1], _hook_inner, detach=False)]
hc_c = [x.shape[1]]
for i,idx in enumerate(sfs_idxs):
not_final = i!=len(sfs_idxs)-1
up_in_c, x_in_c = int(x.shape[1]), int(sfs_szs[idx][1])
 do_blur = blur and(not_final or blur_final)
 sa = self_attention and(i==len(sfs_idxs)-3)
 unet_block = UnetBlock(up_in_c, x_in_c, self.sfs[i], final_div=not_final,
blur=blur, self_attention=sa, **kwargs ).eval()
layers.append(unet_block)
 x = unet_block(x)
 self.hc_hooks.append(Hook(layers[-1], _hook_inner, detach=False))
hc_c.append(x.shape[1])
 ni = x.shape[1]
if imsize != sfs_szs[0][-2:]: layers.append(PixelShuffle_ICNR(ni, **kwargs))
if last_cross:
layers.append(MergeLayer(dense=True))
ni += in_channels(encoder)
 layers.append(res_block(ni, bottle=bottle, **kwargs))
hc_c.append(ni)
 layers.append(Hcolumns(self.hc_hooks, hc_c))
layers += [conv_layer(ni*len(hc_c), n_classes, ks=1, use_activ=False, **kwargs)]
if y_range is not None: layers.append(SigmoidRange(*y_range))
super().__init__(*layers)
 def __del__(self):
if hasattr(self, "sfs"): self.sfs.remove()
def unet_learner(data:DataBunch, arch:Callable, pretrained:bool=True, blur_final:bool=True,
norm_type:Optional[NormType]=NormType, split_on:Optional[SplitFuncOrIdxList]=None,
blur:bool=False, self_attention:bool=False, y_range:Optional[Tuple[float,float]]=None,
last_cross:bool=True, bottle:bool=False, cut:Union[int,Callable]=None,
hypercolumns=True, **learn_kwargs:Any)->Learner:
"Build Unet learner from `data` and `arch`."
meta = cnn_config(arch)
 body = create_body(arch, pretrained, cut)
 M = DynamicUnet_Hcolumns if hypercolumns else DynamicUnet
model = to_device(M(body, n_classes=data.c, blur=blur, blur_final=blur_final,
self_attention=self_attention, y_range=y_range, norm_type=norm_type,
last_cross=last_cross, bottle=bottle), data.device)
 learn = Learner(data, model, **learn_kwargs)
 learn.split(ifnone(split_on, meta['split']))
if pretrained: learn.freeze()
apply_init(model[2], nn.init.kaiming_normal_)
 return learn
class SegmentationLabelList(SegmentationLabelList):
def open(self, fn): return open_mask(fn, div=True)
 class SegmentationItemList(SegmentationItemList):
_label_cls = SegmentationLabelList
def transform(self, tfms:Optional[Tuple[TfmList,TfmList]]=(None,None), **kwargs):
if not tfms: tfms=(None,None)
 assert is_listy(tfms)and len(tfms)== 2
self.train.transform(tfms[0], **kwargs)
 self.valid.transform(tfms[1], **kwargs)
 kwargs['tfm_y'] = False
if self.test: self.test.transform(tfms[1], **kwargs)
 return self
fastai.data_block.ItemLists.transform = transform
def open_mask(fn:PathOrStr, div:bool=True, convert_mode:str='L', cls:type=ImageSegment,
after_open:Callable=None)->ImageSegment:
with warnings.catch_warnings() :
warnings.simplefilter("ignore", UserWarning)
 x = PIL.Image.open(fn ).convert(convert_mode)
 if after_open: x = after_open(x)
 x = pil2tensor(x,np.float32)
 return cls(x )<categorify>
train_df = pd.merge(train_df, exch_rate )
def enc2mask(encs, shape=(1600,256)) :
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 for m,enc in enumerate(encs):
if isinstance(enc,np.float)and np.isnan(enc): continue
s = enc.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1 + m
return img.reshape(shape ).T
def mask2enc(mask, n=n_cls):
pixels = mask.T.flatten()
encs = []
for i in range(1,n+1):
p =(pixels == i ).astype(np.int8)
 if p.sum() == 0: encs.append('')
 else:
p = np.concatenate([[0], p, [0]])
 runs = np.where(p[1:] != p[:-1])[0] + 1
runs[1::2] -= runs[::2]
encs.append(' '.join(str(x)for x in runs))
return encs<load_pretrained>
train_df['price_usd'] = train_df['item_price'] / train_df['rate']
rles,ids_test = [],[]
learns = []
for fold in range(nfolds):
learn = unet_learner(data, models.resnet34, pretrained=False)
 learn.model.load_state_dict(torch.load(Path(BASE)/f'models/fold{fold}.pth')['model'])
 learns.append(learn)
 with zipfile.ZipFile('pred.zip', 'w')as archive_out:
def to_mask(yp, y, id):
name = id + '.png'
save_img(yp[:,:,1:],name,archive_out)
 yp = np.argmax(yp, axis=-1)
 for i in range(n_cls):
idxs = yp == i+1
if idxs.sum() < noise_th: yp[idxs] = 0
encs = mask2enc(yp)
 for i, enc in enumerate(encs):
ids_test.append(id + '.jpg_' + str(i+1))
rles.append(enc)
 model_pred(learns,to_mask,DatasetType.Test)
 sub_df = pd.DataFrame({'ImageId_ClassId': ids_test, 'EncodedPixels': rles})
 sub_df.sort_values(by='ImageId_ClassId' ).to_csv('submission.csv', index=False )<install_modules>
train_df['revenue'] = train_df['item_price'] * train_df['item_cnt_day']
train_df['revenue_usd'] = train_df['price_usd'] * train_df['item_cnt_day']
!pip install.. /input/efficientnet-keras-source-code/repository/qubvel-efficientnet-c993591<import_modules>
train_df.loc[train_df['item_price']<0, :]
import os
import json
import gc
import cv2
import keras
from keras import backend as K
from keras import layers
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Model, load_model
from keras.layers import Input
from keras.layers.convolutional import Conv2D, Conv2DTranspose
from keras.layers.pooling import MaxPooling2D
from keras.layers.merge import concatenate
from keras.optimizers import Adam
from keras.callbacks import Callback, ModelCheckpoint
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from tqdm import tqdm
from sklearn.model_selection import train_test_split<feature_engineering>
train_df.loc[train_df['item_price']<0, 'item_price'] = train_df.loc[
train_df['item_id']==train_df.loc[train_df['item_price']<0,
'item_id'].values[0], 'item_price'].mean()
train_df = pd.read_csv('.. /input/severstal-steel-defect-detection/train.csv')
 train_df['ImageId'] = train_df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 train_df['ClassId'] = train_df['ImageId_ClassId'].apply(lambda x: x.split('_')[1])
 train_df['hasMask'] = ~ train_df['EncodedPixels'].isna()
print(train_df.shape)
 train_df.head()<groupby>
train_df = train_df.loc[(train_df['item_price']<10**5), :]
mask_count_df = train_df.groupby('ImageId' ).agg(np.sum ).reset_index()
mask_count_df.sort_values('hasMask', ascending=False, inplace=True)
 print(mask_count_df.shape)
 mask_count_df.head()<load_from_csv>
train_df = train_df.loc[(train_df['item_cnt_day']<750), :]
sub_df = pd.read_csv('.. /input/severstal-steel-defect-detection/sample_submission.csv')
 sub_df['ImageId'] = sub_df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 test_imgs = pd.DataFrame(sub_df['ImageId'].unique() , columns=['ImageId'])
 test_imgs.head()<set_options>
train_df.loc[train_df['item_cnt_day']==1, :].shape[0] / train_df.shape[0]
def load_img(code, base, resize=True):
path = f'{base}/{code}'
img = cv2.imread(path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 if resize:
img = cv2.resize(img,(224, 224))
return img
def validate_path(path):
if not os.path.exists(path):
os.makedirs(path )<feature_engineering>
t_df = train_df.groupby(['date_block_num', 'item_id', 'shop_id'])[['item_cnt_day']].sum()
t_df = t_df.reset_index().rename(columns={'item_cnt_day': 'item_cnt_month'} )
test_path = '.. /tmp/test'
validate_path(test_path)
 for code in tqdm(test_imgs['ImageId'].values):
img = load_img(code, base='.. /input/severstal-steel-defect-detection/test_images')
 path = code.replace('.jpg', '')
 cv2.imwrite(f'{test_path}/{path}.png', img)
 test_imgs['ImageId'] = test_imgs['ImageId'].apply(lambda x: x.replace('.jpg', '.png'))
sub_df['ImageId'] = sub_df['ImageId'].apply(lambda x: x.replace('.jpg', '.png'))<define_variables>
t_df.loc[t_df['item_cnt_month']==1, :].shape[0] / t_df.shape[0]
BATCH_SIZE = 64
def create_test_gen() :
return ImageDataGenerator(rescale=1/255.).flow_from_dataframe(
 test_imgs,
directory='.. /tmp/test',
x_col='ImageId',
class_mode=None,
target_size=(256, 256),
batch_size=BATCH_SIZE,
shuffle=False 
)
 test_gen = create_test_gen()<load_pretrained>
t_df.loc[t_df['item_cnt_month']<=5, :].shape[0] / t_df.shape[0]
remove_model = load_model('.. /input/severstal-predict-missing-masks/model.h5')
 remove_model.summary()<predict_on_test>
del t_df
gc.collect()
test_missing_pred = remove_model.predict_generator(
 test_gen,
steps=len(test_gen),
verbose=1 
)
 test_imgs['missingCount'] = test_missing_pred.argmax(axis=1)+ 1
test_imgs.head()<count_values>
t_df = train_df[['item_cnt_day', 'date']].copy()
test_imgs['missingCount'].value_counts()<feature_engineering>
t_df['month'] = t_df['date'].dt.month
t_df['year'] = t_df['date'].dt.year
filtered_mask = sub_df['ImageId'].isin(filtered_test_imgs["ImageId"].values)
 filtered_sub_df = sub_df[filtered_mask].copy()
null_sub_df = sub_df[~filtered_mask].copy()
null_sub_df['EncodedPixels'] = null_sub_df['EncodedPixels'].apply(
 lambda x: ' ')
 filtered_sub_df['ImageId'] = filtered_sub_df['ImageId'].apply(
 lambda x: x.replace('.png', '.jpg'))
filtered_test_imgs['ImageId'] = filtered_test_imgs['ImageId'].apply(
 lambda x: x.replace('.png', '.jpg'))
filtered_sub_df.reset_index(drop=True, inplace=True)
 filtered_test_imgs.reset_index(drop=True, inplace=True)
 print(filtered_sub_df.shape)
 print(null_sub_df.shape)
 filtered_sub_df.head()<categorify>
del t_df, data
gc.collect()
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def rle2mask(rle, input_shape):
width, height = input_shape[:2]
mask= np.zeros(width*height ).astype(np.uint8)
 array = np.asarray([int(x)for x in rle.split() ])
 starts = array[0::2]
lengths = array[1::2]
current_position = 0
for index, start in enumerate(starts):
mask[int(start):int(start+lengths[index])] = 1
current_position += lengths[index]
return mask.reshape(height, width ).T
def build_masks(rles, input_shape):
depth = len(rles)
 masks = np.zeros(( *input_shape, depth))
for i, rle in enumerate(rles):
if type(rle)is str:
masks[:, :, i] = rle2mask(rle, input_shape)
 return masks
def build_rles(masks):
width, height, depth = masks.shape
rles = [mask2rle(masks[:, :, i])
 for i in range(depth)]
return rles<train_model>
temp_sum = train_df.groupby(['date_block_num'])[['item_cnt_day', 'revenue', 'revenue_usd']].sum()
temp_sum = temp_sum.reset_index().rename(columns={'item_cnt_day': 'item_cnt_month'} )
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, df, target_df=None, mode='fit',
base_path='.. /input/severstal-steel-defect-detection/train_images',
batch_size=32, dim=(256, 1600), n_channels=3,
n_classes=4, random_state=2019, shuffle=True,aug=False):
self.dim = dim
self.batch_size = batch_size
self.df = df
self.mode = mode
self.base_path = base_path
self.target_df = target_df
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.random_state = random_state
self.seq = aug
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(list_IDs_batch)
 if self.mode == 'fit':
y = self.__generate_y(list_IDs_batch)
 return X, y.astype(int)
 elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.seed(self.random_state)
 np.random.shuffle(self.indexes)
 def __generate_X(self, list_IDs_batch):
'Generates data containing batch_size samples'
X = np.empty(( self.batch_size, *self.dim, self.n_channels))
for i, ID in enumerate(list_IDs_batch):
im_name = self.df['ImageId'].iloc[ID]
img_path = f"{self.base_path}/{im_name}"
img = self.__load_grayscale(img_path)
 X[i,] = gray2rgb(img[:,:,0])
 return X
def __generate_y(self, list_IDs_batch):
y = np.empty(( self.batch_size, *self.dim, self.n_classes), dtype=int)
 for i, ID in enumerate(list_IDs_batch):
im_name = self.df['ImageId'].iloc[ID]
image_df = self.target_df[self.target_df['ImageId'] == im_name]
rles = image_df['EncodedPixels'].values
masks = build_masks(rles, input_shape=self.dim)
 y[i, ] = masks
return y
def __load_grayscale(self, img_path):
img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
 img = img.astype(np.float32)/ 255.
 img = np.expand_dims(img, axis=-1)
 return img
def __load_rgb(self, img_path):
img = cv2.imread(img_path)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 img = img.astype(np.float32)/ 255.
 return img<split>
temp_mean = train_df.groupby(['date_block_num'])[['item_cnt_day', 'revenue', 'revenue_usd']].mean()
temp_mean = temp_mean.reset_index().rename(columns={'item_cnt_day': 'item_cnt_mean',
'revenue': 'av_revenue'} )
BATCH_SIZE = 8
seq = iaa.Sequential([
iaa.Sharpen(( 0.0, 1.0)) ,
iaa.Fliplr() ,
iaa.Flipud() ,
iaa.ElasticTransformation(alpha=50, sigma=5)
 ], random_order=True)
 train_idx, val_idx = train_test_split(
 non_missing_train_idx.index,
random_state=2019,
test_size=0.15 
)
 train_generator = DataGenerator(
 train_idx,
df=mask_count_df,
target_df=train_df,
batch_size=BATCH_SIZE,
n_classes=4,
aug = seq 
)
 val_generator = DataGenerator(
 val_idx,
df=mask_count_df,
target_df=train_df,
batch_size=BATCH_SIZE,
n_classes=4,
aug = seq
)<compute_test_metric>
del temp_mean, temp_sum
gc.collect()
def dice_coef(y_true, y_pred, smooth=1):
y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = K.sum(y_true_f * y_pred_f)
 return(2.* intersection + smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth )<import_modules>
shop_sum = train_df.groupby(['shop_id'], as_index=False)[['item_cnt_day', 'revenue']].sum()
shop_mean = train_df.groupby(['shop_id'], as_index=False)[['item_cnt_day', 'revenue']].mean()
def convolution_block(x, filters, size, strides=(1,1), padding='same', activation=True):
x = Conv2D(filters, size, strides=strides, padding=padding )(x)
 x = BatchNormalization()(x)
 if activation == True:
x = LeakyReLU(alpha=0.1 )(x)
 return x
def residual_block(blockInput, num_filters=16):
x = LeakyReLU(alpha=0.1 )(blockInput)
 x = BatchNormalization()(x)
 blockInput = BatchNormalization()(blockInput)
 x = convolution_block(x, num_filters,(3,3))
x = convolution_block(x, num_filters,(3,3), activation=False)
 x = Add()([x, blockInput])
 return x<load_pretrained>
shop_sum = shop_sum.rename(columns={'item_cnt_day': 'item_cnt_sum'} ).sort_values(by='item_cnt_sum')
 shop_mean = shop_mean.rename(columns={'item_cnt_day': 'item_cnt_mean'} )
def UEfficientNet(input_shape=(None, None, 3),dropout_rate=0.1):
backbone = efn.EfficientNetB2(weights=None,
include_top=False,
input_shape=input_shape)
 input = backbone.input
start_neurons = 8
i=2
lr = []
for l in backbone.layers:
if l.name == 'block{}a_expand_activation'.format(i):
lr.append(l)
 i+=1
conv4 = lr[-1].output
conv4 = LeakyReLU(alpha=0.1 )(conv4)
 pool4 = MaxPooling2D(( 2, 2))(conv4)
 pool4 = Dropout(dropout_rate )(pool4)
 convm = Conv2D(start_neurons * 32,(3, 3), activation=None, padding="same" )(pool4)
 convm = residual_block(convm,start_neurons * 32)
 convm = residual_block(convm,start_neurons * 32)
 convm = LeakyReLU(alpha=0.1 )(convm)
 deconv4 = Conv2DTranspose(start_neurons * 16,(3, 3), strides=(2, 2), padding="same" )(convm)
 deconv4_up1 = Conv2DTranspose(start_neurons * 16,(3, 3), strides=(2, 2), padding="same" )(deconv4)
 deconv4_up2 = Conv2DTranspose(start_neurons * 16,(3, 3), strides=(2, 2), padding="same" )(deconv4_up1)
 deconv4_up3 = Conv2DTranspose(start_neurons * 16,(3, 3), strides=(2, 2), padding="same" )(deconv4_up2)
 uconv4 = concatenate([deconv4, conv4])
 uconv4 = Dropout(dropout_rate )(uconv4)
 uconv4 = Conv2D(start_neurons * 16,(3, 3), activation=None, padding="same" )(uconv4)
 uconv4 = residual_block(uconv4,start_neurons * 16)
 uconv4 = LeakyReLU(alpha=0.1 )(uconv4)
 deconv3 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(uconv4)
 deconv3_up1 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(deconv3)
 deconv3_up2 = Conv2DTranspose(start_neurons * 8,(3, 3), strides=(2, 2), padding="same" )(deconv3_up1)
 conv3 = lr[-2].output
uconv3 = concatenate([deconv3,deconv4_up1, conv3])
 uconv3 = Dropout(dropout_rate )(uconv3)
 uconv3 = Conv2D(start_neurons * 8,(3, 3), activation=None, padding="same" )(uconv3)
 uconv3 = residual_block(uconv3,start_neurons * 8)
 uconv3 = LeakyReLU(alpha=0.1 )(uconv3)
 deconv2 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="same" )(uconv3)
 deconv2_up1 = Conv2DTranspose(start_neurons * 4,(3, 3), strides=(2, 2), padding="same" )(deconv2)
 conv2 = lr[-4].output
uconv2 = concatenate([deconv2,deconv3_up1,deconv4_up2, conv2])
 uconv2 = Dropout(0.1 )(uconv2)
 uconv2 = Conv2D(start_neurons * 4,(3, 3), activation=None, padding="same" )(uconv2)
 uconv2 = residual_block(uconv2,start_neurons * 4)
 uconv2 = LeakyReLU(alpha=0.1 )(uconv2)
 deconv1 = Conv2DTranspose(start_neurons * 2,(3, 3), strides=(2, 2), padding="same" )(uconv2)
 conv1 = lr[-5].output
uconv1 = concatenate([deconv1,deconv2_up1,deconv3_up2,deconv4_up3, conv1])
 uconv1 = Dropout(0.1 )(uconv1)
 uconv1 = Conv2D(start_neurons * 2,(3, 3), activation=None, padding="same" )(uconv1)
 uconv1 = residual_block(uconv1,start_neurons * 2)
 uconv1 = LeakyReLU(alpha=0.1 )(uconv1)
 uconv0 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="same" )(uconv1)
 uconv0 = Dropout(0.1 )(uconv0)
 uconv0 = Conv2D(start_neurons * 1,(3, 3), activation=None, padding="same" )(uconv0)
 uconv0 = residual_block(uconv0,start_neurons * 1)
 uconv0 = LeakyReLU(alpha=0.1 )(uconv0)
 uconv0 = Dropout(dropout_rate/2 )(uconv0)
 uconv0 = Conv2DTranspose(start_neurons * 1,(3, 3), strides=(2, 2), padding="same" )(uconv0)
 output_layer = Conv2D(4,(1,1), padding="same", activation="sigmoid" )(uconv0)
 model = Model(input, output_layer)
 model.name = 'u-xception'
return model<choose_model_class>
pd.merge(shop_sum, shop_df ).tail()
model = UEfficientNet(( 256,1600,3))<compute_test_metric>
del shop_sum, shop_mean
gc.collect()
def dice_coef(y_true, y_pred):
y_true_f = K.flatten(y_true)
 y_pred = K.cast(y_pred, 'float32')
 y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')
 intersection = y_true_f * y_pred_f
score = 2.* K.sum(intersection)/(K.sum(y_true_f)+ K.sum(y_pred_f))
return score
def dice_loss(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = y_true_f * y_pred_f
score =(2.* K.sum(intersection)+ smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 return 1.- score
def bce_dice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)+ dice_loss(y_true, y_pred)
 def bce_logdice_loss(y_true, y_pred):
return binary_crossentropy(y_true, y_pred)- K.log(1.- dice_loss(y_true, y_pred))<import_modules>
item_sum = train_df.groupby(['item_category'], as_index=False)[['item_cnt_day', 'revenue']].sum()
item_mean = train_df.groupby(['item_category'], as_index=False)[['item_cnt_day', 'revenue']].mean()
from tta_wrapper import tta_segmentation<load_pretrained>
item_sum = item_sum.rename(columns={'item_cnt_day': 'item_cnt_sum',
'revenue': 'revenue_sum'} ).sort_values(by='item_cnt_sum')
 item_mean = item_mean.rename(columns={'item_cnt_day': 'item_cnt_mean',
'revenue': 'av_revenue'} )
model.load_weights('.. /input/severstal-simple-2-step-pipeline/modelB0.h5')
 print("load_model")
 test_df = []
tta_model = tta_segmentation(model, h_flip=True, h_shift=(-10, 10), merge='mean')
 for i in range(0, filtered_test_imgs.shape[0], 300):
batch_idx = list(
 range(i, min(filtered_test_imgs.shape[0], i + 300)) 
)
 test_generator = DataGenerator(
 batch_idx,
df=filtered_test_imgs,
shuffle=False,
mode='predict',
base_path='.. /input/severstal-steel-defect-detection/test_images',
target_df=filtered_sub_df,
batch_size=1,
n_classes=4,aug = None 
)
 batch_pred_masks = tta_model.predict_generator(
 test_generator,
workers=1,
verbose=1,
use_multiprocessing=False 
)
 for j, b in tqdm(enumerate(batch_idx)) :
filename = filtered_test_imgs['ImageId'].iloc[b]
image_df = filtered_sub_df[filtered_sub_df['ImageId'] == filename].copy()
pred_masks = batch_pred_masks[j, ]
pred_masks[pred_masks<0.95]=0
pred_masks[pred_masks>0.5]=1
pred_rles = build_rles(pred_masks.astype(int))
image_df['EncodedPixels'] = pred_rles
test_df.append(image_df)
 gc.collect()<concatenate>
movies_df = train_df.loc[train_df['item_category']=='Кино',
:].groupby('item_id', as_index=False ).sum() [['item_id', 'item_cnt_day']]
test_df = pd.concat(test_df)
 print(test_df.shape)
 test_df.head()<concatenate>
top20_movies = pd.merge(movies_df, item_df ).sort_values(by='item_cnt_day', ascending=False ).head(20 )
final_submission_df = pd.concat([test_df, null_sub_df])
 print(final_submission_df.shape)
 final_submission_df.head()<save_to_csv>
top20_movies = top20_movies.rename(columns={'item_cnt_day': 'item_cnt_sum'} )
final_submission_df[['ImageId_ClassId', 'EncodedPixels']].to_csv('submission.csv', index=False )<install_modules>
del top20_movies
gc.collect()
!pip install.. /input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null
package_path = '.. /input/unetmodelscript'
sys.path.append(package_path )<import_modules>
item_sum = item_sum.sort_values(by='revenue_sum' )
import pdb
import os
import cv2
import torch
import pandas as pd
import numpy as np
from tqdm import tqdm
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader, Dataset
from albumentations import(Normalize, Compose)
 from albumentations.torch import ToTensor
import torch.utils.data as data
from model import Unet<categorify>
del item_sum, item_mean
gc.collect()
def mask2rle(img):

 pixels= img.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<categorify>
games_df = train_df.loc[train_df['item_category']=='Игры',
:].groupby('item_id', as_index=False ).sum() [['item_id', 'revenue', 'item_cnt_day']]
class TestDataset(Dataset):

 def __init__(self, root, df, mean, std):
self.root = root
df['ImageId'] = df['ImageId_ClassId'].apply(lambda x: x.split('_')[0])
 self.fnames = df['ImageId'].unique().tolist()
self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname)
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return fname, images
def __len__(self):
return self.num_samples<categorify>
top20_games = pd.merge(games_df, item_df ).sort_values(by='revenue', ascending=False ).head(20 )
def post_process(probability, threshold, min_size):

 mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 256, 1600), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num<define_variables>
consoles_df = train_df.loc[train_df['item_category']=='Игровые консоли',
:].groupby('item_id', as_index=False ).sum() [['item_id', 'revenue', 'item_cnt_day']]
sample_submission_path = '.. /input/severstal-steel-defect-detection/sample_submission.csv'
test_data_folder = ".. /input/severstal-steel-defect-detection/test_images"<load_from_csv>
top20_consoles = pd.merge(consoles_df, item_df ).sort_values(by='revenue', ascending=False ).head(20 )
best_threshold = 0.5
num_workers = 2
batch_size = 4
print('best_threshold', best_threshold)
 min_size = 3500
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 df = pd.read_csv(sample_submission_path)
 testset = DataLoader(
 TestDataset(test_data_folder, df, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True
)<load_pretrained>
gr_date = train_df.groupby('date_block_num' )
ckpt_path = ".. /input/unetstartermodelfile/model.pth"
device = torch.device("cuda")
 model = Unet("resnet18", encoder_weights=None, classes=4, activation=None)
 model.to(device)
 model.eval()
state = torch.load(ckpt_path, map_location=lambda storage, loc: storage)
 model.load_state_dict(state["state_dict"] )<save_to_csv>
all_vals = pd.DataFrame()
for t in tqdm(t_d.values):
all_vals = pd.concat([all_vals, pd.DataFrame(t)], axis=0 )
predictions = []
for i, batch in enumerate(tqdm(testset)) :
fnames, images = batch
batch_preds = torch.sigmoid(model(images.to(device)))
 batch_preds = batch_preds.detach().cpu().numpy()
for fname, preds in zip(fnames, batch_preds):
for cls, pred in enumerate(preds):
pred, num = post_process(pred, best_threshold, min_size)
 rle = mask2rle(pred)
 name = fname + f"_{cls+1}"
predictions.append([name, rle])
 df = pd.DataFrame(predictions, columns=['ImageId_ClassId', 'EncodedPixels'])
 df.to_csv("submission.csv", index=False )<import_modules>
all_vals.columns = ['date_block_num', 'shop_id', 'item_id']
all_vals.head()
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from pandas_profiling import ProfileReport
import tensorflow as tf
import tensorflow_hub as hub
from IPython.display import display, Image
from matplotlib.pyplot import imread
from sklearn.model_selection import train_test_split
import os 
<import_modules>
test_vals = pd.DataFrame()
test_vals['shop_id'] = test_df['shop_id'].copy()
test_vals['item_id'] = test_df['item_id'].copy()
test_vals['date_block_num'] = 34
print("TF version:", tf.__version__)
 print("GPU", "available(YESS!!!!)" if tf.config.list_physical_devices("GPU")else "not available :(" )<load_from_csv>
all_vals = pd.concat([all_vals, test_vals], axis=0 )
labels_csv = pd.read_csv(".. /input/dog-breed-identification/labels.csv")
<save_to_csv>
gr_month = train_df.groupby(['shop_id', 'item_id', 'date_block_num'] )
Labels_Profile = ProfileReport(labels_csv, progress_bar= False)
 Labels_Profile.to_notebook_iframe()<define_variables>
month_sum = month_sum.rename(columns={'item_cnt_day':'item_cnt_month'} )
filenames = [".. /input/dog-breed-identification/train/" + fname + ".jpg" for fname in labels_csv["id"]]
filenames[:5] 
<train_model>
month_sum['item_cnt_month'] = month_sum['item_cnt_month'].clip(0, 20 )
Image(filenames[42] )<data_type_conversions>
month_sum = month_sum.reset_index() [['shop_id', 'item_id',
'item_cnt_month', 'date_block_num',
'revenue', 'revenue_usd']]
labels = labels_csv["breed"].to_numpy()
labels[:10]<define_variables>
month_sum = pd.merge(all_vals, month_sum,
on=['shop_id','item_id', 'date_block_num'],
how='left' ).fillna(0 )
if len(labels)== len(filenames):
print("Number of labels matches number of filenames!")
 else:
print("Number of labels does not match number of filenames, check data directories." )<count_unique_values>
month_sum = reduce_mem_usage(month_sum)
 shop_df = reduce_mem_usage(shop_df)
 item_df = reduce_mem_usage(item_df )
unique_breeds = np.unique(labels)
 len(unique_breeds )<define_variables>
month_sum = month_sum.merge(shop_df[['shop_id', 'city', 'shop_type']],
how='left', on=['shop_id'] )
boolean_labels = [label == np.array(unique_breeds)for label in labels]
boolean_labels[:2]<prepare_x_and_y>
alles = month_sum.merge(item_df, how='left', on=['item_id'] )
X = filenames
y = boolean_labels<split>
exch_rate['month'] = exch_rate['date'].dt.month
exch_rate['year'] = exch_rate['date'].dt.year
NUM_IMAGES = 1000
X_train, X_val, y_train, y_val = train_test_split(X[:NUM_IMAGES],
y[:NUM_IMAGES],
test_size=0.2,
random_state=42)
 len(X_train), len(y_train), len(X_val), len(y_val )<normalization>
temp_mean = exch_rate.groupby(['year', 'month'] ).mean().reset_index()
IMG_SIZE = 224
def process_image(image_path):

 image = tf.io.read_file(image_path)
 image = tf.image.decode_jpeg(image, channels=3)
 image = tf.image.convert_image_dtype(image, tf.float32)
 image = tf.image.resize(image, size=[IMG_SIZE, IMG_SIZE])
 return image<categorify>
temp_mean['date_block_num'] = range(35 )
def get_image_label(image_path, label):

 image = process_image(image_path)
 return image, label<define_variables>
temp_mean = reduce_mem_usage(temp_mean )
BATCH_SIZE = 32
def create_data_batches(x, y=None, batch_size=BATCH_SIZE, valid_data=False, test_data=False):

 if test_data:
print("Creating test data batches...")
 data = tf.data.Dataset.from_tensor_slices(( tf.constant(x)))
 data_batch = data.map(process_image ).batch(BATCH_SIZE)
 return data_batch
elif valid_data:
print("Creating validation data batches...")
 data = tf.data.Dataset.from_tensor_slices(( tf.constant(x),
tf.constant(y)))
 data_batch = data.map(get_image_label ).batch(BATCH_SIZE)
 return data_batch
else:
print("Creating training data batches...")
 data = tf.data.Dataset.from_tensor_slices(( tf.constant(x),
tf.constant(y)))
 data = data.shuffle(buffer_size=len(x))
data = data.map(get_image_label)
 data_batch = data.batch(BATCH_SIZE)
 return data_batch<define_variables>
alles = alles.merge(temp_mean, how='left', on=['date_block_num'] )
train_data = create_data_batches(X_train, y_train)
 val_data = create_data_batches(X_val, y_val, valid_data=True )<split>
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
train_images, train_labels = next(train_data.as_numpy_iterator())
 show_25_images(train_images, train_labels )<define_variables>
gr = train_df.groupby(['item_id', 'shop_id', 'date_block_num'] )
INPUT_SHAPE = [None, IMG_SIZE, IMG_SIZE, 3]
OUTPUT_SHAPE = len(unique_breeds)
 MODEL_URL = "https://tfhub.dev/google/imagenet/mobilenet_v2_130_224/classification/4"<choose_model_class>
m_price = gr.mean()
def create_model(input_shape=INPUT_SHAPE, output_shape=OUTPUT_SHAPE, model_url=MODEL_URL):
print("Building model with:", MODEL_URL)
 model = tf.keras.Sequential([
hub.KerasLayer(MODEL_URL),
tf.keras.layers.Dense(units=OUTPUT_SHAPE,
activation="softmax")
 ])
 model.compile(
 loss=tf.keras.losses.CategoricalCrossentropy() ,
optimizer=tf.keras.optimizers.SGD() ,
metrics=["accuracy"] 
)
 model.build(INPUT_SHAPE)
 return model<load_pretrained>
m_price = m_price.rename(columns={'revenue': 'av_revenue', 'revenue_usd': 'av_revenue_usd',
'item_cnt_day': 'av_item_cnt_day', 'item_price': 'av_item_price',
'price_usd': 'av_price_usd'} )
%load_ext tensorboard 
<choose_model_class>
m_price = m_price.drop(['rate'], axis=1 )
early_stopping = tf.keras.callbacks.EarlyStopping(monitor="val_accuracy",
patience=3 )<import_modules>
m_price['item_purchase_days'] = gr.count() ['date']
m_price = m_price.reset_index()
m_price = reduce_mem_usage(m_price )
print("GPU", "available(YESS!!!!)" if tf.config.list_physical_devices("GPU")else "not available :(" )<train_model>
m_price['av_item_cnt_day'] /= m_price['item_purchase_days']
NUM_EPOCHS = 100
def train_model() :

 model = create_model()
tensorboard = create_tensorboard_callback()
model.fit(x=train_data,
epochs=NUM_EPOCHS,
validation_data=val_data,
validation_freq=1,
callbacks=[tensorboard, early_stopping])
 return model<train_model>
m_group = m_price.groupby(['item_id'], as_index=False)['av_item_price'].mean()
model = train_model()<set_options>
m_group = m_group.rename(columns={'av_item_price': 'mean_price'} )
%tensorboard --logdir./logs 
<predict_on_test>
m_price = pd.merge(all_vals, m_price,
on=['shop_id','item_id', 'date_block_num'],
how='left' ).fillna(0 )
predictions = model.predict(val_data, verbose=1)
 predictions<predict_on_test>
m_price = pd.merge(m_price, m_group, on=['item_id'], how='left' )
print(predictions[0])
 print(f"Max value(probability of prediction): {np.max(predictions[0])}")
 print(f"Sum: {np.sum(predictions[0])}")
 print(f"Max index: {np.argmax(predictions[0])}")
 print(f"Predicted label: {unique_breeds[np.argmax(predictions[0])]}" )<predict_on_test>
m_price['price_dev'] =(m_price['av_item_price'] - m_price['mean_price'])/ m_price['mean_price']
def get_pred_label(prediction_probabilities):

 return unique_breeds[np.argmax(prediction_probabilities)]
pred_label = get_pred_label(predictions[0])
 pred_label<define_variables>
m_price = lag_feature(m_price, [1, 2, 3], 'av_revenue')
 m_price = lag_feature(m_price, [1, 2, 3], 'av_revenue_usd' )
def unbatchify(data):

 images = []
labels = []
for image, label in data.unbatch().as_numpy_iterator() :
images.append(image)
 labels.append(unique_breeds[np.argmax(label)])
 return images, labels
val_images, val_labels = unbatchify(val_data)
 val_images[0], val_labels[0]<define_variables>
m_price['proj_av_revenue'] = 2 *(m_price['av_revenue_lag_1'] -
m_price['av_revenue_lag_2'])+ m_price['av_revenue_lag_2']
m_price['proj_av_revenue_2'] = 3 *(m_price['av_revenue_lag_1'] -
m_price['av_revenue_lag_3'])/ 2 + m_price['av_revenue_lag_3']
full_data = create_data_batches(X, y )<train_model>
m_price['proj_price'] = 2 *(m_price['av_item_price_lag_1'] -
m_price['av_item_price_lag_2'])+ m_price['av_item_price_lag_2']
m_price['proj_price_2'] = 3 *(m_price['av_item_price_lag_1'] -
m_price['av_item_price_lag_3'])/ 2 + m_price['av_item_price_lag_3']
full_model = create_model()<choose_model_class>
cols = [c for c in m_price.columns if c not in ['item_cnt_day', 'rate',
'av_item_price', 'av_price_usd',
'revenue', 'revenue_usd', 'item_purchase_days',
'av_item_cnt_day', 'price_dev',
'av_revenue', 'av_revenue_usd']]
full_model_tensorboard = create_tensorboard_callback()
full_model_early_stopping = tf.keras.callbacks.EarlyStopping(monitor="accuracy",
patience=3 )<set_options>
m_price = reduce_mem_usage(m_price.loc[:, cols] )
%tensorboard --logdir./logs 
<train_model>
alles = pd.merge(alles, m_price,
on=['item_id', 'date_block_num', 'shop_id'], how='left' )
full_model.fit(x=full_data,
epochs=NUM_EPOCHS,
callbacks=[full_model_tensorboard,
full_model_early_stopping] )<train_model>
del m_price
gc.collect()
def save_model(model, suffix=None):

 modeldir = os.path.join("./",
datetime.datetime.now().strftime("%Y%m%d-%H%M%s"))
model_path = modeldir + "-" + suffix + ".h5"
print(f"Saving model to: {model_path}...")
 model.save(model_path)
 return model_path<load_pretrained>
alles = lag_feature(alles, [1, 2, 3, 6, 12], 'item_cnt_month' )
def load_model(model_path):

 print(f"Loading saved model from: {model_path}")
 model = tf.keras.models.load_model(model_path,
custom_objects={"KerasLayer":hub.KerasLayer})
 return model<save_model>
alles['proj_item_cnt'] = 2 *(alles['item_cnt_month_lag_1'] -
alles['item_cnt_month_lag_2'])+ alles['item_cnt_month_lag_2']
alles['proj_item_cnt_2'] = 3 *(alles['item_cnt_month_lag_1'] -
alles['item_cnt_month_lag_3'])/ 2 + alles['item_cnt_month_lag_3']
alles['proj_item_cnt_12'] = 12 *(alles['item_cnt_month_lag_1'] -
alles['item_cnt_month_lag_12'])/ 11 + alles['item_cnt_month_lag_12']
save_model(full_model, suffix="all-images-Adam" )<load_pretrained>
alles['mean_proj_item_cnt'] = np.mean([alles['proj_item_cnt'],
alles['proj_item_cnt_2'], alles['proj_item_cnt_12']], axis=0 )
loaded_full_model = load_model('./20210104-08491609750160-all-images-Adam.h5' )<define_variables>
alles = lag_feature(alles, [1, 2, 3, 6, 12], 'revenue')
 alles = lag_feature(alles, [1, 2, 3, 6, 12], 'revenue_usd' )
test_path = ".. /input/dog-breed-identification/test/"
test_filenames = [test_path + fname for fname in os.listdir(test_path)]
test_filenames[:5]<define_variables>
alles['proj_monthly_revenue'] = 2 *(alles['revenue_lag_1'] -
alles['revenue_lag_2'])+ alles['revenue_lag_2']
alles['proj_monthly_revenue_2'] = 3 *(alles['revenue_lag_1'] -
alles['revenue_lag_3'])/ 2 + alles['revenue_lag_3']
alles['proj_monthly_revenue_12'] = 12 *(alles['revenue_lag_1'] -
alles['revenue_lag_12'])/ 11 + alles['revenue_lag_12']
test_data = create_data_batches(test_filenames, test_data=True )<predict_on_test>
alles['mean_proj_revenue'] = np.mean([alles['proj_monthly_revenue'],
alles['proj_monthly_revenue_2'],
alles['proj_monthly_revenue_12']], axis=0 )
test_predictions = loaded_full_model.predict(test_data,
verbose=-1 )<prepare_output>
alles = alles.drop(['revenue', 'revenue_usd'], axis=1 )
preds_df = pd.DataFrame(columns=["id"] + list(unique_breeds))
preds_df.head()<feature_engineering>
gr_shop = train_df.groupby(['shop_id', 'date_block_num'] )
test_path = ".. /input/dog-breed-identification/test/"
preds_df["id"] = [os.path.splitext(path)[0] for path in os.listdir(test_path)]
preds_df.head()<prepare_output>
pdf = pd.merge(all_vals, gr_shop.count() ['date'], on=['shop_id', 'date_block_num'], how='left' )
preds_df[list(unique_breeds)] = test_predictions
preds_df.head()<save_to_csv>
pdf = pdf.rename(columns={'date':'purchases_per_month'})[['date_block_num',
'shop_id',
'purchases_per_month']].drop_duplicates()
preds_df.to_csv("./full_submission_1_mobilienetV2_adam.csv",
index=False )<import_modules>
tmp = pdf[['date_block_num','shop_id', 'purchases_per_month']]
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id', 'purchases_per_month'+'_lag_'+str(1)]
shifted['date_block_num'] += 1
pdf = pd.merge(pdf, shifted, on=['date_block_num','shop_id'], how='left' )
import numpy as np
import pandas as pd
import os
import time
import matplotlib.pyplot as plt
from keras import Sequential
from keras.layers import Dense, Dropout, InputLayer, Lambda, Input
from keras.preprocessing.image import load_img, img_to_array
from sklearn.model_selection import train_test_split<load_from_csv>
pdf = reduce_mem_usage(pdf )
labels_df = pd.read_csv('/kaggle/input/dog-breed-identification/labels.csv')
 sample = pd.read_csv('/kaggle/input/dog-breed-identification/sample_submission.csv' )<define_variables>
pdf = pdf.drop(['purchases_per_month'], axis=1 )
print('no of images in train dataset: {}'.format(len(labels_df)))
 print('no of images in test dataset: {}'.format(len(sample)) )<categorify>
alles = pd.merge(alles, pdf, on=['shop_id', 'date_block_num'], how='left' )
def images_to_array(directory, label_dataframe, target_size =(331, 331, 3)) :
images = np.zeros([len(label_dataframe), target_size[0], target_size[1], target_size[2]], dtype=np.uint8)
 img = ''
for ix, image_name in enumerate(label_dataframe['id'].values):
img_dir = os.path.join(directory, image_name + '.jpg')
 img = load_img(img_dir, target_size = target_size)
 images[ix] = img_to_array(img)
 del img
label_dict = dict(enumerate(label_dataframe['breed'].unique()))
return images, label_dict<train_model>
cc_df = pd.DataFrame(gr_shop.mean().groupby('shop_id' ).cumcount() , columns=['active_months'])
 cc_df = cc_df.reset_index()
t = time.time()
train_images, labels = images_to_array('/kaggle/input/dog-breed-identification/train', labels_df[:])
 print('runtime in seconds: {}'.format(time.time() - t))<choose_model_class>
cc_df = pd.merge(all_vals[['date_block_num', 'shop_id']], cc_df,
on=['date_block_num', 'shop_id'], how='left' ).drop_duplicates()
def get_feature(model_name, preprocess_input, images, target_size =(331,331,3)) :
base_model = model_name(input_shape = target_size, include_top=False, pooling = 'avg')
 model = Sequential()
model.add(InputLayer(input_shape = target_size))
model.add(Lambda(preprocess_input))
model.add(base_model)
 feature = model.predict(images)
 print('feature-map shape: {}'.format(feature.shape))
return feature<feature_engineering>
tmp = cc_df[['date_block_num','shop_id', 'active_months']]
for i in [1]:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id', 'active_months'+'_lag_'+str(i)]
shifted['date_block_num'] += i
cc_df = pd.merge(cc_df, shifted, on=['date_block_num','shop_id'], how='left' )
inception_preprocess = preprocess_input
inception_feature = get_feature(InceptionV3, preprocess_input, train_images )<feature_engineering>
cc_df = cc_df.drop(['active_months'], axis=1 )
nasnet_preprocessor = preprocess_input
nasnet_features = get_feature(NASNetLarge, nasnet_preprocessor, train_images )<feature_engineering>
for i in [1]:
cc_df['active_months_lag_'+str(i)] += i
cc_df['active_months_lag_'+str(i)] = cc_df['active_months_lag_'+str(i)].fillna(0 ).astype('int' )
xception_preprocess = preprocess_input
xception_feature = get_feature(Xception, xception_preprocess, train_images )<feature_engineering>
alles = pd.merge(alles, cc_df, on=['shop_id', 'date_block_num'], how='left' )
resnet_preprocess = preprocess_input
resnet_feature = get_feature(InceptionResNetV2, resnet_preprocess, train_images )<concatenate>
gr_item = train_df.groupby(['item_id', 'date_block_num'] )
final_features = np.concatenate([inception_feature, nasnet_features, xception_feature, resnet_feature], axis = 1)
 print('final features shape: {}'.format(final_features.shape))
del train_images, inception_feature, nasnet_features, xception_feature, resnet_feature<define_variables>
cc_df = pd.DataFrame(gr_item.mean().groupby('item_id' ).cumcount() , columns=['item_active_months'])
 cc_df = cc_df.reset_index()
class_to_index = dict({labels[ix]:ix for ix in labels.keys() })
 index_to_class = labels<categorify>
cc_df = pd.merge(all_vals[['date_block_num', 'item_id']], cc_df,
on=['date_block_num', 'item_id'], how='left' ).drop_duplicates()
labels = labels_df['breed'].map(class_to_index )<choose_model_class>
tmp = cc_df[['date_block_num','item_id', 'item_active_months']]
for i in [1]:
shifted = tmp.copy()
shifted.columns = ['date_block_num','item_id', 'item_active_months'+'_lag_'+str(i)]
shifted['date_block_num'] += i
cc_df = pd.merge(cc_df, shifted, on=['date_block_num','item_id'], how='left' )
def create_model(features_shape = 1024):
model = Sequential()
model.add(InputLayer(input_shape =(features_shape,)))
 model.add(Dropout(0.7))
model.add(Dense(len(class_to_index), activation = 'softmax'))
model.compile(loss = 'sparse_categorical_crossentropy', optimizer ='Adam', metrics = ['accuracy'])
 return model<train_on_grid>
cc_df = cc_df.drop(['item_active_months'], axis=1 )
<train_model>
for i in [1]:
cc_df['item_active_months_lag_'+str(i)] += i
cc_df['item_active_months_lag_'+str(i)] = cc_df['item_active_months_lag_'+str(i)].fillna(0 ).astype('int' )
model.fit(final_features, labels, epochs = 6, validation_split = 0.2 )<drop_column>
alles = pd.merge(alles, cc_df, on=['item_id', 'date_block_num'], how='left' )
del final_features, labels<categorify>
cc_df = pd.DataFrame(gr.mean().groupby(['shop_id','item_id'] ).cumcount() , columns=['shop_item_active_months'])
 cc_df = cc_df.reset_index()
def images_to_array(directory, label_dataframe, target_size =(331, 331,3)) :
images = np.zeros([len(label_dataframe), target_size[0], target_size[1], target_size[2]], dtype=np.uint8)
 img = ''
for ix, image_name in enumerate(label_dataframe['id'].values):
img_dir = os.path.join(directory, image_name + '.jpg')
 img = load_img(img_dir, target_size = target_size)
 images[ix] = img_to_array(img)
 del img
return images<define_variables>
cc_df = pd.merge(all_vals, cc_df,
on=['date_block_num', 'shop_id', 'item_id'], how='left' ).drop_duplicates()
t = time.time()
test_images = images_to_array('/kaggle/input/dog-breed-identification/test', sample)
 print('runtime in seconds: {}'.format(time.time() - t))<feature_engineering>
cc_df = lag_feature(cc_df, [1], 'shop_item_active_months' )
resnet_feature = get_feature(InceptionResNetV2, resnet_preprocess, test_images)
 xception_feature = get_feature(Xception, xception_preprocess, test_images)
 nasnet_features = get_feature(NASNetLarge, nasnet_preprocessor, test_images)
 inception_feature = get_feature(InceptionV3, preprocess_input, test_images )<concatenate>
for i in [1]:
cc_df['shop_item_active_months_lag_'+str(i)] += i
cc_df['shop_item_active_months_lag_'+str(i)] = cc_df['shop_item_active_months_lag_'
+str(i)].fillna(0 ).astype('int' )
final_features = np.concatenate([inception_feature, nasnet_features, xception_feature, resnet_feature], axis = 1)
 print('final features shape: {}'.format(final_features.shape))
del test_images, inception_feature, nasnet_features, xception_feature, resnet_feature<prepare_output>
cc_df = cc_df.drop(['shop_item_active_months'], axis=1 )
prediction = model.predict(final_features)
 submission = pd.DataFrame({'id':sample.id})
 prediction = pd.DataFrame(prediction)
 prediction.columns = class_to_index.keys()<save_to_csv>
alles = pd.merge(alles, cc_df, on=['item_id', 'shop_id', 'date_block_num'], how='left' )
submission = pd.concat([submission, prediction], axis = 1)
 submission.to_csv('submission.csv', index = False )<set_options>
cum_revs = cum_revs[['revenue', 'revenue_usd']]
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
cum_revs = pd.merge(all_vals[['shop_id', 'date_block_num']],
cum_revs, on=['shop_id', 'date_block_num'], how='left' ).drop_duplicates()
from fastai.vision import *<define_variables>
cum_revs['c_shop_revenue_usd'] = cum_revs.groupby('shop_id')['revenue_usd'].cumsum().values
cum_revs['c_shop_revenue'] = cum_revs.groupby('shop_id')['revenue'].cumsum().values
path = Path('/kaggle/input/dog-breed-identification' )<set_options>
tmp = cum_revs[['date_block_num','shop_id', 'c_shop_revenue']]
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id', 'c_shop_revenue'+'_lag_'+str(1)]
shifted['date_block_num'] += 1
cum_revs = pd.merge(cum_revs, shifted, on=['date_block_num','shop_id'], how='left' )
path.ls()<load_from_csv>
tmp = cum_revs[['date_block_num','shop_id', 'c_shop_revenue_usd']]
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id', 'c_shop_revenue_usd'+'_lag_'+str(1)]
shifted['date_block_num'] += 1
cum_revs = pd.merge(cum_revs, shifted, on=['date_block_num','shop_id'], how='left' )
df = pd.read_csv(path/'labels.csv')
 df.head()<load_from_csv>
cum_revs = reduce_mem_usage(cum_revs )
np.random.seed(42)
 src =(ImageList.from_csv(path, 'labels.csv', folder='train', suffix='.jpg')
.split_by_rand_pct(0.2)
.label_from_df(cols='breed')
.add_test_folder(test_folder = 'test'))<categorify>
alles = pd.merge(alles, cum_revs[['shop_id', 'date_block_num',
'c_shop_revenue_lag_1', 'c_shop_revenue_usd_lag_1']],
on=['shop_id', 'date_block_num'], how='left' )
tfms = get_transforms(max_rotate=20, max_zoom=1.3, max_lighting=0.4, max_warp=0.4, p_affine = 1., p_lighting=1.)<define_variables>
cum_revs = cum_revs[['revenue', 'revenue_usd']]
bs, size = 64, 224<categorify>
cum_revs = pd.merge(all_vals, cum_revs, on=['shop_id', 'date_block_num', 'item_id'], how='left' )
data = src.transform(tfms, size=size ).databunch(bs=bs ).normalize(imagenet_stats )<find_best_params>
cum_revs['revenue'] = cum_revs['revenue'].fillna(0)
 cum_revs['revenue_usd'] = cum_revs['revenue_usd'].fillna(0 )
arch = models.resnet50
metrics = [error_rate, accuracy]<choose_model_class>
cum_revs['c_revenue_usd'] = cum_revs.groupby('shop_id')['revenue_usd'].cumsum().values
cum_revs['c_revenue'] = cum_revs.groupby('shop_id')['revenue'].cumsum().values
learn = cnn_learner(data, arch, metrics=metrics )<define_variables>
cum_revs = reduce_mem_usage(cum_revs )
learn.model_dir = '/kaggle/working'<find_best_params>
alles = pd.merge(alles, cum_revs[['shop_id', 'date_block_num', 'item_id',
'c_revenue_lag_1', 'c_revenue_usd_lag_1']],
on=['shop_id', 'date_block_num', 'item_id'], how='left' )
min_grad_lr = learn.recorder.min_grad_lr
print(min_grad_lr )<train_model>
group = month_sum.groupby(['date_block_num'])
 gr_df = group.mean()
gr_df = gr_df.rename(columns={'item_cnt_month':'av_month'} ).reset_index()
learn.fit_one_cycle(3, slice(min_grad_lr))<save_model>
tmp = gr_df[['date_block_num', 'av_month']]
shifted = tmp.copy()
shifted.columns = ['date_block_num', 'av_month'+'_lag_'+str(1)]
shifted['date_block_num'] += 1
gr_df = pd.merge(gr_df, shifted, on=['date_block_num'], how='left' )
learn.save('stage-1-224-rn50' )<find_best_params>
alles = pd.merge(alles, gr_df[['av_month_lag_1', 'date_block_num']], on=['date_block_num'], how='left' )
min_grad_lr = learn.recorder.min_grad_lr
print(min_grad_lr )<train_model>
group = month_sum.groupby(['date_block_num', 'item_id'])
 gr_df = group.mean()
gr_df = gr_df.rename(columns={'item_cnt_month':'av_month_item'} ).reset_index()
learn.fit_one_cycle(3, slice(min_grad_lr))<save_model>
tmp = gr_df[['date_block_num', 'item_id', 'av_month_item']]
shifted = tmp.copy()
shifted.columns = ['date_block_num', 'item_id', 'av_month_item'+'_lag_'+str(1)]
shifted['date_block_num'] += 1
gr_df = pd.merge(gr_df, shifted, on=['date_block_num', 'item_id'], how='left' )
learn.save('stage-2-224-rn50' )<choose_model_class>
alles = pd.merge(alles, gr_df[['av_month_item_lag_1', 'date_block_num', 'item_id']],
on=['date_block_num', 'item_id'], how='left' )
interp = ClassificationInterpretation.from_learner(learn )<predict_on_test>
group = month_sum.groupby(['date_block_num', 'shop_id'])
 gr_df = group.mean()
gr_df = gr_df.rename(columns={'item_cnt_month':'av_month_shop'} ).reset_index()
predictions, targets = learn.get_preds(ds_type=DatasetType.Test )<define_variables>
tmp = gr_df[['date_block_num', 'shop_id', 'av_month_shop']]
shifted = tmp.copy()
shifted.columns = ['date_block_num', 'shop_id', 'av_month_shop'+'_lag_'+str(1)]
shifted['date_block_num'] += 1
gr_df = pd.merge(gr_df, shifted, on=['date_block_num', 'shop_id'], how='left' )
classes = predictions.argmax(1)
 class_dict = dict(enumerate(learn.data.classes))
labels = [class_dict[i] for i in list(classes[:9].tolist())]
test_images = [i.name for i in learn.data.test_ds.items][:9]<compute_test_metric>
alles = pd.merge(alles, gr_df[['av_month_shop_lag_1', 'date_block_num', 'shop_id']],
on=['date_block_num', 'shop_id'], how='left' )
preds, y = learn.TTA()
print(accuracy(preds,y))
print(log_loss(y, preds))<save_to_csv>
month_cat = pd.merge(month_sum, item_df )
preds_test, y_test = learn.TTA(ds_type=DatasetType.Test)
 df = pd.DataFrame(array(preds_test))
df.columns = data.classes
df.insert(0, "id", [str(pth ).split('/')[5][:-4] for pth in data.test_ds.items])
 df.to_csv("submission.csv", index=False )<import_modules>
group = month_cat.groupby(['date_block_num', 'item_category_id'])
 gr_df = group.mean()
gr_df = gr_df.rename(columns={'item_cnt_month':'av_month_cat'} ).reset_index()
import numpy as np
import pandas as pd
import cv2
import seaborn as sns
import matplotlib.pyplot as plt
from IPython.display import clear_output
from keras import backend as K
from keras.layers import Dense, Activation, Dropout, BatchNormalization, Input, Flatten, Conv2D, MaxPooling2D, Lambda, UpSampling2D, Concatenate
from keras.models import Model
from keras.optimizers import Adam
from keras.callbacks import Callback, EarlyStopping, ReduceLROnPlateau
from keras.applications.inception_resnet_v2 import InceptionResNetV2
from keras.initializers import he_normal
from keras.preprocessing.image import ImageDataGenerator
import os
import shutil<load_from_csv>
tmp = gr_df[['date_block_num', 'item_category_id', 'av_month_cat']]
shifted = tmp.copy()
shifted.columns = ['date_block_num', 'item_category_id', 'av_month_cat'+'_lag_'+str(1)]
shifted['date_block_num'] += 1
gr_df = pd.merge(gr_df, shifted, on=['date_block_num', 'item_category_id'], how='left' )
labels = pd.read_csv('.. /input/dog-breed-identification/labels.csv')
 labels.head(5 )<count_unique_values>
alles = pd.merge(alles, gr_df[['av_month_cat_lag_1', 'date_block_num', 'item_category_id']],
on=['date_block_num', 'item_category_id'], how='left' )
classes = np.unique(labels.breed)
 classes_num = classes.size
classes_num<define_variables>
group = month_cat.groupby(['date_block_num', 'item_subcategory'])
 gr_df = group.mean()
gr_df = gr_df.rename(columns={'item_cnt_month':'av_month_subcat'} ).reset_index()
train_dir = '.. /input/dog-breed-identification/train'
images_names = os.listdir(train_dir)
 images_num = len(images_names)
 print(f'Number of images: {images_num}' )<feature_engineering>
tmp = gr_df[['date_block_num', 'item_subcategory', 'av_month_subcat']]
shifted = tmp.copy()
shifted.columns = ['date_block_num', 'item_subcategory', 'av_month_subcat'+'_lag_'+str(1)]
shifted['date_block_num'] += 1
gr_df = pd.merge(gr_df, shifted, on=['date_block_num', 'item_subcategory'], how='left' )
labels_jpg = labels.copy(deep=True)
 labels_jpg['id'] += '.jpg'
grouped_ids = labels_jpg.groupby('breed')['id'].apply(list ).to_dict()
print(classes[0], grouped_ids[classes[0]] )<init_hyperparams>
alles = pd.merge(alles, gr_df[['av_month_subcat_lag_1', 'date_block_num', 'item_subcategory']],
on=['date_block_num', 'item_subcategory'], how='left' )
test_split = 0.1
valid_split = 0.2<define_variables>
group = month_sum.groupby(['shop_id', 'item_id'])
 gr_df = group.mean()
gr_df = gr_df.rename(columns={'item_cnt_month':'av_shop_item'} ).reset_index()
train_size = 0
valid_size = 0
test_size = 0
for breed_idx,(breed, breed_images)in enumerate(grouped_ids.items()):
for img in breed_images:
rnd_prob = np.random.rand()
if rnd_prob <= test_split:
shutil.copy(train_dir+'/'+img, new_test_dir+'/'+breed)
 test_size += 1
elif rnd_prob <=(test_split + valid_split):
shutil.copy(train_dir+'/'+img, new_valid_dir+'/'+breed)
 valid_size += 1
else:
shutil.copy(train_dir+'/'+img, new_train_dir+'/'+breed)
 train_size += 1
clear_output(wait=True)
 print(f'Organized {breed_idx+1} out of {classes_num} breeds: {breed}' )<init_hyperparams>
alles = pd.merge(alles, gr_df[['av_shop_item', 'shop_id', 'item_id']],
on=['shop_id', 'item_id'], how='left' )
norm_factor = 1 / 255
transform_params = {
'featurewise_center': False,
'featurewise_std_normalization': False,
'samplewise_center': False,
'samplewise_std_normalization': False,
'rotation_range': 30,
'width_shift_range': 0.15,
'height_shift_range': 0.15,
'horizontal_flip': True,
'rescale': norm_factor
}
img_gen = ImageDataGenerator(**transform_params )<define_variables>
alles['first_sale'] =(alles['date_block_num'] -
alles.groupby(['item_id','shop_id'])['date_block_num'].transform('min'))
alles['item_first_sale'] =(alles['date_block_num'] -
alles.groupby('item_id')['date_block_num'].transform('min'))
alles['shop_first_sale'] =(alles['date_block_num'] -
alles.groupby('shop_id')['date_block_num'].transform('min'))
img_feed = ImageDataGenerator(rescale=1/255 )<choose_model_class>
del train_df, month_sum, cum_revs, month_cat, shifted, tmp, gr_df, cc_df
gc.collect()
plateau_reduce = ReduceLROnPlateau(monitor='val_loss', factor=0.01,
patience=1, min_lr=1e-20 )<choose_model_class>
with open('data/alles_final.pkl', 'wb')as handle:
pickle.dump(alles, handle )
e_stop = EarlyStopping(monitor='val_loss', patience=15, mode='min', restore_best_weights=True )<define_variables>
%reset -f
callbacks = [plotter, plateau_reduce, e_stop]<choose_model_class>
mean_absolute_error)
def dense_block(x, neurons, layer_no):
x = Dense(neurons, kernel_initializer=he_normal(layer_no), name=f'topDense{layer_no}' )(x)
 x = Activation('relu', name=f'Relu{layer_no}' )(x)
 x = BatchNormalization(name=f'BatchNorm{layer_no}' )(x)
 x = Dropout(0.5, name=f'Dropout{layer_no}' )(x)
 return x<choose_model_class>
alles = pd.read_pickle('data/alles_final.pkl')
 alles = alles.drop(['mean_price', 'av_shop_item'], axis=1 )
def create_model(shape):
input_layer = Input(shape, name='input_layer')
 incep_res = InceptionResNetV2(include_top=False, weights='imagenet', input_tensor=input_layer)
 for layer in incep_res.layers:
layer.trainable = False
pool = MaxPooling2D(pool_size=[3, 3], strides=[3, 3], padding='same' )(incep_res.output)
 flat1 = Flatten(name='Flatten1' )(pool)
 flat1_bn = BatchNormalization(name='BatchNormFlat' )(flat1)
 dens1 = dense_block(flat1_bn, neurons=512, layer_no=1)
 dens2 = dense_block(dens1, neurons=512, layer_no=2)
 dens3 = dense_block(dens2, neurons=1024, layer_no=3)
 dens_final = Dense(classes_num, name='Dense4' )(dens3)
 output_layer = Activation('softmax', name='Softmax' )(dens_final)
 model = Model(inputs=[input_layer], outputs=[output_layer])
 return model<define_search_space>
rows = alles['date_block_num'] > 11
height, width, channels_num = 512, 512, 3
learning_rate = 0.004
epochs = 15
batch_size = 32<choose_model_class>
alles[['item_name', 'item_category', 'item_subcategory', 'city', 'shop_type']] = \
alles[['item_name', 'item_category',
'item_subcategory', 'city', 'shop_type']].apply(LabelEncoder().fit_transform )
model = create_model(( height, width, channels_num))
optimizer = Adam(learning_rate)
 model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['acc'])
 model.summary()<define_variables>
new_features = pd.read_pickle('.. /input/1c-params/new_features.pkl' )
train_gen = img_gen.flow_from_directory(directory=new_train_dir, target_size=(height, width), color_mode='rgb', classes=list(classes),
class_mode='categorical', batch_size=batch_size, shuffle=True, interpolation='nearest')
 valid_gen = img_feed.flow_from_directory(directory=new_valid_dir, target_size=(height, width), color_mode='rgb', classes=list(classes),
class_mode='categorical', batch_size=batch_size, shuffle=True, interpolation='nearest')
<load_pretrained>
features = list(new_features)+ ['date_block_num']
cols = list(features)+['item_cnt_month']
!wget "https://drive.google.com/uc?id=1-3Y-DB5uhOaY69pvVl5rmfirw9aKCYYB&export=download" -O '/root/training_curves.npy'<load_pretrained>
data = alles.loc[rows, cols]
data = data.fillna(0 )
!wget "https://drive.google.com/uc?id=1Omp4wFOWc2WslToduWAgBkyRv9poQVKb&export=download" -O '/root/decoder_weights.npy'<load_pretrained>
train_rows = data.date_block_num < 33
test_rows = data.date_block_num == 33
decoder_weights = np.load('/root/decoder_weights.npy', allow_pickle=True ).item()
for layer_name, layer_weights in decoder_weights.items() :
model.get_layer(layer_name ).set_weights(layer_weights )<split>
X_train = data[train_rows].drop(['item_cnt_month'], axis=1)
 y_train = data[train_rows]['item_cnt_month']
X_test = data[test_rows].drop(['item_cnt_month'], axis=1)
 y_test = data[test_rows]['item_cnt_month']
X_final_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )
metrics = model.evaluate_generator(test_flow, steps=test_size )<find_best_params>
categorical_features =(data.dtypes[data.dtypes.apply(str ).str.startswith('int')].index ).tolist()
m_names = model.metrics_names
print(f'{m_names[0]} = {metrics[0]}
{m_names[1]} = {metrics[1]}' )<categorify>
categorical_features = ['item_category_id',
'shop_id']
one_hot_map = train_gen.class_indices
one_hot_map<define_variables>
params = pd.read_pickle('.. /input/1c-params/final_params.pkl' )
input_dir = '.. /input/dog-breed-identification'<define_variables>
kf = KFold(n_splits = 5, random_state=42)
 resu1 = 0
impor1 = 0
resu3_mae=0
y_pred = 0
y_final_test = 0
stack_train = np.zeros([X_train.shape[0], ])
 models = []
for train_index, test_index in tqdm(list(kf.split(X_train, y_train))):
X_train2 = X_train.iloc[train_index, :]
y_train2 = y_train.iloc[train_index]
X_test2 = X_train.iloc[test_index, :]
y_test2 = y_train.iloc[test_index]
d_training = lgb.Dataset(X_train2, label=y_train2,
categorical_feature=categorical_features,
free_raw_data=False)
 d_test = lgb.Dataset(X_test2, label=y_test2,
categorical_feature=categorical_features,
free_raw_data=False)
 clf = lgb.train(params, train_set=d_training, num_boost_round=1000,
valid_sets=[d_training, d_test], verbose_eval=25,
early_stopping_rounds=50)
 models.append(clf)
 temp_predict = clf.predict(X_test2, num_iteration=clf.best_iteration)
 stack_train[test_index] = temp_predict
y_pred += clf.predict(X_test, num_iteration=clf.best_iteration)/5
y_final_test += clf.predict(X_final_test, num_iteration=clf.best_iteration)/5
mse = mean_squared_error(y_test2, temp_predict)
 mae = mean_absolute_error(y_test2, temp_predict)
 resu1 += mse/5
resu3_mae += mae/5
impor1 += clf.feature_importance() /5
del X_train2, y_train2, X_test2, y_test2; gc.collect()
test_names = os.listdir(input_dir+'/test')
 test_names.sort()
test_size = len(test_names)
 test_size<predict_on_test>
test_df = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 submission = pd.DataFrame({
"ID": test_df.index,
"item_cnt_month": y_final_test
})
 submission.to_csv('submission.csv', index=False )
<load_from_csv>
r_features = ['item_cnt_month_lag_1', 'item_category_id', 'item_subcategory',
'item_category', 'shop_type', 'city', 'shop_id', 'date_block_num']
r_cols = list(r_features)+['item_cnt_month']
data = alles.loc[rows, r_cols]
data = data.fillna(0 )
!wget 'https://drive.google.com/uc?id=18IQ9WVf1KcVTgwTgGZXKzx61W0ISayI8&export=download' -O 'submission_colab.csv'<load_from_csv>
X_train = data[train_rows].drop(['item_cnt_month'], axis=1)
 y_train = data[train_rows]['item_cnt_month']
X_test = data[test_rows].drop(['item_cnt_month'], axis=1)
 y_test = data[test_rows]['item_cnt_month']
X_final_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )
X_test = pd.read_csv(".. /input/X_test.csv")
 X_train = pd.read_csv(".. /input/X_train.csv")
 y_train = pd.read_csv(".. /input/y_train.csv" )<count_unique_values>
categorical_features =(data.dtypes[data.dtypes.apply(str ).str.startswith('int')].index ).tolist()
y_train.group_id.nunique()<count_unique_values>
categorical_features = ['item_category_id',
'item_subcategory',
'item_category',
'shop_type',
'city',
'shop_id']
y_train.groupby(['group_id', 'surface'] ).nunique().head(10 )<merge>
kf = KFold(n_splits = 5, random_state=42)
 resu1 = 0
impor1 = 0
resu3_mae=0
y_pred = 0
y_final_test = 0
stack_train = np.zeros([X_train.shape[0], ])
 models = []
for train_index, test_index in tqdm(list(kf.split(X_train, y_train))):
X_train2 = X_train.iloc[train_index, :]
y_train2 = y_train.iloc[train_index]
X_test2 = X_train.iloc[test_index, :]
y_test2 = y_train.iloc[test_index]
d_training = lgb.Dataset(X_train2, label=y_train2,
categorical_feature=categorical_features,
free_raw_data=False)
 d_test = lgb.Dataset(X_test2, label=y_test2,
categorical_feature=categorical_features,
free_raw_data=False)
 clf = lgb.train(params, train_set=d_training, num_boost_round=1000,
valid_sets=[d_training, d_test], verbose_eval=25,
early_stopping_rounds=50)
 models.append(clf)
 temp_predict = clf.predict(X_test2, num_iteration=clf.best_iteration)
 stack_train[test_index] = temp_predict
y_pred += clf.predict(X_test, num_iteration=clf.best_iteration)/5
y_final_test += clf.predict(X_final_test, num_iteration=clf.best_iteration)/5
mse = mean_squared_error(y_test2, temp_predict)
 mae = mean_absolute_error(y_test2, temp_predict)
 resu1 += mse/5
resu3_mae += mae/5
impor1 += clf.feature_importance() /5
del X_train2, y_train2, X_test2, y_test2; gc.collect()
X_train = pd.merge(y_train[['series_id', 'group_id', 'surface']], X_train, on='series_id')
 X_train.head()<load_from_csv>
submission = pd.DataFrame({
"ID": test_df.index,
"item_cnt_month": y_final_test
})
 submission.to_csv('submission_no_features.csv', index=False )