Kaggle competition: iMet Collection 2019 - FGVC6
<SOS> metric: MeanFScoreBeta Kaggle data source: imet-2019-fgvc6<drop_column>
fastai.__version__
zero_sales = data[data['Sales']==0].copy()
data = data[data['Sales']!=0].drop('Open', 1 )<data_type_conversions>
BATCH = 250
SIZE = 150
path = Path('.. /input/imet-2019-fgvc6/' )
int_cols = data.select_dtypes(include='int' ).drop(['Store', 'Customers', 'Sales'], 1 ).columns
categorical_cols = data.select_dtypes(include='object' ).columns
float_cols = data.select_dtypes(include='float' ).drop('CompetitionDistance', 1 ).columns<correct_missing_values>
Path('models' ).mkdir(exist_ok=True)
 !cp '.. /input/resnet50/resnet50.pth' 'models/'
def load_url(*args, **kwargs):
model_dir = Path('models')
 filename = 'resnet50.pth'
if not(model_dir/filename ).is_file() : raise FileNotFoundError
return torch.load(model_dir/filename)
 model_zoo.load_url = load_url
data.fillna(0, inplace=True)
 test.fillna(1, inplace=True )<feature_engineering>
train_df = pd.read_csv(path/'train.csv')
 train_df.head()
data['part'] = 'train'
test['part'] = 'test'
all_data = pd.concat([data, test], 0)[data.columns.tolist() +['Id']]
all_data['Date'] = pd.to_datetime(all_data['Date'])
 all_data['Month'] = all_data['Date'].dt.month
all_data['Year'] = all_data['Date'].dt.year
all_data['Day'] = all_data['Date'].dt.day
all_data['WeekOfYear'] = data.Date.dt.weekofyear
all_data['Quarter'] = data.Date.dt.quarter
all_data.sort_values('Date', inplace=True )<feature_engineering>
labels_df = pd.read_csv(path/'labels.csv')
 labels_df.head()
all_data['CompetitionOpen'] = 12 *(2015 - all_data.Year - all_data.CompetitionOpenSinceYear)+ \ 
(all_data.Month - all_data.CompetitionOpenSinceMonth)
 all_data['CompetitionOpen'] = all_data['CompetitionOpen'].apply(lambda x: x if x > 0 else 0 )<feature_engineering>
test_df = pd.read_csv(path/'sample_submission.csv')
 test_df.head()
all_data['PromoOpen'] = 12 *(all_data.Year - all_data.Promo2SinceYear)+ \ 
(all_data.WeekOfYear - all_data.Promo2SinceWeek)/ 4.0
all_data['PromoOpen'] = all_data['PromoOpen'].apply(lambda x: x if x > 0 else 0 )<feature_engineering>
tfms = get_transforms(do_flip=True, flip_vert=False, max_rotate=0.10, max_zoom=1.5, max_warp=0.2, max_lighting=0.2,
xtra_tfms=[(symmetric_warp(magnitude=(-0,0), p=0)) ,] )
month2str = {1:'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', \
7:'Jul', 8:'Aug', 9:'Sept', 10:'Oct', 11:'Nov', 12:'Dec'}
all_data['month_str'] = all_data.Month.map(month2str)
 def check(row):
if isinstance(row['PromoInterval'],str)and row['month_str'] in row['PromoInterval']:
return 1
else:
return 0
all_data['IsPromoMonth'] = all_data.apply(lambda row: check(row),axis=1 )<filter>
data.show_batch()
all_data ['isBeforeCompetition'] = all_data.apply(lambda x: 1 if x['Year'] < x['CompetitionOpenSinceYear'] else 0, 1 )<feature_engineering>
class FocalLoss(nn.Module):
def __init__(self, gamma=2):
super().__init__()
self.gamma = gamma
def forward(self, logit, target):
target = target.float()
max_val =(-logit ).clamp(min=0)
 loss = logit - logit * target + max_val + \ 
(( -max_val ).exp() +(-logit - max_val ).exp() ).log()
invprobs = F.logsigmoid(-logit *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
if len(loss.size())==2:
loss = loss.sum(dim=1)
 return loss.mean()
all_data['SalesPerCustomer'] = data['Sales']/data['Customers']<categorify>
learn = cnn_learner(data, base_arch=models.resnet50, loss_func=FocalLoss() , metrics=fbeta,ps=0.5 )
mappings = {'0':0, 'a':1, 'b':2, 'c':3, 'd':4}
all_data.StoreType.replace(mappings, inplace=True)
 all_data.Assortment.replace(mappings, inplace=True)
 all_data.StateHoliday.replace(mappings, inplace=True )<drop_column>
learn.fit_one_cycle(3)
 learn.save('stage-1', return_path=True)
 learn.unfreeze()
learn.lr_find()
learn.recorder.plot()
learn.fit_one_cycle(4, slice(1e-5,1e-2))
learn.save('stage-2', return_path=True)
def prepareDf(df, submission=False):
tests_date = "2015-06-12"
tmp_data = all_data[all_data['part']=='train'].copy()
if not submission:
tmp_data = tmp_data[tmp_data['Date']<tests_date]
shift = 48
cols = ['isBeforeCompetition', 'IsPromoMonth', 'PromoOpen', 'DayOfWeek', 'Promo', 'Promo2', 'Month', 'Year', 'Day', 'StateHoliday', 'Assortment', 'Quarter']
for i in ['Sales', 'Customers']:
for j in cols:
tmp = pd.pivot_table(tmp_data, i, ['Store', 'StoreType',j], aggfunc='mean' ).reset_index().rename(columns={i: i+'_StrType_'+j})
 df = pd.merge(df, tmp, 'left', ['Store','StoreType', j])
 cols = ['DayOfWeek', 'Promo', 'Promo2', 'Month', 'Year', 'Day', 'StateHoliday', 'StoreType', 'Assortment', 'PromoInterval', 'Quarter',
'isBeforeCompetition', 'IsPromoMonth', 'PromoOpen']
for i in ['Sales', 'Customers']:
for j in cols:
tmp = pd.pivot_table(tmp_data, i, ['Store', j], aggfunc='mean' ).reset_index().rename(columns={i: i+'_'+j})
 df = pd.merge(df, tmp, 'left', ['Store', j])
 df['rolling_mean_t7_sales'] = df.groupby(['Store'])['Sales'].transform(lambda x: x.shift(shift ).rolling(7 ).mean())
 df['rolling_mean_t30_sales'] = df.groupby(['Store'])['Sales'].transform(lambda x: x.shift(shift ).rolling(30 ).mean())
 df['rolling_mean_t360_sales'] = df.groupby(['Store'])['Sales'].transform(lambda x: x.shift(shift ).rolling(360 ).mean())
 df['rolling_mean_t7_customer'] = df.groupby(['Store'])['Customers'].transform(lambda x: x.shift(shift ).rolling(7 ).mean())
 df['rolling_mean_t30_customer'] = df.groupby(['Store'])['Customers'].transform(lambda x: x.shift(shift ).rolling(30 ).mean())
 df['rolling_mean_t360_customer'] = df.groupby(['Store'])['Customers'].transform(lambda x: x.shift(shift ).rolling(360 ).mean())
 df['shift_sales'] = df.groupby(['Store'])['Sales'].transform(lambda x: x.shift(shift))
df['shift_t7_sales'] = df.groupby(['Store'])['Sales'].transform(lambda x: x.shift(shift+7))
df['shift_t30_sales'] = df.groupby(['Store'])['Sales'].transform(lambda x: x.shift(shift+30))
df['shift_t360_sales'] = df.groupby(['Store'])['Sales'].transform(lambda x: x.shift(shift+360))
df['shift_customer'] = df.groupby(['Store'])['Customers'].transform(lambda x: x.shift(shift))
df['shift_t7_customer'] = df.groupby(['Store'])['Customers'].transform(lambda x: x.shift(shift+7))
df['shift_t30_customer'] = df.groupby(['Store'])['Customers'].transform(lambda x: x.shift(shift+30))
df['shift_t360_customer'] = df.groupby(['Store'])['Customers'].transform(lambda x: x.shift(shift+360))
cols = ['Quarter']
for i in ['Sales', 'Customers']:
for j in cols:
tmp = pd.pivot_table(tmp_data, i, ['Store', 'Year',j], aggfunc='mean' ).reset_index().rename(columns={i: i+'_Year_'+j})
 df = pd.merge(df, tmp, 'left', ['Store','Year', j])
 cols = ['DayOfWeek', 'Promo', 'Promo2', 'Month', 'Year', 'Day', 'StateHoliday', 'Assortment']
for i in ['SalesPerCustomer']:
for j in cols:
tmp = pd.pivot_table(tmp_data, i, ['Store', 'StoreType',j], aggfunc='mean' ).reset_index().rename(columns={i: i+'_StrType_'+j})
 df = pd.merge(df, tmp, 'left', ['Store','StoreType', j])
 return df<categorify>
valid_preds = learn.get_preds(DatasetType.Valid)
 best_thr = find_best_fixed_threshold(*valid_preds )
df = prepareDf(all_data )<sort_values>
test_preds = learn.TTA(ds_type=DatasetType.Test)
 test_df.attribute_ids = join_preds(test_preds, best_thr)
 test_df.head()
<filter><EOS> 
test_df.to_csv('submission.csv', index=False )