Kaggle competition: Predict Future Sales
subs = []
for i in range(5):
out = test_fn(model, testloader)
 output = pd.DataFrame(softmax(out,1), columns = ['healthy','multiple_diseases','rust','scab'])
 output.drop(0, inplace = True)
 output.reset_index(drop=True,inplace=True)
 subs.append(output)
 sub_eff2 = sum(subs)/5<save_to_csv>
items = pd.read_csv('.. /input/items.csv')
 shops = pd.read_csv('.. /input/shops.csv')
 cats = pd.read_csv('.. /input/item_categories.csv')
 train = pd.read_csv('.. /input/sales_train.csv')
 test = pd.read_csv('.. /input/test.csv' ).set_index('ID' )
sub2 = sub_eff2.copy()
sub2['image_id'] = test.image_id
sub2 = sub2[['image_id','healthy','multiple_diseases','rust','scab']]
sub2.to_csv('submission_efficientnet2.csv', index = False )<save_to_csv>
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]
sub =(sub_eff1 + sub_eff2)/2
sub['image_id'] = test.image_id
sub = sub[['image_id','healthy','multiple_diseases','rust','scab']]
sub.to_csv('submission.csv', index = False )<set_options>
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median
!pip install efficientnet_pytorch
warnings.filterwarnings("ignore" )<define_variables>
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11
IMAGE_FOLDER = '.. /input/plant-pathology-npy-images/kaggle/working/image_pickles/'
def get_image_path(filename):
return(IMAGE_FOLDER + filename + '.npy' )<load_from_csv>
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )
train = pd.read_csv('.. /input/plant-pathology-2020-fgvc7/train.csv')
 test = pd.read_csv('.. /input/plant-pathology-2020-fgvc7/test.csv')
 train['image_path'] = train['image_id'].apply(get_image_path)
 test['image_path'] = test['image_id'].apply(get_image_path)
 train_labels = train.loc[:, 'healthy':'scab']
train_paths = train.image_path
test_paths = test.image_path<split>
len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test )
train_paths, valid_paths, train_labels, valid_labels = train_test_split(train_paths, train_labels, test_size = 0.2, random_state=23, stratify = train_labels)
 train_paths.reset_index(drop=True,inplace=True)
 train_labels.reset_index(drop=True,inplace=True)
 valid_paths.reset_index(drop=True,inplace=True)
 valid_labels.reset_index(drop=True,inplace=True )<train_model>
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts
class LeafDataset(Data.Dataset):
def __init__(self, image_paths, labels = None, train = True, test = False):
self.paths = image_paths
self.test = test
if self.test == False:
self.labels = labels
self.train = train
self.train_transform = Compose([HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(rotate_limit=25.0, p=0.7),
OneOf([IAAEmboss(p=1),
IAASharpen(p=1),
Blur(p=1)], p=0.5),
IAAPiecewiseAffine(p=0.5)])
 self.test_transform = Compose([HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
ShiftScaleRotate(rotate_limit=25.0, p=0.7)])
 self.default_transform = Compose([Normalize(( 0.485, 0.456, 0.406),(0.229, 0.224, 0.225), always_apply=True),
ToTensor() ])
 def __len__(self):
return self.paths.shape[0]
def __getitem__(self, i):
image = np.load(self.paths[i])
 if self.test==False:
label = torch.tensor(np.argmax(self.labels.loc[i,:].values))
if self.train:
image = self.train_transform(image=image)['image']
image = self.default_transform(image=image)['image']
elif self.test:
image = self.test_transform(image=image)['image']
image = self.default_transform(image=image)['image']
else:
image = self.default_transform(image=image)['image']
if self.test==False:
return image, label
return image<train_model>
train['revenue'] = train['item_price'] * train['item_cnt_day']
def train_fn(net, loader):
running_loss = 0
preds_for_acc = []
labels_for_acc = []
pbar = tqdm(total = len(loader), desc='Training')
 for _,(images, labels)in enumerate(loader):
images, labels = images.to(device), labels.to(device)
 net.train()
optimizer.zero_grad()
predictions = net(images)
 loss = loss_fn(predictions, labels)
 loss.backward()
optimizer.step()
scheduler.step()
running_loss += loss.item() *labels.shape[0]
labels_for_acc = np.concatenate(( labels_for_acc, labels.cpu().numpy()), 0)
 preds_for_acc = np.concatenate(( preds_for_acc, np.argmax(predictions.cpu().detach().numpy() , 1)) , 0)
 pbar.update()
accuracy = accuracy_score(labels_for_acc, preds_for_acc)
 pbar.close()
return running_loss/TRAIN_SIZE, accuracy
def valid_fn(net, loader):
running_loss = 0
preds_for_acc = []
labels_for_acc = []
pbar = tqdm(total = len(loader), desc='Validation')
 with torch.no_grad() :
for _,(images, labels)in enumerate(loader):
images, labels = images.to(device), labels.to(device)
 net.eval()
predictions = net(images)
 loss = loss_fn(predictions, labels)
 running_loss += loss.item() *labels.shape[0]
labels_for_acc = np.concatenate(( labels_for_acc, labels.cpu().numpy()), 0)
 preds_for_acc = np.concatenate(( preds_for_acc, np.argmax(predictions.cpu().detach().numpy() , 1)) , 0)
 pbar.update()
accuracy = accuracy_score(labels_for_acc, preds_for_acc)
 conf_mat = confusion_matrix(labels_for_acc, preds_for_acc)
 pbar.close()
return running_loss/VALID_SIZE, accuracy, conf_mat
def test_fn(net, loader):
preds_for_output = np.zeros(( 1,4))
with torch.no_grad() :
pbar = tqdm(total = len(loader))
for _, images in enumerate(loader):
images = images.to(device)
 net.eval()
predictions = net(images)
 preds_for_output = np.concatenate(( preds_for_output, predictions.cpu().detach().numpy()), 0)
 pbar.update()
pbar.close()
return preds_for_output<define_variables>
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts
BATCH_SIZE = 8
NUM_EPOCHS = 30
TRAIN_SIZE = train_labels.shape[0]
VALID_SIZE = valid_labels.shape[0]
MODEL_NAME = 'efficientnet-b5'
device = 'cuda'
lr = 8e-4<create_dataframe>
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )
train_dataset = LeafDataset(train_paths, train_labels)
 trainloader = Data.DataLoader(train_dataset, shuffle=True, batch_size = BATCH_SIZE, num_workers = 2)
 valid_dataset = LeafDataset(valid_paths, valid_labels, train = False)
 validloader = Data.DataLoader(valid_dataset, shuffle=False, batch_size = BATCH_SIZE, num_workers = 2)
 test_dataset = LeafDataset(test_paths, train = False, test = True)
 testloader = Data.DataLoader(test_dataset, shuffle=False, batch_size = BATCH_SIZE, num_workers = 2 )<choose_model_class>
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts
model = EfficientNet.from_pretrained(MODEL_NAME)
 num_ftrs = model._fc.in_features
model._fc = nn.Sequential(nn.Linear(num_ftrs,1000,bias=True),
nn.ReLU() ,
nn.Dropout(p=0.5),
nn.Linear(1000,4, bias = True))
model.to(device)
 optimizer = AdamW(model.parameters() , lr = lr, weight_decay = 1e-3)
 num_train_steps = int(len(train_dataset)/ BATCH_SIZE * NUM_EPOCHS)
 scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=len(train_dataset)/BATCH_SIZE*5, num_training_steps=num_train_steps)
 loss_fn = torch.nn.CrossEntropyLoss()<define_variables>
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts
train_loss = []
valid_loss = []
train_acc = []
val_acc = []<train_model>
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
for epoch in range(NUM_EPOCHS):
tl, ta = train_fn(model, loader = trainloader)
 vl, va, conf_mat = valid_fn(model, loader = validloader)
 train_loss.append(tl)
 valid_loss.append(vl)
 train_acc.append(ta)
 val_acc.append(va)
 if(epoch+1)%10==0:
path = 'epoch' + str(epoch)+ '.pt'
torch.save(model.state_dict() , path)
 printstr = 'Epoch: '+ str(epoch)+ ', Train loss: ' + str(tl)+ ', Val loss: ' + str(vl)+ ', Train acc: ' + str(ta)+ ', Val acc: ' + str(va)
 tqdm.write(printstr)
<create_dataframe>
ts = time.time()
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
subs = []
for i in range(5):
out = test_fn(model, testloader)
 output = pd.DataFrame(softmax(out,1), columns = ['healthy','multiple_diseases','rust','scab'])
 output.drop(0, inplace = True)
 output.reset_index(drop=True,inplace=True)
 subs.append(output)
 sub_eff1 = sum(subs)/5<save_to_csv>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
sub1 = sub_eff1.copy()
sub1['image_id'] = test.image_id
sub1 = sub1[['image_id','healthy','multiple_diseases','rust','scab']]
sub1.to_csv('submission_efficientnet1.csv', index = False )<set_options>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
del model
del optimizer
del scheduler
torch.cuda.empty_cache()<choose_model_class>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
model = EfficientNet.from_pretrained(MODEL_NAME)
 num_ftrs = model._fc.in_features
model._fc = nn.Sequential(nn.Linear(num_ftrs,1000,bias=True),
nn.ReLU() ,
nn.Dropout(p=0.5),
nn.Linear(1000,4, bias = True))
model.to(device)
 optimizer = AdamW(model.parameters() , lr = lr, weight_decay = 1e-3)
 scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=len(train_dataset)/BATCH_SIZE*5, num_training_steps=num_train_steps )<define_variables>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
train_loss = []
valid_loss = []
train_acc = []
val_acc = []<train_model>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_type_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')
 matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')
 matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
for epoch in range(NUM_EPOCHS):
tl, ta = train_fn(model, loader = trainloader)
 vl, va, conf_mat = valid_fn(model, loader = validloader)
 train_loss.append(tl)
 valid_loss.append(vl)
 train_acc.append(ta)
 val_acc.append(va)
 if(epoch+1)%10==0:
path = 'epoch' + str(epoch)+ '.pt'
torch.save(model.state_dict() , path)
 printstr = 'Epoch: '+ str(epoch)+ ', Train loss: ' + str(tl)+ ', Val loss: ' + str(vl)+ ', Train acc: ' + str(ta)+ ', Val acc: ' + str(va)
 tqdm.write(printstr)
<create_dataframe>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
subs = []
for i in range(5):
out = test_fn(model, testloader)
 output = pd.DataFrame(softmax(out,1), columns = ['healthy','multiple_diseases','rust','scab'])
 output.drop(0, inplace = True)
 output.reset_index(drop=True,inplace=True)
 subs.append(output)
 sub_eff2 = sum(subs)/5<save_to_csv>
ts = time.time()
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
sub2 = sub_eff2.copy()
sub2['image_id'] = test.image_id
sub2 = sub2[['image_id','healthy','multiple_diseases','rust','scab']]
sub2.to_csv('submission_efficientnet2.csv', index = False )<save_to_csv>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
sub =(sub_eff1 + sub_eff2)/2
sub['image_id'] = test.image_id
sub = sub[['image_id','healthy','multiple_diseases','rust','scab']]
sub.to_csv('submission.csv', index = False )<init_hyperparams>
ts = time.time()
group = matrix.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')
 matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')
 matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
bool_random_flip_left_right=0
bool_random_flip_up_down=0
bool_random_brightness=0
bool_random_contrast=0
bool_random_hue=0
bool_random_saturation=0
cutmix_rate=0.
 mixup_rate= 0.
 gridmask_rate = 0.
 pre_trained='imagenet'
dense_activation='softmax'
bool_lr_scheduler=0
tta_times=0
cross_validation_folds=5
bool_focal_loss = 0
label_smoothing_rate=0.
<define_variables>
ts = time.time()
group = matrix.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')
 matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')
 matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
bool_tta = tta_times and max(bool_random_flip_left_right,
bool_random_flip_up_down,
bool_random_brightness,
bool_random_contrast,
bool_random_hue,
bool_random_saturation)
 print(bool_tta)
 assert(bool_focal_loss and label_smoothing_rate)== 0 , 'focal_loss和label_smoothing最多同时使用一个'
assert(tta_times and cross_validation_folds)== 0 , 'focal_loss和label_smoothing最多同时使用一个'<import_modules>
ts = time.time()
group = train.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 time.time() - ts
!pip install -U efficientnet
!pip install tensorflow_addons
print(tf.__version__)
 print(tf.keras.__version__ )<load_from_csv>
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 time.time() - ts
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
img_size = 768
EPOCHS = 18
lr_if_without_scheduler = 0.0003
nb_classes = 4
print('BATCH_SIZE是：',BATCH_SIZE)
 path='.. /input/plant-pathology-2020-fgvc7/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sub = pd.read_csv(path + 'sample_submission.csv')
 train_labels = train.loc[:, 'healthy':].values
print('
',train_labels[0])
 train_paths = train.image_id.apply(lambda x: path + '/images/' + x + '.jpg' ).values
img = plt.imread(train_paths[0])
 print('
',img.shape)
 plt.imshow(img)
 if tpu:
path = GCS_DS_PATH
train_paths = train.image_id.apply(lambda x: path + '/images/' + x + '.jpg' ).values
test_paths = test.image_id.apply(lambda x: path + '/images/' + x + '.jpg' ).values<normalization>
matrix['month'] = matrix['date_block_num'] % 12
def decode_image(filename, label=None, image_size=(img_size, img_size)) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_aug(image, label=None):
if bool_random_flip_left_right:
image = tf.image.random_flip_left_right(image)
 if bool_random_flip_up_down:
image = tf.image.random_flip_up_down(image)
 if bool_random_brightness:
image = tf.image.random_brightness(image,0.2)
 if bool_random_contrast:
image = tf.image.random_contrast(image,0.6,1.4)
 if bool_random_hue:
image = tf.image.random_hue(image,0.07)
 if bool_random_saturation:
image = tf.image.random_saturation(image,0.5,1.5)
 if label is None:
return image
else:
return image, label<categorify>
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )
def cutmix(image, label, PROBABILITY = cutmix_rate):
DIM = img_size
imgs = []; labs = []
for j in range(BATCH_SIZE):
P = tf.cast(tf.random.uniform([], 0, 1)<= PROBABILITY, tf.int32)
 k = tf.cast(tf.random.uniform([], 0, BATCH_SIZE), tf.int32)
 x = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)
 y = tf.cast(tf.random.uniform([], 0, DIM), tf.int32)
 b = tf.random.uniform([], 0, 1)
 WIDTH = tf.cast(DIM * tf.math.sqrt(1-b),tf.int32)* P
ya = tf.math.maximum(0,y-WIDTH//2)
 yb = tf.math.minimum(DIM,y+WIDTH//2)
 xa = tf.math.maximum(0,x-WIDTH//2)
 xb = tf.math.minimum(DIM,x+WIDTH//2)
 one = image[j,ya:yb,0:xa,:]
two = image[k,ya:yb,xa:xb,:]
three = image[j,ya:yb,xb:DIM,:]
middle = tf.concat([one,two,three],axis=1)
 img = tf.concat([image[j,0:ya,:,:],middle,image[j,yb:DIM,:,:]],axis=0)
 imgs.append(img)
 a = tf.cast(WIDTH*WIDTH/DIM/DIM,tf.float32)
 lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE, nb_classes))
return image2,label2<categorify>
ts = time.time()
cache = {}
matrix['item_shop_last_sale'] = -1
matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = str(row.item_id)+' '+str(row.shop_id)
 if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
matrix.at[idx, 'item_shop_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts
def mixup(image, label, PROBABILITY = mixup_rate):
DIM = img_size
imgs = []; labs = []
for j in range(BATCH_SIZE):
k = tf.cast(tf.random.uniform([],0,BATCH_SIZE),tf.int32)
 a = tf.random.uniform([],0,1)
 P = tf.cast(tf.random.uniform([], 0, 1)<= PROBABILITY, tf.int32)
 if P==1:
a=0.
 img1 = image[j,]
img2 = image[k,]
imgs.append(( 1-a)*img1 + a*img2)
 lab1 = label[j,]
lab2 = label[k,]
labs.append(( 1-a)*lab1 + a*lab2)
 image2 = tf.reshape(tf.stack(imgs),(BATCH_SIZE,DIM,DIM,3))
label2 = tf.reshape(tf.stack(labs),(BATCH_SIZE,nb_classes))
return image2,label2<prepare_x_and_y>
ts = time.time()
cache = {}
matrix['item_last_sale'] = -1
matrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = row.item_id
if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
if row.date_block_num>last_date_block_num:
matrix.at[idx, 'item_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts
def transform(image, inv_mat, image_shape):
h, w, c = image_shape
cx, cy = w//2, h//2
new_xs = tf.repeat(tf.range(-cx, cx, 1), h)
 new_ys = tf.tile(tf.range(-cy, cy, 1), [w])
 new_zs = tf.ones([h*w], dtype=tf.int32)
 old_coords = tf.matmul(inv_mat, tf.cast(tf.stack([new_xs, new_ys, new_zs]), tf.float32))
old_coords_x, old_coords_y = tf.round(old_coords[0, :] + w//2), tf.round(old_coords[1, :] + h//2)
 clip_mask_x = tf.logical_or(old_coords_x<0, old_coords_x>w-1)
 clip_mask_y = tf.logical_or(old_coords_y<0, old_coords_y>h-1)
 clip_mask = tf.logical_or(clip_mask_x, clip_mask_y)
 old_coords_x = tf.boolean_mask(old_coords_x, tf.logical_not(clip_mask))
old_coords_y = tf.boolean_mask(old_coords_y, tf.logical_not(clip_mask))
new_coords_x = tf.boolean_mask(new_xs+cx, tf.logical_not(clip_mask))
new_coords_y = tf.boolean_mask(new_ys+cy, tf.logical_not(clip_mask))
old_coords = tf.cast(tf.stack([old_coords_y, old_coords_x]), tf.int32)
 new_coords = tf.cast(tf.stack([new_coords_y, new_coords_x]), tf.int64)
 rotated_image_values = tf.gather_nd(image, tf.transpose(old_coords))
rotated_image_channel = list()
for i in range(c):
vals = rotated_image_values[:,i]
sparse_channel = tf.SparseTensor(tf.transpose(new_coords), vals, [h, w])
 rotated_image_channel.append(tf.sparse.to_dense(sparse_channel, default_value=0, validate_indices=False))
return tf.transpose(tf.stack(rotated_image_channel), [1,2,0])
 def random_rotate(image, angle, image_shape):
def get_rotation_mat_inv(angle):
angle = math.pi * angle / 180
cos_val = tf.math.cos(angle)
 sin_val = tf.math.sin(angle)
 one = tf.constant([1], tf.float32)
 zero = tf.constant([0], tf.float32)
 rot_mat_inv = tf.concat([cos_val, sin_val, zero, -sin_val, cos_val, zero, zero, zero, one], axis=0)
 rot_mat_inv = tf.reshape(rot_mat_inv, [3,3])
 return rot_mat_inv
angle = float(angle)* tf.random.normal([1],dtype='float32')
 rot_mat_inv = get_rotation_mat_inv(angle)
 return transform(image, rot_mat_inv, image_shape)
 def GridMask(image_height, image_width, d1, d2, rotate_angle=1, ratio=0.5):
h, w = image_height, image_width
hh = int(np.ceil(np.sqrt(h*h+w*w)))
 hh = hh+1 if hh%2==1 else hh
d = tf.random.uniform(shape=[], minval=d1, maxval=d2, dtype=tf.int32)
 l = tf.cast(tf.cast(d,tf.float32)*ratio+0.5, tf.int32)
 st_h = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 st_w = tf.random.uniform(shape=[], minval=0, maxval=d, dtype=tf.int32)
 y_ranges = tf.range(-1 * d + st_h, -1 * d + st_h + l)
 x_ranges = tf.range(-1 * d + st_w, -1 * d + st_w + l)
 for i in range(0, hh//d+1):
s1 = i * d + st_h
s2 = i * d + st_w
y_ranges = tf.concat([y_ranges, tf.range(s1,s1+l)], axis=0)
 x_ranges = tf.concat([x_ranges, tf.range(s2,s2+l)], axis=0)
 x_clip_mask = tf.logical_or(x_ranges < 0 , x_ranges > hh-1)
 y_clip_mask = tf.logical_or(y_ranges < 0 , y_ranges > hh-1)
 clip_mask = tf.logical_or(x_clip_mask, y_clip_mask)
 x_ranges = tf.boolean_mask(x_ranges, tf.logical_not(clip_mask))
y_ranges = tf.boolean_mask(y_ranges, tf.logical_not(clip_mask))
hh_ranges = tf.tile(tf.range(0,hh), [tf.cast(tf.reduce_sum(tf.ones_like(x_ranges)) , tf.int32)])
 x_ranges = tf.repeat(x_ranges, hh)
 y_ranges = tf.repeat(y_ranges, hh)
 y_hh_indices = tf.transpose(tf.stack([y_ranges, hh_ranges]))
x_hh_indices = tf.transpose(tf.stack([hh_ranges, x_ranges]))
y_mask_sparse = tf.SparseTensor(tf.cast(y_hh_indices, tf.int64), tf.zeros_like(y_ranges), [hh, hh])
 y_mask = tf.sparse.to_dense(y_mask_sparse, 1, False)
 x_mask_sparse = tf.SparseTensor(tf.cast(x_hh_indices, tf.int64), tf.zeros_like(x_ranges), [hh, hh])
 x_mask = tf.sparse.to_dense(x_mask_sparse, 1, False)
 mask = tf.expand_dims(tf.clip_by_value(x_mask + y_mask, 0, 1), axis=-1)
 mask = random_rotate(mask, rotate_angle, [hh, hh, 1])
 mask = tf.image.crop_to_bounding_box(mask,(hh-h)//2,(hh-w)//2, image_height, image_width)
 return mask
def apply_grid_mask(image, image_shape, PROBABILITY = gridmask_rate):
AugParams = {
'd1' : 100,
'd2': 160,
'rotate' : 45,
'ratio' : 0.3
}
mask = GridMask(image_shape[0], image_shape[1], AugParams['d1'], AugParams['d2'], AugParams['rotate'], AugParams['ratio'])
 if image_shape[-1] == 3:
mask = tf.concat([mask, mask, mask], axis=-1)
 mask = tf.cast(mask,tf.float32)
 P = tf.cast(tf.random.uniform([], 0, 1)<= PROBABILITY, tf.int32)
 if P==1:
return image*mask
else:
return image
def gridmask(img_batch, label_batch):
return apply_grid_mask(img_batch,(img_size,img_size, 3)) , label_batch<categorify>
ts = time.time()
matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')
 time.time() - ts
def get_train_dataset(train_paths,train_labels):
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( train_paths, train_labels.astype(np.float32)))
.map(decode_image, num_parallel_calls=AUTO)
.cache() 
.map(data_aug, num_parallel_calls=AUTO)
.shuffle(512)
.batch(BATCH_SIZE,drop_remainder=True)
)
 if cutmix_rate:
print('启用cutmix')
 train_dataset =train_dataset.map(cutmix, num_parallel_calls=AUTO)
 if mixup_rate:
print('启用mixup')
 train_dataset =train_dataset.map(mixup, num_parallel_calls=AUTO)
 if gridmask_rate:
print('启用gridmask')
 train_dataset =train_dataset.map(gridmask, num_parallel_calls=AUTO)
 if(cutmix_rate or mixup_rate):
train_dataset =train_dataset.unbatch().shuffle(512 ).batch(BATCH_SIZE)
 train_dataset = train_dataset.repeat().prefetch(AUTO)
 return train_dataset<categorify>
ts = time.time()
matrix = matrix[matrix.date_block_num > 11]
time.time() - ts
def get_validation_dataset(valid_paths,valid_labels):
dataset = tf.data.Dataset.from_tensor_slices(( valid_paths, valid_labels))
dataset = dataset.map(decode_image, num_parallel_calls=AUTO)
 dataset = dataset.cache()
dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset<categorify>
ts = time.time()
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix)
 time.time() - ts
def re_produce_test_dataset(test_paths):
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_paths)
.map(decode_image, num_parallel_calls=AUTO)
)
 if bool_tta:
test_dataset = test_dataset.cache().map(data_aug, num_parallel_calls=AUTO)
 test_dataset = test_dataset.batch(BATCH_SIZE)
 return test_dataset<init_hyperparams>
matrix.to_pickle('data.pkl')
 del matrix
del cache
del group
del items
del shops
del cats
del train
gc.collect() ;
LR_START = 0.00003
LR_MAX = 0.0003 * strategy.num_replicas_in_sync
LR_MIN = 0.00003
LR_RAMPUP_EPOCHS = 3
LR_SUSTAIN_EPOCHS = 4
LR_EXP_DECAY =.8
def lrfn(epoch):
if epoch < LR_RAMPUP_EPOCHS:
lr =(LR_MAX - LR_START)/ LR_RAMPUP_EPOCHS * epoch + LR_START
elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:
lr = LR_MAX
else:
lr =(LR_MAX - LR_MIN)* LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS)+ LR_MIN
return lr
lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 rng = [i for i in range(EPOCHS)]
y = [lrfn(x)for x in rng]
if bool_lr_scheduler:
plt.plot(rng, y)
 print("Learning rate schedule: {:.3g} to {:.3g} to {:.3g}".format(y[0], max(y), y[-1]))<choose_model_class>
data = pd.read_pickle('data.pkl' )
def get_model() :
with strategy.scope() :
base_model = efn.EfficientNetB0(weights=pre_trained, include_top=False, pooling='avg', input_shape=(img_size, img_size, 3))
x = base_model.output
predictions = Dense(nb_classes, activation=dense_activation )(x)
 model = Model(inputs=base_model.input, outputs=predictions)
 if label_smoothing_rate:
print('启用label_smoothing')
 my_loss=tf.keras.losses.CategoricalCrossentropy(label_smoothing=label_smoothing_rate)
 elif bool_focal_loss:
my_loss = tfa.losses.SigmoidFocalCrossEntropy(reduction=tf.keras.losses.Reduction.AUTO)
 else:
my_loss='categorical_crossentropy'
model.compile(optimizer=tf.keras.optimizers.Adam(lr_if_without_scheduler),
loss=my_loss,
metrics=['accuracy'])
 return model<define_variables>
data = data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'item_cnt_month_lag_6',
'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3',
'date_item_avg_item_cnt_lag_6',
'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2',
'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6',
'date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month',
'days',
'item_shop_last_sale',
'item_last_sale',
'item_shop_first_sale',
'item_first_sale',
]]
callbacks=[]
if bool_lr_scheduler:
callbacks.append(lr_callback )<train_model>
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )
if not cross_validation_folds:
model = get_model()
model.fit(
 get_train_dataset(train_paths,train_labels),
steps_per_epoch=train_labels.shape[0] // BATCH_SIZE,
callbacks=callbacks,
epochs=EPOCHS 
)
 if bool_tta:
probabilities = []
for i in range(tta_times+1):
print('TTA Number: ',i,'
')
 test_dataset = re_produce_test_dataset(test_paths)
 probabilities.append(model.predict(test_dataset))
y_pred = np.mean(probabilities,axis =0)
 else:
test_dataset = re_produce_test_dataset(test_paths)
 y_pred = model.predict(test_dataset )<save_to_csv>
del data
gc.collect() ;
sub.loc[:, 'healthy':] = y_pred
sub.to_csv('submission.csv', index=False)
 sub.head()<import_modules>
ts = time.time()
model = XGBRegressor(
 max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 10)
 time.time() - ts
<categorify><EOS> 
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('xgb_test.pickle', 'wb'))