Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<data_type_conversions>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
ts = time.time()
cache = {}
matrix['item_last_sale'] = -1
matrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = row.item_id
if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
if row.date_block_num>last_date_block_num:
matrix.at[idx, 'item_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts<feature_engineering>
warnings.filterwarnings('always')
 warnings.filterwarnings('ignore')
 %matplotlib inline
style.use('fivethirtyeight')
 sns.set(style='whitegrid', color_codes=True)
ts = time.time()
matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')
 time.time() - ts<filter>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(42 )
ts = time.time()
matrix = matrix[matrix.date_block_num > 11]
time.time() - ts<correct_missing_values>
temp = vision.data.open_image
ts = time.time()
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix)
 time.time() - ts<load_pretrained>
def get_df() :
base_image_dir = os.path.join('.. ', 'input/aptos2019-blindness-detection/')
 train_dir = os.path.join(base_image_dir,'train_images/')
 df = pd.read_csv(os.path.join(base_image_dir, 'train.csv'))
df['path'] = df['id_code'].map(lambda x: os.path.join(train_dir,'{}.png'.format(x)))
 df = df.drop(columns=['id_code'])
 df = df.sample(frac=1 ).reset_index(drop=True)
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 return df, test_df
df, test_df = get_df()
bs = 16
sz = 380
matrix.to_pickle('data.pkl')
 del matrix
del cache
del group
del items
del shops
del cats
del train
gc.collect() ;<load_pretrained>
from fastai import *
from fastai.vision import *
from torchvision.models import *
import torch
import torch.optim as optim
from fastai.vision.models import *
from fastai.vision.learner import model_meta
from fastai.callbacks import *
data = pd.read_pickle('data.pkl' )<drop_column>
aptos19_stats =([0.42, 0.22, 0.075], [0.27, 0.15, 0.081])
 tsfm1 = get_transforms(do_flip=False, flip_vert=False, max_rotate=0., max_zoom=0., max_warp=0.0, max_lighting=0.,p_affine=0.)
data =(ImageList.from_df(df=df,path='./',cols='path')
.split_by_rand_pct(0.1)
.label_from_df(cols='diagnosis',label_cls=FloatList)
.transform(tsfm1,size=380,)
.databunch(bs=bs,num_workers=4)
.normalize(imagenet_stats)
 )
data = data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'item_cnt_month_lag_6',
'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3',
'date_item_avg_item_cnt_lag_6',
'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2',
'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6',
'date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month',
'days',
'item_shop_last_sale',
'item_last_sale',
'item_shop_first_sale',
'item_first_sale',
]]<prepare_x_and_y>
data.show_batch(3,(5,5))
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )<set_options>
md_ef = EfficientNet.from_pretrained('efficientnet-b5', num_classes=1 )
del data
gc.collect() ;<train_model>
def quadratic_kappa(y_hat, y):
return torch.tensor(cohen_kappa_score(torch.round(y_hat), y, weights='quadratic'),device='cuda:0' )
ts = time.time()
model = XGBRegressor(
 max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 10)
 time.time() - ts<save_to_csv>
learn = Learner(data,
md_ef,
metrics = [quadratic_kappa]
,callback_fns=[BnFreeze,partial(SaveModelCallback, monitor='quadratic_kappa', name='best_accuracy')] ).to_fp16()
learn.data.add_test(ImageList.from_df(test_df,
'.. /input/aptos2019-blindness-detection',
folder='test_images',
suffix='.png'))
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('xgb_test.pickle', 'wb'))<import_modules>
learn.model_dir="/kaggle/working/models"
print(os.listdir(".. /input"))
print(lgb.__name__, lgb.__version__)
 sns.set() 
<load_from_csv>
learn.load('modelk');
df_shop = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv')
 df_item_categories = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv')
 df_items = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 df_sales_train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 df_test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv')
 df_sample_submission = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sample_submission.csv')
<define_variables>
learn.freeze_to(-1 )
show_EDA = False<categorify>
learn.fit_one_cycle(5,3e-04 )
if show_EDA:
df_items.head()<count_unique_values>
learn.load('best_accuracy')
 learn.save('stage1' )
if show_EDA:
print(df_data[df_data['item_cnt_month'] !=0].item_cnt_month.value_counts())
 print('training unique(id)with zero:')
 print('shop: ', df_data[df_data['date_block_num'] <= 33].shop_id.nunique())
 print('item: ', df_data[df_data['date_block_num'] <= 33].item_id.nunique())
 print('training unique(id)w/o zero:')
 print('shop: ', df_data[(df_data['date_block_num'] <= 33)&(df_data['item_cnt_month'] !=0)].shop_id.nunique())
 print('item: ', df_data[(df_data['date_block_num'] <= 33)&(df_data['item_cnt_month'] !=0)].item_id.nunique())
 print('testing unique(id):')
 print('shop: ', df_data[df_data['date_block_num'] == 34].shop_id.nunique())
 print('item: ', df_data[df_data['date_block_num'] == 34].item_id.nunique() )<categorify>
learn.fit_one_cycle(10, 7.59E-07 )
if show_EDA:
for shop, item in zip(df_test.shop_id[:10], df_test.item_id[:10]):
print(df_data[(df_data['shop_id'] == shop)&(df_data['item_id'] == item)&(df_data['date_block_num'] < 34)])
 print('next===========================================================')
 for shop, item in zip(df_test.shop_id[10:20], df_test.item_id[10:20]):
print('combination accounted for {0:.4%} of sample in the data set'.format(df_data[ 
(df_data['shop_id'] == shop)& 
(df_data['item_id'] == item)& 
(df_data['date_block_num'] < 34)& 
(df_data['item_cnt_month'] != 0)].shape[0] /
df_data[(df_data['date_block_num'] < 34)& 
(df_data['item_cnt_month'] != 0)].shape[0])) 
<categorify>
import scipy as sp
from sklearn import metrics
splits = df_shop['shop_name'].str.split(' ', expand=True)
 df_shop['pre_shop'] = splits[0]
df_shop['pre_shop'] = df_shop['pre_shop'].astype('category')
 df_shop['pre_shop_encode'] = df_shop['pre_shop'].cat.codes
df_shop.head()<data_type_conversions>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
splits = df_item_categories['item_category_name'].str.split(' - ', expand=True)
 df_item_categories['pre_cat'], df_item_categories['post_cat'] = splits[0], splits[1]
df_item_categories['pre_cat'] = df_item_categories['pre_cat'].fillna('unknown')
 df_item_categories['post_cat'] = df_item_categories['post_cat'].fillna('unknown')
 df_item_categories['pre_cat'] = df_item_categories['pre_cat'].astype('category')
 df_item_categories['post_cat'] = df_item_categories['post_cat'].astype('category')
 df_item_categories['pre_cat_encode'] = df_item_categories['pre_cat'].cat.codes
df_item_categories['post_cat_encode'] = df_item_categories['post_cat'].cat.codes
df_item_categories.head() 
<categorify>
learn.load('best_accuracy');
df_data['item_cnt_month'] = df_data['item_cnt_month'].clip(0, 20)
 df_data = create_aggregation(df_data, ['shop_id', 'date_block_num'], 'mean', 'item_cnt_day', 'shop_total_item_cnt_month')
 df_data = create_aggregation(df_data, ['item_id', 'date_block_num'], 'mean', 'item_cnt_day', 'total_item_cnt_month')
 df_data = create_aggregation(df_data, ['shop_id', 'item_id', 'date_block_num'], 'mean', 'item_price', 'mean_item_price')
 df_data = create_aggregation(df_data, ['date_block_num'], pd.Series.nunique, 'date', 'days_in_month')
 df_data = pd.merge(df_data, df_items.loc[:, ['item_id', 'item_category_id']], how='left', on='item_id')
 df_data = pd.merge(df_data, df_item_categories.loc[:, ['item_category_id', 'pre_cat_encode', 'post_cat_encode']], how='left', on='item_category_id')
 df_data = pd.merge(df_data, df_shop.loc[:, ['shop_id', 'pre_shop_encode']], how='left', on='shop_id')
 df_sales_train = pd.merge(df_sales_train, df_items.loc[:, ['item_id', 'item_category_id']], how='left', on='item_id')
 df_sales_train = pd.merge(df_sales_train, df_item_categories.loc[:, ['item_category_id', 'pre_cat_encode', 'post_cat_encode']], how='left', on='item_category_id')
 df_sales_train = pd.merge(df_sales_train, df_shop.loc[:, ['shop_id', 'pre_shop_encode']], how='left', on='shop_id')
 df_data = create_aggregation(df_data, ['pre_cat_encode', 'date_block_num'], 'mean', 'item_cnt_day', 'pre_cat_total_item_cnt_month')
 df_data = create_aggregation(df_data, ['post_cat_encode', 'date_block_num'], 'mean', 'item_cnt_day', 'post_cat_total_item_cnt_month')
 df_data = create_aggregation(df_data, ['pre_shop_encode', 'date_block_num'], 'mean', 'item_cnt_day', 'pre_shop_total_item_cnt_month')
 df_data = create_aggregation(df_data, ['item_category_id', 'date_block_num'], 'mean', 'item_cnt_day', 'item_cat_total_item_cnt_month')
 del df_sales_train
gc.collect()
current_feature = list(df_data.columns.difference(['shop_id', 'item_id', 'date_block_num']))
print(current_feature)
 df_data.head()<data_type_conversions>
valid_preds = learn.get_preds(ds_type=DatasetType.Valid )
col_int = ['shop_id', 'item_id', 'date_block_num', 'item_category_id', 'pre_cat_encode', 'post_cat_encode', 'pre_shop_encode']
col_float = list(df_data.columns.difference(col_int))
df_data.loc[:, col_int] = df_data.loc[:, col_int].astype(np.int16)
 df_data.loc[:, col_float] = df_data.loc[:, col_float].astype(np.float16)
 df_data.info()<feature_engineering>
optR = OptimizedRounder()
optR.fit(valid_preds[0],valid_preds[1])
 coefficients = optR.coefficients()
coefficients=np.around(( coefficients), decimals=2)
 print(coefficients )
def create_lag(to_merge, lag_range, col_target):
col_index = ['shop_id', 'item_id', 'date_block_num']
for month_shift in lag_range:
print('now running for lag {}'.format(month_shift))
temp_df = to_merge[col_index + col_target].copy()
gc.collect()
temp_df['date_block_num'] = temp_df['date_block_num'] + month_shift
naming_lag_col = lambda x: '{}_lag_{}'.format(x, month_shift)if x in col_target else x
temp_df = temp_df.rename(columns=naming_lag_col)
 col_to_float = list(temp_df.columns.difference(col_index))
temp_df.loc[:, col_to_float] = temp_df.loc[:, col_to_float].astype(np.float16)
 to_merge = pd.merge(to_merge, temp_df, how='left', on=col_index ).fillna(0)
 gc.collect()
print('completed!')
 return to_merge 
<feature_engineering>
preds,y = learn.get_preds(DatasetType.Test )
def create_diff(lag_range, col_target):
for col in col_target:
print('running diff for ', col, '...')
 for i in lag_range:
name = col + '_' + str(i)+ '-' + str(i + 1)
 if i == 0:
after = col
else:
after = col + '_' + 'lag_' + str(i)
 before = col + '_' + 'lag_' + str(i + 1)
 df_data[name] = df_data[after] - df_data[before]
print('completed!')
<load_pretrained>
tst_pred = optR.predict(preds, coefficients)
 test_df.diagnosis = tst_pred.astype(int)
 test_df.to_csv('submission.csv',index=False )
try:
os.remove('df_data.pickle')
 print('old file removed successfully')
 except:
print('file does not exist, just moving on...')
 with open('df_data.pickle', 'wb')as handle:
pickle.dump(df_data, handle )<load_pretrained>
test_df['diagnosis'].value_counts()
with open('df_data.pickle', 'rb')as handle:
df_data = pickle.load(handle)
 gc.collect()
full_feature = df_data.columns
del df_data
gc.collect()
len(gc.get_objects() )<drop_column>
data.show_batch()
ID_and_Category = ['item_id', 'shop_id', 'item_category_id','pre_cat_encode','post_cat_encode','pre_shop_encode']
current_feature = ['item_cnt_month','shop_total_item_cnt_month','total_item_cnt_month','mean_item_price','pre_cat_total_item_cnt_month','post_cat_total_item_cnt_month',
'pre_shop_total_item_cant_month','item_cat_total_item_cnt_month']
diff_feature = [col + '_1-2' for col in col_target]
drop_current_col = current_feature + diff_feature<load_from_csv>
from fastai.vision import *
<features_selection><EOS> 
vision.data.open_image=temp