Kaggle competition: [Student] Shopee Code League - Product Detection
<SOS> metric: categorizationaccuracy Kaggle data source: [student]-shopee-code-league-product-detection<data_type_conversions>
import numpy as np
import pandas as pd
prior_question_elapsed_time_sum_dict = prior_question_elapsed_time_agg['sum'].astype('int64' ).to_dict(defaultdict(int))
prior_question_elapsed_time_count_dict = prior_question_elapsed_time_agg['count'].astype('int32' ).to_dict(defaultdict(int))<split>
!pip install "torch==1.4" "torchvision==0.5.0"
torch.__version__
env = riiideducation.make_env()
iter_test = env.iter_test()
prior_test_df = None<normalization>
from fastai.vision import *
from fastai.metrics import error_rate
def clip(count): return np.clip(count, 1e-8, np.inf )<feature_engineering>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
for(test_df, sample_prediction_df)in iter_test:
if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
prior_question_elapsed_times = prior_test_df['prior_question_elapsed_time'].values
targets = prior_test_df[target].values
for user_id, content_id, prior_question_elapsed_time, answered_correctly in zip(user_ids, content_ids, prior_question_elapsed_times, targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
content_sum_dict[content_id] += answered_correctly
content_count_dict[content_id] += 1
mean_accuracy = content_sum_dict[content_id] / clip(content_count_dict[content_id])
 residual_sum_dict[user_id] += answered_correctly - mean_accuracy
prior_question_elapsed_time_sum_dict[user_id] += 0 if np.isnan(prior_question_elapsed_time)else prior_question_elapsed_time
prior_question_elapsed_time_count_dict[user_id] += 0 if np.isnan(prior_question_elapsed_time)else 1
prior_test_df = test_df.copy()
test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(False ).astype('uint8')
 user_sum = np.zeros(len(test_df), dtype=np.int16)
 user_count = np.zeros(len(test_df), dtype=np.int16)
 res_sum = np.zeros(len(test_df), dtype=np.float32)
 content_sum = np.zeros(len(test_df), dtype=np.int32)
 content_count = np.zeros(len(test_df), dtype=np.int32)
 prior_question_elapsed_time_sum = np.zeros(len(test_df), dtype=np.int32)
 prior_question_elapsed_time_count = np.zeros(len(test_df), dtype=np.int32)
 for i,(user_id, content_id)in enumerate(zip(test_df['user_id'].values, test_df['content_id'].values)) :
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
res_sum[i] = residual_sum_dict[user_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
prior_question_elapsed_time_sum[i] = prior_question_elapsed_time_sum_dict[user_id]
prior_question_elapsed_time_count[i] = prior_question_elapsed_time_count_dict[user_id]
content_count = clip(content_count)
 user_count = clip(user_count)
 prior_question_elapsed_time_count = clip(prior_question_elapsed_time_count)
 test_df['user_correctness'] = user_sum / user_count
test_df['residual_user_mean'] = res_sum / user_count
test_df['content_count'] = content_count
test_df['content_id'] = content_sum / content_count
test_df['prior_question_elapsed_time_mean'] = prior_question_elapsed_time_sum / prior_question_elapsed_time_count
test_df['prior_question_elapsed_time'].fillna(train_df['prior_question_elapsed_time'].mean() , inplace=True)
 test_df[cat_features] = test_df[cat_features].apply(pd.to_numeric, downcast='integer')
 test_df.fillna(0, inplace=True)
 test_df[target] = np.average([
clf_tabnet.predict_proba(test_df[features].values)[:,1],
lin_model.predict(test_df[features]),
learn.predict_batch(learn, test_df[features])[:,0],
lgb_model.predict(test_df[features])
 ], weights=[0.25, 0.2, 0.25, 0.3], axis=0)
 env.predict(test_df[['row_id', target]] )<predict_on_test>
copyfile(src = ".. /input/cutmix/shared/0_image_data_augmentation/exp/nb_new_data_augmentation.py" ,
dst = ".. /working/nb_new_data_augmentation.py" )
test_df[target] = np.average([
clf_tabnet.predict_proba(test_df[features].values)[:,1],
lin_model.predict(test_df[features]),
learn.predict_batch(learn, test_df[features])[:,0],
lgb_model.predict(test_df[features])
 ], weights=[0.25, 0.2, 0.25, 0.3], axis=0 )<predict_on_test>
from nb_new_data_augmentation import *
lgb_model.predict(test_df[features] )<load_from_csv>
path = Path('.. /input/shopee-product-detection-student' )
full_train = pd.read_pickle(".. /input/riiid-train-data-multiple-formats/riiid_train.pkl.gzip")
 questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv')
 lectures = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )<data_type_conversions>
N = 1000
df = pd.read_csv('.. /input/shopee-product-detection-student/train.csv')
 df = df.groupby('category' ).apply(lambda x: x[:N] ).reset_index(drop=True)
 df['filename'] = 'train/' + df['category'].apply(lambda x: '{0:0>2}'.format(x)).apply(str)+'/'+df['filename']
<data_type_conversions>
df['category'] = df['category'].replace({0: '00 Dress',1: '01 Sarung',2: '02 Shirt(Top)',3: '03 Long Sleeves/Hoodie',4: '04 Jeans(Female)',5: '05 Ring',6: '06 Ear Rings',7: '07 Cap',8: '08 Purse/Wallet',9: '09 Bags',10: '10 Phone Cover',11: '11 Phone',12: '12 Clock',13: '13 Plastic Baby Bottles',14: '14 Rice Cooker',15: '15 Coffee',16: '16 Shoe',17: '17 High Heels',18: '18 Aircon',19: '19 Pendrive',20: '20 Chair',21: '21 Racket',22: '22 Helmet',23: '23 Gloves',24: '24 Watch',25: '25 Belt',26: '26 Headphones/Earpiece',27: '27 Toy Car',28: '28 Suit/Jacket(Male)',29: '29 Tuxedo Pants(Male)',30: '30 Sport Shoes',31: '31 Biscuit/ Junk Food',32: '32 Face Mask',33: '33 Sanitizer/Antiseptic',34: '34 Skin Care(?)',35: '35 Perfume/Cologne',36: '36 Cleaning Supplies',37: '37 Laptop',38: '38 Bowls',39: '39 Vases?',40: '40 Shower Stuff(Showerhead)',41: '41 Sofa',})
 df
<data_type_conversions>
np.random.seed(0)
 data = ImageDataBunch.from_df(path, df, folder='/train/train',valid_pct=0.2,
test='test/test/test',
ds_tfms=get_transforms() ,
size=224, bs=64
).normalize(imagenet_stats )
<data_type_conversions>
data.show_batch(rows=3, figsize=(10, 5))
<categorify>
learn = cnn_learner(data, models.resnet50,
metrics=[error_rate, accuracy]
).cutmix().show_multi_img_tfms()
<feature_engineering>
learn.model_dir = '/kaggle/working/'
<merge>
learn.lr_find()
learn.recorder.plot(suggestion=True )
<filter>
learn.fit_one_cycle(8, max_lr=slice(2e-3))
<drop_column>
learn.fit_one_cycle(8, max_lr=slice(1e-5 , 4e-4))
full_train = full_train[['row_id','user_id','content_id','content_type_id','answered_correctly']]
train = full_train.groupby('user_id' ).tail(500)
 test = full_train.groupby('user_id' ).tail(4)
 train = train.drop(test.index )<merge>
interp = ClassificationInterpretation.from_learner(learn )
question_average = pd.DataFrame(full_train.loc[full_train['content_type_id'] == 0].groupby(['content_id'])['answered_correctly'].mean() ).rename(columns={'answered_correctly':'question_average'})
 question_count = pd.DataFrame(full_train.loc[full_train['content_type_id'] == 0].groupby(['content_id'] ).size() ,columns=['question_count'])
 question_df = question_average.join(question_count)
 question_df['question_sum'] = question_df['question_average'] * question_df['question_count']
question_df = question_df.join(questions,how='outer')[['question_average','question_count','question_sum']]
question_df['question_average'].fillna(0,inplace=True)
 question_df['question_count'].fillna(0,inplace=True)
 question_df['question_sum'].fillna(0,inplace=True)
 del question_average,question_count
gc.collect()<drop_column>
interp.most_confused(min_val=40 )
del full_train
gc.collect()<merge>
preds, y = learn.TTA(ds_type=DatasetType.Test )
train = train.join(question_df,on=['content_id'], rsuffix='_question' )<groupby>
category = [data.classes[int(x)][:2] for x in np.argmax(preds, 1)]
mean_user = train.loc[train['content_type_id'] == False].groupby(['user_id'])['answered_correctly'].mean().mean()
mean_question = train.loc[train['content_type_id'] == False].groupby(['content_id'])['answered_correctly'].mean().mean()<feature_engineering>
filename = []
num = len(learn.data.test_ds)
 for i in range(num):
filename.append(str(learn.data.test_ds.items[i] ).split('/')[-1] )
train['user_shift_question'] = train.loc[train['content_type_id'] == False].groupby(['user_id'])['question_average'].shift()
cumulated_question = train.loc[train['content_type_id'] == False].groupby(['user_id'])['user_shift_question'].agg(['cumsum','cumcount'])
 train.loc[train['content_type_id'] == False,'average_past_questions'] = cumulated_question['cumsum'] / cumulated_question['cumcount']
train['average_past_questions'].fillna(mean_question,inplace=True)
 train.drop(['user_shift_question','question_count','question_sum'],axis=1,inplace=True)
 del cumulated_question<feature_engineering>
submit = pd.DataFrame({'filename':filename,'category':category})
 test_csv = pd.read_csv('.. /input/shopee-product-detection-student/test.csv')
 submit = submit[submit['filename'].isin(test_csv['filename'])]
submit
<create_dataframe><EOS> 
submit.to_csv('submission.csv', header=True, index=False )