Kaggle competition: VietAI Advance Course - Retinal Disease Detection
print('The number of unique shop ids is', df_test['shop_id'].nunique())
 print('The number of unique item ids is', df_test['item_id'].nunique())
 print('The average number of item ids per shop is', df_test.groupby('shop_id' ).agg(count=('item_id', 'count')).mean() [0] )<merge>
tf.enable_eager_execution()
tf.executing_eagerly()
train_combos = df_sales_train.drop_duplicates(['shop_id','item_id'] ).shape[0]
test_combos = df_test.drop_duplicates(['shop_id','item_id'] ).shape[0]
seen_combos = pd.merge(df_sales_train.drop_duplicates(['shop_id','item_id']), df_test, on=['shop_id','item_id'], how='inner' ).shape[0]
print("The number of train combos is {} and the number of test combos is {}".format(train_combos, test_combos))
print("The number of combos in test that are present in train is {}".format(seen_combos))<merge>
for label in data.columns[1:]:
print("Distribution of", label)
 print(data[label].value_counts() )
df_test = pd.merge(df_test, df_items[['item_id','item_category_id']], on=['item_id'], how='left')
 df_sales_train = pd.merge(df_sales_train, df_items[['item_id','item_category_id']], on=['item_id'], how='left')
 unseen_item_categories = set(df_test['item_category_id'])- set(df_sales_train['item_category_id'])
 print('The number of item categories that have never been sold before is', len(unseen_item_categories))<define_variables>
LABELS = data.columns[1:]
def build_label(row):
return ",".join([LABELS[idx] for idx, val in enumerate(row[1:])if val == 1])
 data.apply(lambda x: build_label(x), axis=1 ).value_counts()
unseen_items = set(df_test['item_id'])- set(df_sales_train['item_id'])
 print('The number of items that have never been sold before is', len(unseen_items))<drop_column>
train_data, val_data = train_test_split(data, test_size=0.2, random_state=2019 )
del df_sales_timeseries, unseen_items, unseen_item_categories, df_item_price, df_item_category_price
gc.collect()<feature_engineering>
IMAGE_SIZE = 224
IMAGENET_MEAN = [0.485, 0.456, 0.406]
IMAGENET_STD = [0.229, 0.224, 0.225]
BATCH_SIZE = 64
LEARNING_RATE = 0.001
LEARNING_RATE_SCHEDULE_FACTOR = 0.1
LEARNING_RATE_SCHEDULE_PATIENCE = 5
MAX_EPOCHS = 100
df_sales_train.loc[df_sales_train.shop_id == 0, 'shop_id'] = 57
df_test.loc[df_test.shop_id == 0, 'shop_id'] = 57
df_sales_train.loc[df_sales_train.shop_id == 1, 'shop_id'] = 58
df_test.loc[df_test.shop_id == 1, 'shop_id'] = 58
df_sales_train.loc[df_sales_train.shop_id == 11, 'shop_id'] = 10
df_test.loc[df_test.shop_id == 11, 'shop_id'] = 10
df_sales_train.loc[df_sales_train.shop_id == 40, 'shop_id'] = 39
df_test.loc[df_test.shop_id == 40, 'shop_id'] = 39
df_shops_translated['shop_name'] = df_shops_translated['shop_name_translated'].apply(lambda x: x.lower() ).str.replace('[^\w\s]', '' ).str.replace('\d+','' ).str.strip()
df_shops_translated['shop_city'] = df_shops_translated['shop_name'].str.partition(' ')[0]
display(df_shops_translated[df_shops_translated['shop_id'].isin([0,57,1,58,11,10,40,39])])
 df_shops_translated['city_code'] = LabelEncoder().fit_transform(df_shops_translated['shop_city'])
 df_shops_translated = df_shops_translated[['shop_id','city_code']]
df_categories_translated['split'] = df_categories_translated['item_category_name_translated'].str.lower().str.split('-')
 df_categories_translated['type'] = df_categories_translated['split'].map(lambda x: x[0].strip())
 df_categories_translated['subtype'] = df_categories_translated['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 display(df_categories_translated.head())
 df_categories_translated['type_code'] = LabelEncoder().fit_transform(df_categories_translated['type'])
 df_categories_translated['subtype_code'] = LabelEncoder().fit_transform(df_categories_translated['subtype'])
 df_categories_translated = df_categories_translated[['item_category_id','type_code','subtype_code']]<merge>
def preprocessing_image(image):

 return image
df_prices_shop_item = df_sales_train[['shop_id','item_id','item_price']].drop_duplicates(subset=['shop_id','item_id'], keep='last')
 df_prices_item = df_sales_train[['item_id','item_price']].drop_duplicates(subset=['item_id'], keep='last')
 df_prices_category = df_sales_train.groupby('item_category_id' ).agg(item_price=('item_price','mean')).reset_index()
df_test = df_grid[df_grid['date_block_num']==34]
df_test = pd.merge(df_test.drop('item_price',axis=1), df_prices_shop_item, on=['shop_id','item_id'], how='left')
 item_price = pd.merge(df_test[pd.isnull(df_test['item_price'])], df_prices_item, on='item_id',
how='left' ).set_index(df_test[pd.isnull(df_test['item_price'])].index)['item_price_y']
df_test['item_price'] = df_test['item_price'].fillna(item_price)
 category_price = pd.merge(df_test[pd.isnull(df_test['item_price'])], df_prices_category, on='item_category_id',
how='left' ).set_index(df_test[pd.isnull(df_test['item_price'])].index)['item_price_y']
df_test['item_price'] = df_test['item_price'].fillna(category_price)
 df_test['item_price_bin'] = pd.cut(df_test['item_price'], bins=bins, labels=np.arange(1,8), include_lowest=True)
 df_grid = pd.concat([df_grid[df_grid['date_block_num']<34], df_test[df_grid.columns]], axis=0)
 df_grid['item_price_bin'] = df_grid['item_price_bin'].astype(int)
 del df_prices_shop_item, df_prices_item, df_prices_category, df_test, df_sales_train
gc.collect()<categorify>
train_datagen = keras.preprocessing.image.ImageDataGenerator(rescale=1./255,
featurewise_center=True,
featurewise_std_normalization=True,
preprocessing_function=preprocessing_image )
df_grid['date_item_mean'] = df_grid.groupby(['date_block_num','item_id'])['item_cnt_month'].transform('mean')
 df_grid['date_category_mean'] = df_grid.groupby(['date_block_num','item_category_id'])['item_cnt_month'].transform('mean')
 df_grid['date_item_shop_mean'] = df_grid.groupby(['date_block_num','item_id','shop_id'])['item_cnt_month'].transform('mean')
 df_grid['date_item_category_mean'] = df_grid.groupby(['date_block_num','item_id','item_category_id'])['item_cnt_month'].transform('mean')
 df_grid['date_shop_category_mean'] = df_grid.groupby(['date_block_num','shop_id','item_category_id'])['item_cnt_month'].transform('mean')
 df_grid['month'] = df_grid['date_block_num'] % 12
number_of_days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 df_grid['days'] = df_grid['month'].map(number_of_days)
 df_grid['months_since_item_sale'] = df_grid['date_block_num'] - df_grid.groupby('item_id')['date_block_num'].transform('min')
 df_grid['months_since_item_shop_sale'] = df_grid['date_block_num'] - df_grid.groupby(['item_id','shop_id'])['date_block_num'].transform('min' )<merge>
def build_label_list(row):
return [LABELS[idx] for idx, val in enumerate(row[1:])if val == 1]
train_data["label"] = train_data.apply(lambda x: build_label_list(x), axis=1)
 val_data["label"] = val_data.apply(lambda x: build_label_list(x), axis=1 )
df_grid['shop_item_revenue'] = df_grid['item_cnt_month']*df_grid['item_price']
df_revenue = df_grid.groupby(['shop_id','date_block_num'] ).agg(shop_revenue=('shop_item_revenue',sum)).reset_index()
df_grid = pd.merge(df_grid, df_revenue, on=['shop_id','date_block_num'], how='left')
 df_grid.drop('shop_item_revenue', axis=1, inplace=True)
 del df_revenue
gc.collect()<data_type_conversions>
train_gen = train_datagen.flow_from_dataframe(dataframe=train_data,
directory=".. /input/train/train",
x_col="filename",
y_col="label",
class_mode="categorical",
target_size=(IMAGE_SIZE,IMAGE_SIZE),
batch_size=BATCH_SIZE )
def downcast_dtypes(df):

 float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype == "int64"]
df[float_cols] = df[float_cols].astype(np.float32)
 df[int_cols] = df[int_cols].astype(np.int16)
 return df
shop_revenue_cols = [col for col in df_grid.columns if 'shop_revenue' in col]
cols_to_downcast = list(set(df_grid.columns)- set(shop_revenue_cols))
df_grid[shop_revenue_cols] = df_grid[shop_revenue_cols].astype(np.float32)
 df_grid[cols_to_downcast] = downcast_dtypes(df_grid[cols_to_downcast] )<merge>
val_gen = train_datagen.flow_from_dataframe(dataframe=val_data,
directory=".. /input/train/train",
x_col="filename",
y_col="label",
class_mode="categorical",
shuffle=False,
target_size=(IMAGE_SIZE,IMAGE_SIZE),
batch_size=BATCH_SIZE )
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 df.drop(col, axis=1, inplace=True)
 del shifted
return(df)
 mean_encoded = [col for col in df_grid.columns if 'mean' in col]
cols_to_lag = ['shop_revenue','item_price'] + mean_encoded
for col in cols_to_lag:
df_grid = lag_feature(df_grid, [1,2,3,6,12], col)
 print('Lagged values for {} have been calculated'.format(col))
df_grid = df_grid[df_grid['date_block_num']>=12]
lagged_cols = [col for col in df_grid.columns if 'lag' in col]
df_grid[lagged_cols] = df_grid[lagged_cols].fillna(0)
 df_grid.reset_index(drop=True, inplace=True)
 gc.collect()<categorify>
base_model = keras.applications.ResNet50(input_shape=(IMAGE_SIZE,IMAGE_SIZE,3),
include_top=False,
weights='imagenet')
 base_model.trainable = True
model = keras.Sequential([
base_model,
keras.layers.GlobalAveragePooling2D() ,
keras.layers.Dense(len(LABELS), activation='sigmoid')
 ])
 model.summary()
df_grid = pd.read_feather('/kaggle/input/data-prep/df_sales.feather')
 def downcast_dtypes(df):

 float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype == "int64"]
df[float_cols] = df[float_cols].astype(np.float16)
 df[int_cols] = df[int_cols].astype(np.int16)
 return df
shop_revenue_cols = [col for col in df_grid.columns if 'shop_revenue' in col]
cols_to_downcast = list(set(df_grid.columns)- set(shop_revenue_cols))
df_grid[shop_revenue_cols] = df_grid[shop_revenue_cols].astype(np.float32)
 df_grid[cols_to_downcast] = downcast_dtypes(df_grid[cols_to_downcast] )<prepare_x_and_y>
def f1(y_true, y_pred):
def recall(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):

 true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
train = df_grid[df_grid['date_block_num']<33]
valid = df_grid[df_grid['date_block_num']==33]
X_test = df_grid[df_grid['date_block_num']==34].drop('item_cnt_month', axis=1)
 X_train = train[train['item_cnt_month']>0].drop('item_cnt_month', axis=1)
 y_train = train[train['item_cnt_month']>0]['item_cnt_month']
X_valid = valid[valid['item_cnt_month']>0].drop('item_cnt_month', axis=1)
 y_valid = valid[valid['item_cnt_month']>0]['item_cnt_month']
df_classreg_valid = pd.DataFrame()
df_classreg_test = pd.DataFrame()
df_reg_valid = pd.DataFrame()
df_reg_test = pd.DataFrame()
del df_grid
gc.collect()<choose_model_class>
mcp = keras.callbacks.ModelCheckpoint("resnet50.h5", monitor="val_f1", save_best_only=True, save_weights_only=True, verbose=1,mode='max')
 rlr = keras.callbacks.ReduceLROnPlateau(monitor='val_f1', factor=LEARNING_RATE_SCHEDULE_FACTOR, mode='max', patience=LEARNING_RATE_SCHEDULE_PATIENCE, min_lr=1e-8, verbose=1)
 callbacks = [mcp, rlr]
regressor_params = {
'feature_fraction': 0.75,
'metric': 'rmse',
'nthread': 1,
'min_data_in_leaf': 2**7,
'bagging_fraction': 0.75,
'learning_rate': 0.03,
'objective': 'mse',
'bagging_seed': 2**7,
'num_leaves': 80,
'bagging_freq':1,
'verbose':0
}
lightgbm_train = lightgbm.Dataset(X_train, label=y_train)
 lightgbm_valid = lightgbm.Dataset(X_valid, label=y_valid)
 regressor = lightgbm.train(regressor_params, lightgbm_train, valid_sets=[lightgbm_train, lightgbm_valid], valid_names=['train','valid'], verbose_eval=10, num_boost_round=500, early_stopping_rounds=10)
 print('LightGBM model for regression has been built!')
 df_classreg_valid['lightgbm'] = regressor.predict(valid.drop('item_cnt_month', axis=1))
df_classreg_test['lightgbm'] = regressor.predict(X_test)
 lightgbm.plot_importance(regressor, figsize=(10,12))
regressor.save_model('lightgbm_classreg_regressor.txt', num_iteration=regressor.best_iteration)
 del lightgbm_train, lightgbm_valid, regressor
gc.collect()<train_model>
device = '/cpu:0' if tfe.num_gpus() == 0 else '/gpu:0'
with tf.device(device):
steps_per_epoch = train_gen.n // BATCH_SIZE
validation_steps = val_gen.n // BATCH_SIZE
model.compile(optimizer=keras.optimizers.Adam(lr=LEARNING_RATE), loss='binary_crossentropy', metrics=[f1])
 history = model.fit_generator(train_gen,
steps_per_epoch=steps_per_epoch,
epochs= MAX_EPOCHS,
verbose=1,
validation_data=val_gen,
validation_steps=validation_steps,
callbacks=callbacks )
cat_features = ['shop_id','item_id','item_category_id','item_price_bin','month','date_block_num','city_code','type_code','subtype_code']
regressor = CatBoostRegressor(
 iterations=100,
random_seed=0,
learning_rate=0.1,
max_ctr_complexity=3,
has_time=True,
boosting_type='Ordered',
loss_function='RMSE',
od_type='Iter',
od_wait=10, 
)
 regressor.fit(
 X_train, y_train,
cat_features=cat_features,
eval_set=(X_valid, y_valid),
verbose=10 
)
 print('CatBoost model for regression has been built!')
 df_classreg_valid['catboost'] = regressor.predict(valid.drop(['item_cnt_month'], axis=1))
df_classreg_test['catboost'] = regressor.predict(X_test)
 print(regressor.get_feature_importance(prettified=True))
regressor.save_model('catboost_classreg_regressor.bin')
 del regressor
gc.collect()<choose_model_class>
test_df = pd.read_csv(".. /input/sample_submission.csv")
 test_df.head()
classifier_params = {
'feature_fraction': 0.75,
'metric': 'auc',
'nthread': 1,
'min_data_in_leaf': 2**7,
'bagging_fraction': 0.75,
'learning_rate': 0.3,
'objective': 'binary',
'bagging_seed': 2**7,
'num_leaves': 80,
'bagging_freq':1,
'is_unbalance':True,
'verbose':0
}
lightgbm_train = lightgbm.Dataset(train.drop('item_cnt_month', axis=1), label=train['item_cnt_month'].apply(lambda x: 1 if x >= 1 else 0))
lightgbm_valid = lightgbm.Dataset(valid.drop('item_cnt_month', axis=1), label=valid['item_cnt_month'].apply(lambda x: 1 if x >= 1 else 0))
classifier = lightgbm.train(classifier_params, lightgbm_train, valid_sets=[lightgbm_train, lightgbm_valid], valid_names=['train','valid'], verbose_eval=10, num_boost_round=300, early_stopping_rounds=10)
 print('Model for classification has been built!')
 df_classreg_valid['prob'] = classifier.predict(valid.drop('item_cnt_month', axis=1))
df_classreg_test['prob'] = classifier.predict(X_test)
 classifier.save_model('lightgbm_classreg_classifier.txt', num_iteration=classifier.best_iteration)
 del lightgbm_train, lightgbm_valid, classifier
gc.collect()<save_to_csv>
test_gen = train_datagen.flow_from_dataframe(dataframe=test_df,
directory=".. /input/test/test",
x_col="filename",
class_mode=None,
shuffle=False,
target_size=(IMAGE_SIZE,IMAGE_SIZE),
batch_size=BATCH_SIZE )
df_classreg_valid['average'] =(df_classreg_valid['lightgbm'] + df_classreg_valid['xgboost'] + df_classreg_valid['catboost'])/3
df_classreg_valid['item_cnt_month'] = df_classreg_valid.apply(lambda row: row['average'] if row['prob']>=0.8 else 0, axis=1)
 print('Validation RMSE is', np.sqrt(mean_squared_error(valid['item_cnt_month'], df_classreg_valid['item_cnt_month'])))
 df_classreg_valid.to_csv('classreg_valid.csv', index=False)
 df_classreg_test['average'] =(df_classreg_test['lightgbm'] + df_classreg_test['xgboost']+ df_classreg_test['catboost'])/3
df_classreg_test['item_cnt_month'] = df_classreg_test.apply(lambda row: row['average'] if row['prob']>=0.8 else 0, axis=1)
 df_classreg_test['ID'] = df_sample_submission['ID']
df_classreg_test = df_classreg_test[['ID','item_cnt_month']]
df_classreg_test.to_csv('classreg_test.csv', index=False )<load_from_csv>
model.load_weights("resnet50.h5" )
path = '/kaggle/input/competitive-data-science-predict-future-sales/sales_train.csv'
train_pairs = pd.read_csv(path)
 train_pairs = train_pairs[train_pairs['date_block_num']<33][['shop_id','item_id']].drop_duplicates(['shop_id','item_id'])
 pairs={(a, b)for a, b in zip(train_pairs.shop_id, train_pairs.item_id)}
items={a for a in train_pairs.item_id}
df_reg_valid['group'] = [2 if(a,b)in pairs else(1 if b in items else 0)for a,b in zip(valid.shop_id, valid.item_id)]
df_reg_test['group'] = [2 if(a,b)in pairs else(1 if b in items else 0)for a,b in zip(X_test.shop_id, X_test.item_id)]
del train_pairs, pairs, items
gc.collect()<train_model>
pred = model.predict_generator(test_gen )
lightgbm_params = {
'feature_fraction': 0.75,
'metric': 'rmse',
'nthread':1,
'min_data_in_leaf': 2**7,
'bagging_fraction': 0.75,
'learning_rate': 0.03,
'objective': 'mse',
'bagging_seed': 2**7,
'num_leaves': 80,
'bagging_freq':1,
'verbose':0
}
lightgbm_train = lightgbm.Dataset(train.drop('item_cnt_month', axis=1), label=train['item_cnt_month'])
 lightgbm_valid = lightgbm.Dataset(valid.drop('item_cnt_month', axis=1), label=valid['item_cnt_month'])
 model = lightgbm.train(lightgbm_params, lightgbm_train, valid_sets=[lightgbm_train, lightgbm_valid], valid_names=['train','valid'], verbose_eval=10, num_boost_round=300,
early_stopping_rounds=10)
 df_reg_valid['lightgbm'] = model.predict(valid.drop('item_cnt_month', axis=1))
df_reg_test['lightgbm'] = model.predict(X_test)
 model.save_model('lightgbm_reg.txt', num_iteration=model.best_iteration)
 del lightgbm_train, lightgbm_valid, model
gc.collect()<prepare_x_and_y>
for idx, row in test_df.iterrows() :
test_df.loc[idx]['predicted'] = probs2label(pred[idx] )
X_train_iloc = train.index[-1] + 1
X_valid_iloc = valid.index[-1] + 1
y_train = train['item_cnt_month']
y_valid = valid['item_cnt_month']
del train, valid
gc.collect()<normalization>
test_df.to_csv("submission.csv", index=False )