Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<data_type_conversions>
base_dir = ".. /input/aptos2019-blindness-detection/"
train_csv = base_dir+"train.csv"
test_csv = base_dir +"test.csv"
test_dir = base_dir + "test_images/"
test_dir_processed = 'test_dir_processed'
train_dir = "train_data_cropped"
IMG_SIZE = 224
SEED = 72
X_val['answered_correctly_user'].fillna(0.65, inplace=True)
 X_val['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X_val['quest_pct'].fillna(content_mean, inplace=True)
 X_val['part'].fillna(4, inplace = True)
 X_val['avg_questions_seen'].fillna(1, inplace = True)
 X_val['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X_val['prior_question_had_explanation_enc'].fillna(0, inplace = True)
 X_val['part_1'].fillna(0, inplace = True)
 X_val['part_2'].fillna(0, inplace = True)
 X_val['part_3'].fillna(0, inplace = True)
 X_val['part_4'].fillna(0, inplace = True)
 X_val['part_5'].fillna(0, inplace = True)
 X_val['part_6'].fillna(0, inplace = True)
 X_val['part_7'].fillna(0, inplace = True)
 X_val['type_of_concept'].fillna(0, inplace = True)
 X_val['type_of_intention'].fillna(0, inplace = True)
 X_val['type_of_solving_question'].fillna(0, inplace = True)
 X_val['type_of_starter'].fillna(0, inplace = True)
 X_val['part_1_boolean'].fillna(0, inplace = True)
 X_val['part_2_boolean'].fillna(0, inplace = True)
 X_val['part_3_boolean'].fillna(0, inplace = True)
 X_val['part_4_boolean'].fillna(0, inplace = True)
 X_val['part_5_boolean'].fillna(0, inplace = True)
 X_val['part_6_boolean'].fillna(0, inplace = True)
 X_val['part_7_boolean'].fillna(0, inplace = True)
 X_val['type_of_concept_boolean'].fillna(0, inplace = True)
 X_val['type_of_intention_boolean'].fillna(0, inplace = True)
 X_val['type_of_solving_question_boolean'].fillna(0, inplace = True)
 X_val['type_of_starter_boolean'].fillna(0, inplace = True )<train_on_grid>
params = {
'num_leaves': 31,
'n_estimators': 200,
'max_depth': 8,
'min_child_samples': 356,
'learning_rate': 0.2982483634778906,
'min_data_in_leaf': 82,
'bagging_fraction': 0.6545628633239445,
'feature_fraction': 0.9164482379289846,
'random_state': 666
}
full_model = LGBMClassifier(**params)
 full_model.fit(X, y)
 preds = full_model.predict_proba(X_val)[:,1]
print('LGB roc auc', roc_auc_score(y_val, preds))
full_xgb = XGBClassifier(random_state=666)
 full_xgb.fit(X, y)
 preds = full_xgb.predict_proba(X_val)[:,1]
print('XGB roc auc', roc_auc_score(y_val, preds))
full_lr = LogisticRegression(random_state=666)
 full_lr.fit(X, y)
 preds = full_lr.predict_proba(X_val)[:,1]
print('LR roc auc', roc_auc_score(y_val, preds))<import_modules>
import optuna
from optuna.samplers import TPESampler<normalization>
import cv2
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.utils import class_weight, shuffle
rfe = RFE(estimator=DecisionTreeClassifier(random_state=666), n_features_to_select=14)
 rfe.fit(X, y)
 X = rfe.transform(X)
 X_val = rfe.transform(X_val )<choose_model_class>
image = cv2.imread(base_dir+'train_images/295fdc964f6e.png')
 plt.imshow(image )
sampler = TPESampler(seed=666)
 def create_model(trial):
num_leaves = trial.suggest_int("num_leaves", 2, 31)
 n_estimators = trial.suggest_int("n_estimators", 20, 300)
 max_depth = trial.suggest_int('max_depth', 3, 9)
 min_child_samples = trial.suggest_int('min_child_samples', 100, 1200)
 learning_rate = trial.suggest_uniform('learning_rate', 0.0001, 0.99)
 min_data_in_leaf = trial.suggest_int('min_data_in_leaf', 5, 90)
 bagging_fraction = trial.suggest_uniform('bagging_fraction', 0.0001, 1.0)
 feature_fraction = trial.suggest_uniform('feature_fraction', 0.0001, 1.0)
 model = LGBMClassifier(
 num_leaves=num_leaves,
n_estimators=n_estimators,
max_depth=max_depth,
min_child_samples=min_child_samples,
min_data_in_leaf=min_data_in_leaf,
learning_rate=learning_rate,
feature_fraction=feature_fraction,
random_state=666 
)
 return model
def objective(trial):
model = create_model(trial)
 model.fit(X, y)
 preds = model.predict_proba(X_val)[:,1]
score = roc_auc_score(y_val, preds)
 return score
params = {
'num_leaves': 28,
'n_estimators': 295,
'max_depth': 8,
'min_child_samples': 1178,
'learning_rate': 0.2379173491475032,
'min_data_in_leaf': 35,
'bagging_fraction': 0.8389723511600549,
'feature_fraction': 0.9606189400533491,
'random_state': 666
}
model = LGBMClassifier(**params)
 model.fit(X, y)
 preds = model.predict_proba(X_val)[:,1]
roc_auc_score(y_val, preds )<create_dataframe>
i = 0
for fileName in os.listdir("train_data_cropped/"):
i = i + 1
print(i )
X = pd.DataFrame(X)
 X_val = pd.DataFrame(X_val)
 y = pd.DataFrame(y)
 y_val = pd.DataFrame(y_val )<choose_model_class>
i = 0
for fileName in os.listdir("test_dir_processed/"):
i = i + 1
print(i )
models = []
preds = []
for n,(tr, te)in enumerate(KFold(n_splits=5, random_state=666, shuffle=True ).split(y)) :
print(f'Fold {n}')
 model = LGBMClassifier(**params)
 model.fit(X.values[tr], y.values[tr])
 pred = model.predict_proba(X_val)[:, 1]
preds.append(pred)
 print('Fold roc auc:', roc_auc_score(y.values[te], model.predict_proba(X.values[te])[:, 1]))
models.append(model )<compute_test_metric>
image1 = cv2.imread('.. /input/aptos2019-blindness-detection/train_images/295fdc964f6e.png')
 image1 = cv2.cvtColor(image1, cv2.COLOR_BGR2RGB)
 plt.imshow(image1 )
predictions = preds[0]
for i in range(1, 5):
predictions += preds[i]
predictions /= 5
print('ROC AUC', roc_auc_score(y_val, predictions))<split>
WORKERS = 2
CHANNEL = 3
warnings.filterwarnings("ignore")
 NUM_CLASSES = 5
SEED = 72
TRAIN_NUM = 1000
iter_test = env.iter_test()<merge>
def df_train_test_split_preprocess(df):
image_ids = df["id_code"].values.tolist()
labels = df["diagnosis"].values.tolist()
for i in range(len(image_ids)) :
imgname = image_ids[i]
newname = str(imgname)+ ".png"
image_ids[i] = newname
xtrain, xval, ytrain, yval = train_test_split(image_ids, labels, test_size = 0.15)
 df_train = pd.DataFrame({"id_code":xtrain, "diagnosis":ytrain})
 df_val = pd.DataFrame({"id_code":xval, "diagnosis":yval})
 df_train["diagnosis"] = df_train["diagnosis"].astype('str')
 df_val["diagnosis"] = df_val["diagnosis"].astype('str')
 print("Length of Training Data :",len(df_train))
print("Length of Validation Data :",len(df_val))
return df_train, df_val
for(test_df, sample_prediction_df)in iter_test:
test_df['task_container_id'] = test_df.task_container_id.mask(test_df.task_container_id > 9999, 9999)
 test_df = pd.merge(test_df, group3, left_on=['task_container_id'], right_index= True, how="left")
 test_df = pd.merge(test_df, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df = pd.merge(test_df, results_u_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_u2_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, user_lecture_stats_part, on=['user_id'], how="left")
 test_df['part_1'].fillna(0, inplace = True)
 test_df['part_2'].fillna(0, inplace = True)
 test_df['part_3'].fillna(0, inplace = True)
 test_df['part_4'].fillna(0, inplace = True)
 test_df['part_5'].fillna(0, inplace = True)
 test_df['part_6'].fillna(0, inplace = True)
 test_df['part_7'].fillna(0, inplace = True)
 test_df['type_of_concept'].fillna(0, inplace = True)
 test_df['type_of_intention'].fillna(0, inplace = True)
 test_df['type_of_solving_question'].fillna(0, inplace = True)
 test_df['type_of_starter'].fillna(0, inplace = True)
 test_df['part_1_boolean'].fillna(0, inplace = True)
 test_df['part_2_boolean'].fillna(0, inplace = True)
 test_df['part_3_boolean'].fillna(0, inplace = True)
 test_df['part_4_boolean'].fillna(0, inplace = True)
 test_df['part_5_boolean'].fillna(0, inplace = True)
 test_df['part_6_boolean'].fillna(0, inplace = True)
 test_df['part_7_boolean'].fillna(0, inplace = True)
 test_df['type_of_concept_boolean'].fillna(0, inplace = True)
 test_df['type_of_intention_boolean'].fillna(0, inplace = True)
 test_df['type_of_solving_question_boolean'].fillna(0, inplace = True)
 test_df['type_of_starter_boolean'].fillna(0, inplace = True)
 test_df['answered_correctly_user'].fillna(0.65, inplace=True)
 test_df['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 test_df['quest_pct'].fillna(content_mean, inplace=True)
 test_df['part'] = test_df.part - 1
test_df['part'].fillna(4, inplace = True)
 test_df['avg_questions_seen'].fillna(1, inplace = True)
 test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.fit_transform(test_df["prior_question_had_explanation"])
 full_preds = full_model.predict_proba(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']])[:, 1]
full_preds_xgb = full_xgb.predict_proba(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']])[:, 1]
full_preds_lr = full_lr.predict_proba(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']])[:, 1]
X_test = rfe.transform(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']])
 preds = [model.predict_proba(X_test)[:,1] for model in models]
predictions = preds[0]
for i in range(1, 5):
predictions += preds[i]
predictions /= 5
test_df['answered_correctly'] = predictions * 0.65 + full_preds * 0.15 + full_preds_xgb * 0.15 + full_preds_lr * 0.05
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<set_options>
df = pd.read_csv(".. /input/aptos2019-blindness-detection/train.csv")
 df_train, df_val = df_train_test_split_preprocess(df )
rcParams['figure.figsize'] = 12, 4
pd.set_option('display.max_columns',100)
 def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type=='object':
df[col] = df[col].astype('category')
 if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
def display_status(message):
display(message)
 clear_output(wait=True)
 def unify_duplicated_shop_id(ids):
global train,test
for pair in ids: 
(origin, replacement)= pair
display_status('replace {0} with {1}'.format(origin, replacement))
train.loc[train.shop_id == origin, "shop_id"] = replacement
test.loc[test.shop_id == origin , "shop_id"] = replacement
def clean_and_expand_shop_data() :
global shops
display_status('Cleaning Shop Data')
 shops.loc[ shops.shop_name == 'Сергиев Посад ТЦ "7Я"',"shop_name" ] = 'СергиевПосад ТЦ "7Я"'
shops["city"] = shops.shop_name.str.split(" " ).map(lambda x: x[0])
 shops["category"] = shops.shop_name.str.split(" " ).map(lambda x: x[1])
 shops.loc[shops.city == "!Якутск", "city"] = "Якутск"
def name_correction(x):
x = x.lower()
x = x.partition('[')[0]
x = x.partition('(')[0]
x = re.sub('[^A-Za-z0-9А-Яа-я]+', ' ', x)
 x = x.replace(' ', ' ')
 x = x.strip()
return x
def clean_text_and_recategorize() :
global shops,item_categories,items
category = []
for cat in shops.category.unique() :
print(cat, len(shops[shops.category == cat]))
if len(shops[shops.category == cat])> 4:
category.append(cat)
 shops.category = shops.category.apply(lambda x: x if(x in category)else "etc")
 shops["shop_category"] = LabelEncoder().fit_transform(shops.category)
 shops["shop_city"] = LabelEncoder().fit_transform(shops.city)
 shops = shops[["shop_id", "shop_category", "shop_city"]]
item_categories["type_code"] = item_categories.item_category_name.apply(lambda x: x.split(" ")[0] ).astype(str)
 item_categories.loc[(item_categories.type_code == "Игровые")|(item_categories.type_code == "Аксессуары"), "category" ] = "Игры"
category = []
for cat in item_categories.type_code.unique() :
print(cat, len(item_categories[item_categories.type_code == cat]))
if len(item_categories[item_categories.type_code == cat])> 4:
category.append(cat)
 item_categories.type_code = item_categories.type_code.apply(lambda x: x if(x in category)else "etc")
 for cat in item_categories.type_code.unique() :
print(cat, len(item_categories[item_categories.type_code == cat]))
item_categories.type_code = LabelEncoder().fit_transform(item_categories.type_code)
 item_categories["split"] = item_categories.item_category_name.apply(lambda x: x.split("-"))
item_categories["subtype"] = item_categories.split.apply(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 item_categories["subtype_code"] = LabelEncoder().fit_transform(item_categories["subtype"])
 item_categories = item_categories[["item_category_id", "subtype_code", "type_code"]]
items["name1"], items["name2"] = items.item_name.str.split("[", 1 ).str
items["name1"], items["name3"] = items.item_name.str.split("(", 1 ).str
items["name2"] = items.name2.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items["name3"] = items.name3.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items = items.fillna('0')
 items["item_name"] = items["item_name"].apply(lambda x: name_correction(x))
items.name2 = items.name2.apply(lambda x: x[:-1] if x !="0" else "0")
 items["type"] = items.name2.apply(lambda x: x[0:8] if x.split(" ")[0] == "xbox" else x.split(" ")[0])
 items.loc[(items.type == "x360")|(items.type == "xbox360")|(items.type == "xbox 360"),"type"] = "xbox 360"
items.loc[ items.type == "", "type"] = "mac"
items.type = items.type.apply(lambda x: x.replace(" ", ""))
items.loc[(items.type == 'pc')|(items.type == 'pс')|(items.type == "pc"), "type" ] = "pc"
items.loc[ items.type == 'рs3' , "type"] = "ps3"
populars = ['alien', 'adele', 'angry', 'anno','edition', 'ultimate', 'gold', 'premium', 'assassin', 'battlefield',
'call of duty', 'fifa', 'football','need for speed', 'pokemon',
'grand theft auto', 'fallout', 'star wars', 'uncharted','Kaspersky', 'Sims', 'medal of honor', 'Фирменный пакет майка',
'mad max', 'metal gear', 'ea', 'diablo', 'warcraft', 'Доставка','Ведьмак 3'
]
index = [False] * len(train)
 display_status('tagging popular items such as Star Wars')
 for word in tqdm(populars):
idx_popular_item_sales = items['item_name'].str.contains(word, case=False)
 items.loc[ idx_popular_item_sales , "type"] = word
group_sum = items.groupby(["type"] ).agg({"item_id": "count"})
 print(group_sum.reset_index())
 group_sum = group_sum.reset_index()
drop_cols = []
for cat in group_sum.type.unique() :
if group_sum.loc[(group_sum.type == cat), "item_id"].values[0] <40:
drop_cols.append(cat)
 items.name2 = items.name2.apply(lambda x: "etc" if(x in drop_cols)else x)
 items = items.drop(["type"], axis = 1)
 items.name2 = LabelEncoder().fit_transform(items.name2)
 items.name3 = LabelEncoder().fit_transform(items.name3)
 items.drop(["item_name", "name1"],axis = 1, inplace= True)
 items.head()
def lag_feature(df,lags, cols):
for col in cols:
print(col)
 tmp = df[["date_block_num", "shop_id","item_id",col ]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ["date_block_num", "shop_id", "item_id", col + "_lag_"+str(i)]
shifted.date_block_num = shifted.date_block_num + i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
basepath= '.. /input/competitive-data-science-predict-future-sales/'
items = pd.read_csv(basepath+"items.csv")
 item_categories = pd.read_csv(basepath+"item_categories.csv")
 shops = pd.read_csv(basepath+"shops.csv")
 train = pd.read_csv(basepath+"sales_train.csv")
 test = pd.read_csv(basepath+"test.csv")
 unify_duplicated_shop_id(ids=[(0,57),(1,58),(11,10),(40,39)])
 clean_and_expand_shop_data()
clean_text_and_recategorize()
ts = time.time()
agg_data = []
cols = ["date_block_num", "shop_id", "item_id"]
display_status('========| Create aggregated data(1/8)|========= ~Time elapsed: {0} secs'.format(str(time.time() - ts)))
 for i in range(34):
sales = train[train.date_block_num == i]
agg_data.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype = np.int16))
agg_data = pd.DataFrame(np.vstack(agg_data), columns = cols)
 agg_data["date_block_num"] = agg_data["date_block_num"].astype(np.int8)
 agg_data["shop_id"] = agg_data["shop_id"].astype(np.int8)
 agg_data["item_id"] = agg_data["item_id"].astype(np.int16)
 agg_data.sort_values(cols, inplace = True)
 train["revenue"] = train["item_cnt_day"] * train["item_price"]
display_status('========| Group Aggregate based on date, shop, item(2/8)|========= ~Time elapsed: {0} secs'.format(str(time.time() - ts)))
 group = train.groupby(["date_block_num", "shop_id", "item_id"] ).agg({"item_cnt_day": ["sum"]})
 group.columns = ["item_cnt_month"]
group.reset_index(inplace = True)
 agg_data = pd.merge(agg_data, group, on = cols, how = "left")
 agg_data["item_cnt_month"] = agg_data["item_cnt_month"].fillna(0 ).clip(0,20 ).astype(np.float16)
 test["date_block_num"] = 34
test["date_block_num"] = test["date_block_num"].astype(np.int8)
 test["shop_id"] = test.shop_id.astype(np.int8)
 test["item_id"] = test.item_id.astype(np.int16)
 agg_data = pd.concat([agg_data, test.drop(["ID"],axis = 1)], ignore_index=True, sort=False, keys=cols)
 agg_data.fillna(0, inplace = True)
 agg_data
display_status('========| Merge shops, items, categoriess(3/8)|========= ~Time elapsed: {0} secs'.format(str(time.time() - ts)))
 agg_data = pd.merge(agg_data, shops, on = ["shop_id"], how = "left")
 agg_data = pd.merge(agg_data, items, on = ["item_id"], how = "left")
 agg_data = pd.merge(agg_data, item_categories, on = ["item_category_id"], how = "left")
 agg_data["shop_city"] = agg_data["shop_city"].astype(np.int8)
 agg_data["shop_category"] = agg_data["shop_category"].astype(np.int8)
 agg_data["item_category_id"] = agg_data["item_category_id"].astype(np.int8)
 agg_data["subtype_code"] = agg_data["subtype_code"].astype(np.int8)
 agg_data["name2"] = agg_data["name2"].astype(np.int8)
 agg_data["name3"] = agg_data["name3"].astype(np.int16)
 agg_data["type_code"] = agg_data["type_code"].astype(np.int8)
 time.time() - ts
agg_data = lag_feature(agg_data, [1,2,3], ["item_cnt_month"])
 display_status('========| Group & merge based on date_block_num(4/8)|========= ~Time elapsed: {0} secs'.format(str(time.time() - ts)))
 group = agg_data.groupby(["date_block_num"] ).agg({"item_cnt_month" : ["mean"]})
 group.columns = ["date_avg_item_cnt"]
group.reset_index(inplace = True)
 agg_data = pd.merge(agg_data, group, on = ["date_block_num"], how = "left")
 agg_data.date_avg_item_cnt = agg_data["date_avg_item_cnt"].astype(np.float16)
 agg_data = lag_feature(agg_data, [1], ["date_avg_item_cnt"])
 agg_data.drop(["date_avg_item_cnt"], axis = 1, inplace = True)
 display_status('========| Group & merge based on [date_block_num, item_id](4/8)|========= ~Time elapsed: {0} secs'.format(str(time.time() - ts)))
 group = agg_data.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 agg_data = pd.merge(agg_data, group, on=['date_block_num','item_id'], how='left')
 agg_data.date_item_avg_item_cnt = agg_data['date_item_avg_item_cnt'].astype(np.float16)
 agg_data = lag_feature(agg_data, [1,2,3], ['date_item_avg_item_cnt'])
 agg_data.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 display_status('========| Group & merge based on [date_block_num , shop_id](5/8)|========= ~Time elapsed: {0} secs'.format(str(time.time() - ts)))
 group = agg_data.groupby(["date_block_num","shop_id"] ).agg({"item_cnt_month" : ["mean"]})
 group.columns = ["date_shop_avg_item_cnt"]
group.reset_index(inplace = True)
 agg_data = pd.merge(agg_data, group, on = ["date_block_num","shop_id"], how = "left")
 agg_data.date_avg_item_cnt = agg_data["date_shop_avg_item_cnt"].astype(np.float16)
 agg_data = lag_feature(agg_data, [1,2,3], ["date_shop_avg_item_cnt"])
 agg_data.drop(["date_shop_avg_item_cnt"], axis = 1, inplace = True)
 group = agg_data.groupby(["date_block_num","shop_id","item_id"] ).agg({"item_cnt_month" : ["mean"]})
 group.columns = ["date_shop_item_avg_item_cnt"]
group.reset_index(inplace = True)
 agg_data = pd.merge(agg_data, group, on = ["date_block_num","shop_id","item_id"], how = "left")
 agg_data.date_avg_item_cnt = agg_data["date_shop_item_avg_item_cnt"].astype(np.float16)
 agg_data = lag_feature(agg_data, [1,2,3], ["date_shop_item_avg_item_cnt"])
 agg_data.drop(["date_shop_item_avg_item_cnt"], axis = 1, inplace = True)
 group = agg_data.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 agg_data = pd.merge(agg_data, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 agg_data.date_shop_subtype_avg_item_cnt = agg_data['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 agg_data = lag_feature(agg_data, [1], ['date_shop_subtype_avg_item_cnt'])
 agg_data.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 group = agg_data.groupby(['date_block_num', 'shop_city'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_city_avg_item_cnt']
group.reset_index(inplace=True)
 agg_data = pd.merge(agg_data, group, on=['date_block_num', "shop_city"], how='left')
 agg_data.date_city_avg_item_cnt = agg_data['date_city_avg_item_cnt'].astype(np.float16)
 agg_data = lag_feature(agg_data, [1], ['date_city_avg_item_cnt'])
 agg_data.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 display_status('========| This will take long time(6/8)|========= ~Time elapsed: {0} secs'.format(str(time.time() - ts)))
 group = agg_data.groupby(['date_block_num', 'item_id', 'shop_city'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 agg_data = pd.merge(agg_data, group, on=['date_block_num', 'item_id', 'shop_city'], how='left')
 agg_data.date_item_city_avg_item_cnt = agg_data['date_item_city_avg_item_cnt'].astype(np.float16)
 agg_data = lag_feature(agg_data, [1], ['date_item_city_avg_item_cnt'])
 agg_data.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 group = train.groupby(["item_id"] ).agg({"item_price": ["mean"]})
 group.columns = ["item_avg_item_price"]
group.reset_index(inplace = True)
 agg_data = agg_data.merge(group, on = ["item_id"], how = "left")
 agg_data["item_avg_item_price"] = agg_data.item_avg_item_price.astype(np.float16)
 group = train.groupby(["date_block_num","item_id"] ).agg({"item_price": ["mean"]})
 group.columns = ["date_item_avg_item_price"]
group.reset_index(inplace = True)
 agg_data = agg_data.merge(group, on = ["date_block_num","item_id"], how = "left")
 agg_data["date_item_avg_item_price"] = agg_data.date_item_avg_item_price.astype(np.float16)
 lags = [1, 2, 3]
agg_data = lag_feature(agg_data, lags, ["date_item_avg_item_price"])
 for i in lags:
agg_data["delta_price_lag_" + str(i)] =(agg_data["date_item_avg_item_price_lag_" + str(i)]- agg_data["item_avg_item_price"])/ agg_data["item_avg_item_price"]
def select_trends(row):
for i in lags:
if row["delta_price_lag_" + str(i)]:
return row["delta_price_lag_" + str(i)]
return 0
agg_data["delta_price_lag"] = agg_data.apply(select_trends, axis = 1)
 agg_data["delta_price_lag"] = agg_data.delta_price_lag.astype(np.float16)
 agg_data["delta_price_lag"].fillna(0 ,inplace = True)
 features_to_drop = ["item_avg_item_price", "date_item_avg_item_price"]
for i in lags:
features_to_drop.append("date_item_avg_item_price_lag_" + str(i))
features_to_drop.append("delta_price_lag_" + str(i))
agg_data.drop(features_to_drop, axis = 1, inplace = True)
 display_status('========| This will take long time too(7/8)|========= ~Time elapsed: {0} secs'.format(str(time.time() - ts)))
 group = train.groupby(["date_block_num","shop_id"] ).agg({"revenue": ["sum"] })
 group.columns = ["date_shop_revenue"]
group.reset_index(inplace = True)
 agg_data = agg_data.merge(group , on = ["date_block_num", "shop_id"], how = "left")
 agg_data['date_shop_revenue'] = agg_data['date_shop_revenue'].astype(np.float32)
 group = group.groupby(["shop_id"] ).agg({ "date_block_num":["mean"] })
 group.columns = ["shop_avg_revenue"]
group.reset_index(inplace = True)
 agg_data = agg_data.merge(group, on = ["shop_id"], how = "left")
 agg_data["shop_avg_revenue"] = agg_data.shop_avg_revenue.astype(np.float32)
 agg_data["delta_revenue"] =(agg_data['date_shop_revenue'] - agg_data['shop_avg_revenue'])/ agg_data['shop_avg_revenue']
agg_data["delta_revenue"] = agg_data["delta_revenue"].astype(np.float32)
 agg_data = lag_feature(agg_data, [1], ["delta_revenue"])
 agg_data["delta_revenue_lag_1"] = agg_data["delta_revenue_lag_1"].astype(np.float32)
 agg_data.drop(["date_shop_revenue", "shop_avg_revenue", "delta_revenue"] ,axis = 1, inplace = True)
 display_status('========| Last(8/8)|========= ~Time elapsed: {0} secs'.format(str(time.time() - ts)))
 agg_data["month"] = agg_data["date_block_num"] % 12
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 agg_data["days"] = agg_data["month"].map(days ).astype(np.int8)
 agg_data["item_shop_first_sale"] = agg_data["date_block_num"] - agg_data.groupby(["item_id","shop_id"])["date_block_num"].transform('min')
 agg_data["item_first_sale"] = agg_data["date_block_num"] - agg_data.groupby(["item_id"])["date_block_num"].transform('min')
 agg_data = agg_data[agg_data["date_block_num"] > 3]
display_status('========| Completed(8/8)|========= ~Time elapsed: {0} secs'.format(str(time.time() - ts)))
 days = pd.Series([10,3,2,1,7,2, 1,0,2,1,1,2])
 agg_data["holiday_num"] = agg_data["month"].map(days ).astype(np.int8)
 days = pd.Series([3,2,1,7,2, 1,0,2,1,1,2,10])
 agg_data["next_month_holiday_num"] = agg_data["month"].map(days ).astype(np.int8)
 days = pd.Series(['winter','etc','spring','etc','may', 'summer','summer','summer','etc','winter','winter','winter'])
 agg_data["vacation_period"] = agg_data["month"].map(days ).astype('category')
 agg_data['vacation_period'] = agg_data.groupby('vacation_period')['item_cnt_month'].transform('mean')
 agg_data.to_csv('agg_data.csv',index=False,header=True)
 agg_data = reduce_mem_usage(agg_data)
 X_train = agg_data[agg_data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 y_train = agg_data[agg_data.date_block_num < 33]['item_cnt_month']
X_val = agg_data[agg_data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 y_val = agg_data[agg_data.date_block_num == 33]['item_cnt_month']
X_test = agg_data[agg_data.date_block_num == 34].drop(['item_cnt_month'], axis=1)
 del agg_data
gc.collect()
X_train.fillna(0,inplace=True)
 X_val.fillna(0,inplace=True)
 X_test.fillna(0,inplace=True )<train_model>
from sklearn.metrics import f1_score, fbeta_score, cohen_kappa_score
est = 100
max_dept = 10
xgb_filename = 'xgb_pred_'+str(est)+'_'+str(max_dept)
 xgb = XGBRegressor(
 max_depth=max_dept,
n_estimators=est,
min_child_weight=0.5,
colsample_bytree=0.8,
subsample=0.8,
eta=0.1,
seed=42)
 xgb.fit(
 X_train,
y_train,
eval_metric="rmse",
eval_set=[(X_train, y_train),(X_val, y_val)],
verbose=True,
early_stopping_rounds = 20)
 pickle.dump(xgb, open(xgb_filename+'.model', 'wb'))
y_test = xgb.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": y_test
})
 submission.to_csv(xgb_filename+'.csv', index=False )<save_to_csv>
train_aug = ImageDataGenerator(rescale=1./255,
horizontal_flip = True,
zoom_range = 0.15,
vertical_flip = True,
shear_range=0.1,
rotation_range = 90
)
est = 30
max_depth = 10
rf_pred_filename = 'rf_pred_'+str(est)+'_'+str(max_dept)
 rf = RandomForestRegressor(n_estimators=est,max_depth=max_depth, random_state=42, oob_score=True, verbose=10, n_jobs=-1)
 rf.fit(X_train, y_train)
 pickle.dump(rf, open(rf_pred_filename+'.model', 'wb'))
y_test = rf.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": y_test
})
 submission.to_csv(rf_pred_filename+'.csv', index=False )<choose_model_class>
train_generator = train_aug.flow_from_dataframe(dataframe = df_train,
directory = train_dir,
x_col = "id_code",
y_col = "diagnosis",
batch_size = 16,
target_size =(IMG_SIZE, IMG_SIZE),
class_mode = "categorical" )
meta_train = pd.DataFrame({'ID':X_train.index,'pred_1':xgb.predict(X_train ).clip(0, 20),'pred_2':rf.predict(X_train ).clip(0, 20)})
 y_meta_train = y_train
meta_model = LinearRegression()
meta_model.fit(meta_train[['pred_1','pred_2']],y_meta_train)
 meta_val = pd.DataFrame({'ID':X_val.index,'pred_1':xgb.predict(X_val ).clip(0, 20),'pred_2':rf.predict(X_val ).clip(0, 20)})
 X_meta_val, y_meta_val = meta_val[['pred_1','pred_2']],y_val
print(meta_model.score(X_meta_val,y_meta_val))
filename = 'metamodel.model'
pickle.dump(meta_model, open(filename, 'wb'))
try:
if xgb_filename and rf_filename:
pass
except Exception:
xgb_filename = 'xgb_pred_100_10'
rf_filename = 'rf_pred_30_10'
pred_1 = pd.read_csv(xgb_filename+'.csv')
 pred_2 = pd.read_csv(rf_filename+'.csv')
 meta_test = pd.DataFrame({'ID':pred_1.ID,'pred_1':pred_1.item_cnt_month,'pred_2':pred_2.item_cnt_month})
 X_meta_test = meta_test[['pred_1','pred_2']]
basepath= '.. /input/competitive-data-science-predict-future-sales/'
test = pd.read_csv(basepath+"test.csv")
 y_test = meta_model.predict(X_meta_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": y_test
})
 submission.to_csv('meta_model_pred.csv', index=False)
 submission.to_csv('submission.csv', index=False)
 print('final prediction submission.csv file has been generated.')
 submission<set_options>
validation_generator = train_aug.flow_from_dataframe(dataframe = df_val,
directory = train_dir,
x_col = "id_code",
y_col = "diagnosis",
batch_size = 16,
target_size =(IMG_SIZE, IMG_SIZE),
class_mode = "categorical" )
%matplotlib inline
pd.set_option('display.max_rows', 600)
 pd.set_option('display.max_columns', 50)
 !pip install bayesian-optimization
def downcast_dtypes(df):

 float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype == "int64"]
df[float_cols] = df[float_cols].astype(np.float32)
 df[int_cols] = df[int_cols].astype(np.int32)
 return df
def clip40(x):
if x>40:
return 40
elif x<0:
return 0
else:
return x
def clip20(x):
if x>20:
return 20
elif x<0:
return 0
else:
return x
def remove_from_list(master_list , to_remove):
for elm in to_remove:
if elm in master_list: master_list.remove(elm)
 return master_list
def over_underfit_plot(model, X, y):
gc.collect()
plt.figure()
plt.xlabel("Training examples")
 plt.ylabel("Score")
 train_sizes, train_scores, test_scores = learning_curve(model, X, y, cv=5, n_jobs=-1)
 train_scores_mean = np.mean(train_scores, axis=1)
 test_scores_mean = np.mean(test_scores, axis=1)
 del train_scores,test_scores
plt.grid()
plt.plot(train_sizes, train_scores_mean, 'o-', color="r",
label="Training score")
 plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
label="Cross-validation score")
 plt.legend(loc="best")
 plt.yticks(sorted(set(np.append(train_scores_mean, test_scores_mean))))
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
multiprocessing.cpu_count()<load_from_csv>
def cohens_kappa(y_true, y_pred):
y_true_classes = tf.argmax(y_true, 1)
 y_pred_classes = tf.argmax(y_pred, 1)
 ck_val = tf.contrib.metrics.cohen_kappa(y_true_classes, y_pred_classes, 5)[1]
print(ck_val)
 return ck_val
DATA_FOLDER = '/kaggle/input/competitive-data-science-predict-future-sales/'
sales = pd.read_csv(os.path.join(DATA_FOLDER, 'sales_train.csv'))
items = pd.read_csv(os.path.join(DATA_FOLDER, 'items.csv'))
item_categories = pd.read_csv(os.path.join(DATA_FOLDER, 'item_categories.csv'))
test = pd.read_csv(os.path.join(DATA_FOLDER, 'test.csv'))
shops = pd.read_csv(os.path.join(DATA_FOLDER, 'shops.csv'))
submissions = pd.read_csv(os.path.join(DATA_FOLDER, 'sample_submission.csv'))
test_block = sales['date_block_num'].iloc[-1] + 1
test['date_block_num'] = test_block<feature_engineering>
test_df_orig = pd.read_csv(test_csv)
 def process_test_df(test_df):
test_ids = test_df["id_code"].values.tolist()
for i in range(len(test_ids)) :
imgname = test_ids[i]
newname = str(imgname)+ ".png"
test_ids[i] = newname
test_df["id_code"] = test_ids
return test_df
test_df = process_test_df(test_df_orig )
sales['time'] = pd.to_datetime(sales['date'], format='%d.%m.%Y')
 sales['year'] = sales.time.dt.year
sales['month'] = sales.time.dt.month
sales['day'] = sales.time.dt.day<count_unique_values>
test_aug = ImageDataGenerator(rescale = 1./255)
 test_generator = test_aug.flow_from_dataframe(dataframe = test_df,
directory = test_dir_processed,
x_col = "id_code",
batch_size = 1,
target_size =(IMG_SIZE, IMG_SIZE),
shuffle = False,
class_mode = None )
print('Number of unique train values in column "shop_id" of the dataframe : ',sales['shop_id'].nunique())
 print('Number of unique train values in column "item_id" of the dataframe : ',sales['item_id'].nunique())
 print('Number of unique test values in column "shop_id" of the dataframe : ',test['shop_id'].nunique())
 print('Number of unique test values in column "item_id" of the dataframe : ',test['item_id'].nunique())
<categorify>
class RAdam(keras.optimizers.Optimizer):

 def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., weight_decay=0., amsgrad=False,
total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):
super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.weight_decay = K.variable(weight_decay, name='weight_decay')
 self.total_steps = K.variable(total_steps, name='total_steps')
 self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')
 self.min_lr = K.variable(lr, name='min_lr')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.initial_weight_decay = weight_decay
self.initial_total_steps = total_steps
self.amsgrad = amsgrad
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = self.total_steps - warmup_steps
lr = K.switch(
 t <= warmup_steps,
lr *(t / warmup_steps),
lr *(1.0 - K.minimum(t, decay_steps)/ decay_steps), 
)
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for(i, p)in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for(i, p)in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for(i, p)in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params)) ]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
 beta_2_t = K.pow(self.beta_2, t)
 sma_inf = 2.0 /(1.0 - self.beta_2)- 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t /(1.0 - beta_2_t)
 for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 m_corr_t = m_t /(1.0 - beta_1_t)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 v_corr_t = K.sqrt(vhat_t /(1.0 - beta_2_t)+ self.epsilon)
 self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t /(1.0 - beta_2_t)+ self.epsilon)
 r_t = K.sqrt(( sma_t - 4.0)/(sma_inf - 4.0)* 
(sma_t - 2.0)/(sma_inf - 2.0)*
sma_inf / sma_t)
 p_t = K.switch(sma_t > 5, r_t * m_corr_t / v_corr_t, m_corr_t)
 if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {
'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.weight_decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad,
'total_steps': float(K.get_value(self.total_steps)) ,
'warmup_proportion': float(K.get_value(self.warmup_proportion)) ,
'min_lr': float(K.get_value(self.min_lr)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
item_categories['split'] = item_categories['item_category_name'].str.split('-')
 item_categories['type'] = item_categories['split'].map(lambda x: x[0].strip())
 item_categories['type_code'] = LabelEncoder().fit_transform(item_categories['type'])
 item_categories['subtype'] = item_categories['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 item_categories['subtype_code'] = LabelEncoder().fit_transform(item_categories['subtype'])
 item_categories = item_categories[['item_category_id','type_code', 'subtype_code' , 'item_category_name']]<count_values>
from keras.models import Model
from keras.models import Sequential
from keras.layers import Dense, GlobalAveragePooling2D, Dropout, Input
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import EarlyStopping, ReduceLROnPlateau
from sklearn.model_selection import train_test_split
import keras
from keras.engine import Layer,InputSpec
zero_df = sales[sales['item_cnt_day']<0]
print('number of rows with negative item_cnt_day %f' %(zero_df['item_cnt_day'].count()))
zero_df = sales[sales['item_price']<0]
print('number of rows with negative item_price %f' %(zero_df['item_price'].count()))<feature_engineering>
class GroupNormalization(Layer):

 def __init__(self,
groups=32,
axis=-1,
epsilon=1e-5,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
beta_regularizer=None,
gamma_regularizer=None,
beta_constraint=None,
gamma_constraint=None,
**kwargs):
super(GroupNormalization, self ).__init__(**kwargs)
 self.supports_masking = True
self.groups = groups
self.axis = axis
self.epsilon = epsilon
self.center = center
self.scale = scale
self.beta_initializer = initializers.get(beta_initializer)
 self.gamma_initializer = initializers.get(gamma_initializer)
 self.beta_regularizer = regularizers.get(beta_regularizer)
 self.gamma_regularizer = regularizers.get(gamma_regularizer)
 self.beta_constraint = constraints.get(beta_constraint)
 self.gamma_constraint = constraints.get(gamma_constraint)
 def build(self, input_shape):
dim = input_shape[self.axis]
if dim is None:
raise ValueError('Axis ' + str(self.axis)+ ' of '
'input tensor should have a defined dimension '
'but the layer received an input with shape ' +
str(input_shape)+ '.')
 if dim < self.groups:
raise ValueError('Number of groups(' + str(self.groups)+ ')cannot be '
'more than the number of channels(' +
str(dim)+ ' ).')
 if dim % self.groups != 0:
raise ValueError('Number of groups(' + str(self.groups)+ ')must be a '
'multiple of the number of channels(' +
str(dim)+ ' ).')
 self.input_spec = InputSpec(ndim=len(input_shape),
axes={self.axis: dim})
 shape =(dim,)
 if self.scale:
self.gamma = self.add_weight(shape=shape,
name='gamma',
initializer=self.gamma_initializer,
regularizer=self.gamma_regularizer,
constraint=self.gamma_constraint)
 else:
self.gamma = None
if self.center:
self.beta = self.add_weight(shape=shape,
name='beta',
initializer=self.beta_initializer,
regularizer=self.beta_regularizer,
constraint=self.beta_constraint)
 else:
self.beta = None
self.built = True
def call(self, inputs, **kwargs):
input_shape = K.int_shape(inputs)
 tensor_input_shape = K.shape(inputs)
 reduction_axes = list(range(len(input_shape)))
 del reduction_axes[self.axis]
broadcast_shape = [1] * len(input_shape)
 broadcast_shape[self.axis] = input_shape[self.axis] // self.groups
broadcast_shape.insert(1, self.groups)
 reshape_group_shape = K.shape(inputs)
 group_axes = [reshape_group_shape[i] for i in range(len(input_shape)) ]
group_axes[self.axis] = input_shape[self.axis] // self.groups
group_axes.insert(1, self.groups)
 group_shape = [group_axes[0], self.groups] + group_axes[2:]
group_shape = K.stack(group_shape)
 inputs = K.reshape(inputs, group_shape)
 group_reduction_axes = list(range(len(group_axes)))
 group_reduction_axes = group_reduction_axes[2:]
mean = K.mean(inputs, axis=group_reduction_axes, keepdims=True)
 variance = K.var(inputs, axis=group_reduction_axes, keepdims=True)
 inputs =(inputs - mean)/(K.sqrt(variance + self.epsilon))
inputs = K.reshape(inputs, group_shape)
 outputs = inputs
if self.scale:
broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
 outputs = outputs * broadcast_gamma
if self.center:
broadcast_beta = K.reshape(self.beta, broadcast_shape)
 outputs = outputs + broadcast_beta
outputs = K.reshape(outputs, tensor_input_shape)
 return outputs
def get_config(self):
config = {
'groups': self.groups,
'axis': self.axis,
'epsilon': self.epsilon,
'center': self.center,
'scale': self.scale,
'beta_initializer': initializers.serialize(self.beta_initializer),
'gamma_initializer': initializers.serialize(self.gamma_initializer),
'beta_regularizer': regularizers.serialize(self.beta_regularizer),
'gamma_regularizer': regularizers.serialize(self.gamma_regularizer),
'beta_constraint': constraints.serialize(self.beta_constraint),
'gamma_constraint': constraints.serialize(self.gamma_constraint)
 }
base_config = super(GroupNormalization, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
print(' setting -ve item_cnt_day to zero ')
 sales['item_cnt_day'] = sales['item_cnt_day'].apply(lambda x: x if x >=0 else 0)
 print(' removing -ve item_price items ')
 sales['item_price'] = sales['item_price'].apply(lambda x: x if x >=0 else 0 )<filter>
input_layer = Input(shape =(IMG_SIZE,IMG_SIZE,3))
base_model = ResNeXt101(weights = None,
include_top = True,
backend=keras.backend,
layers=keras.layers,
utils=keras.utils,
models=keras.models,
input_tensor = input_layer)
 base_model.load_weights('.. /input/resnext/resnext101_tf_kernels.h5')
 base_model.summary()
sales = sales[sales.item_price<100000]
sales = sales[sales.item_cnt_day<=1500]<feature_engineering>
sales['revenue'] = sales['item_cnt_day']*sales['item_price']<merge>
x = Dense(512, activation='relu' )(base_model.layers[-2].output)
 x = Dropout(0.35 )(x)
 out = Dense(5, activation = 'softmax' )(x)
 model = Model(inputs = input_layer, outputs = out )
sales = pd.merge(sales, items , how='left', on='item_id')
 sales = pd.merge(sales, item_categories , how='left', on='item_category_id' )<merge>
optimizer = RAdam(lr=0.0005)
 es = EarlyStopping(monitor='cohens_kappa', mode='auto', verbose=1, patience=3,restore_best_weights=True)
 rlrop = ReduceLROnPlateau(monitor='cohens_kappa',
factor=0.2,
patience=5,
verbose=1,
mode='auto',
min_lr=1e-6)
 callback_list = [ rlrop ]
model.compile(optimizer = optimizer, loss = "categorical_crossentropy", metrics = ["accuracy",cohens_kappa] )
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id', col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df<save_to_csv>
K.get_session().run(tf.local_variables_initializer() )
lagged_submissions = pd.merge(submissions , test , on='ID' ).drop(columns=['item_cnt_month'])
 lagged_submissions = pd.merge(lagged_submissions, all_data[all_data['date_block_num']==33] , how="left" ,on=['shop_id' , 'item_id'])
 lagged_submissions = lagged_submissions.filter(['ID','target_lag_1'])
 values = {'target_lag_1': 0}
lagged_submissions = lagged_submissions.fillna(value=values)
 lagged_submissions['item_cnt_month'] = lagged_submissions['target_lag_1'].clip(0,20)
 lagged_submissions.filter(['ID' , 'item_cnt_month'] ).to_csv('sample_submissions.csv' , index=False )<feature_engineering>
history = model.fit_generator(generator = train_generator,
steps_per_epoch = len(train_generator),
epochs = 17,
validation_data = validation_generator,
validation_steps = len(validation_generator),
callbacks = callback_list )
feature_cnt = 5
tfidf = feature_extraction.text.TfidfVectorizer(max_features=feature_cnt)
 items['item_name_len'] = items['item_name'].map(len)
 items['item_name_wc'] = items['item_name'].map(lambda x: len(str(x ).split(' ')))
 txtFeatures = pd.DataFrame(tfidf.fit_transform(items['item_name'] ).toarray())
 cols = txtFeatures.columns
for i in range(feature_cnt):
items['item_name_tfidf_' + str(i)] = txtFeatures[cols[i]] 
<feature_engineering>
predprobs = model.predict_generator(test_generator, steps=len(test_generator))
feature_cnt = 5
tfidf = feature_extraction.text.TfidfVectorizer(max_features=feature_cnt)
 item_categories['item_cat_name_len'] = item_categories['item_category_name'].map(len)
 item_categories['item_cat_name_wc'] = item_categories['item_category_name'].map(lambda x: len(str(x ).split(' ')))
 txtFeatures = pd.DataFrame(tfidf.fit_transform(item_categories['item_category_name'] ).toarray())
 cols = txtFeatures.columns
for i in range(feature_cnt):
item_categories['item_cat_name_tfidf_' + str(i)] = txtFeatures[cols[i]] 
<merge>
shutil.rmtree('train_data_cropped')
 shutil.rmtree('test_dir_processed' )
all_data = pd.merge(all_data, items[['item_id','item_category_id' , 'item_name_tfidf_0' , 'item_name_tfidf_1',
'item_name_tfidf_2','item_name_tfidf_3','item_name_tfidf_4']], how='left', on=['item_id','item_category_id'])
 all_data = pd.merge(all_data, item_categories[['item_category_id' , 'item_cat_name_tfidf_0' , 'item_cat_name_tfidf_1',
'item_cat_name_tfidf_2','item_cat_name_tfidf_3','item_cat_name_tfidf_4' ]], how='left', on=['item_category_id'] )<create_dataframe>
predictions = []
for i in predprobs:
predictions.append(np.argmax(i))
data = [[0, 8], [1,2] , [2,2], [3,0] , [4,6] , [5,3] , [6,1] , [7,2] , [8,2] , [9,1] , [10,2] , [11,3] , [12,9] ,
[13,3] , [14,4] , [15, 1] , [16,8] , [17,7] , [18,2] , [19,0] , [20,2] , [21,2] , [22, 4] , [23,1] , [24,10],
[25,3] , [26,3] , [27,1] , [28,5] , [29,5] , [30,2] , [31,0] , [32 ,2] , [33,1], [34,3]]
holidays = pd.DataFrame(data, columns = ['date_block_num', 'num_hols'])
 holidays['holiday_season'] = holidays['date_block_num'].apply(lambda x: 1 if x in([0,11,12,23,24])else 0 )<merge>
test_df_orig["diagnosis"] = predictions
all_data = pd.merge(all_data, holidays, how='left', on='date_block_num')
 all_data = downcast_dtypes(all_data )<filter>
test_ids = test_df_orig["id_code"].values.tolist()
for i in range(len(test_ids)) :
imgname = test_ids[i]
newname = imgname.split('.')[0]
test_ids[i] = newname
test_df_orig["id_code"] = test_ids
cor_target = abs(corrmatt["target"])
 relevant_features = cor_target[cor_target>0.5]
relevant_features<load_pretrained>
test_df_orig.to_csv('submission.csv',index=False )
<load_pretrained><EOS> 
subfile = pd.read_csv('submission.csv')
 subfile.head(3 )