Kaggle competition: Predict Future Sales
def compute_predictions(example):

 predictions = []
n_best_size = FLAGS.n_best_size
max_answer_length = FLAGS.max_answer_length
i = 0
for unique_id, result in example.results.items() :
if unique_id not in example.features:
raise ValueError("No feature found with unique_id:", unique_id)
 token_map = np.array(example.features[unique_id]["token_map"])
 start_indexes = top_k_indices(result.start_logits,n_best_size,token_map)
 if len(start_indexes)==0:
continue
end_indexes = top_k_indices(result.end_logits,n_best_size,token_map)
 if len(end_indexes)==0:
continue
indexes = np.array(list(np.broadcast(start_indexes[None],end_indexes[:,None])))
 indexes = indexes[(indexes[:,0]<indexes[:,1])*(indexes[:,1]-indexes[:,0]<max_answer_length)]
for _,(start_index,end_index)in enumerate(indexes):
summary = ScoreSummary()
summary.short_span_score =(
 result.start_logits[start_index] +
result.end_logits[end_index])
 summary.cls_token_score =(
 result.start_logits[0] + result.end_logits[0])
 summary.answer_type_logits = result.answer_type_logits-result.answer_type_logits.mean()
start_span = token_map[start_index]
end_span = token_map[end_index] + 1
score = summary.short_span_score - summary.cls_token_score
predictions.append(( score, i, summary, start_span, end_span))
i += 1
short_span = [Span(-1, -1, -10000.0)]
long_span = [Span(-1, -1, -10000.0)]
summary = ScoreSummary()
if predictions:
short_span, long_span = get_short_long_span(predictions, example)
 summary.predicted_label = {
"example_id": int(example.example_id),
"long_answers": {
"tokens_and_score": long_span,
"start_byte": -1,
"end_byte": -1
},
"short_answers": {
"tokens_and_score": short_span,
"start_byte": -1,
"end_byte": -1,
"yes_no_answer": "NONE"
}
}
return summary<load_pretrained>
train_path = ".. /input/competitive-data-science-predict-future-sales/sales_train.csv"
test_path = ".. /input/competitive-data-science-predict-future-sales/test.csv"
items_path = ".. /input/competitive-data-science-predict-future-sales/items.csv"
shops_path = ".. /input/competitive-data-science-predict-future-sales/shops.csv"
item_cat_path = ".. /input/competitive-data-science-predict-future-sales/item_categories.csv"
def read_candidates_from_one_split(input_path):

 candidates_dict = {}
print("Reading examples from: %s" % input_path)
 if input_path.endswith(".gz"):
with gzip.GzipFile(fileobj=tf.io.gfile.GFile(input_path, "rb")) as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
else:
with tf.io.gfile.GFile(input_path, "r")as input_file:
for index, line in enumerate(input_file):
e = json.loads(line)
 candidates_dict[e["example_id"]] = e["long_answer_candidates"]
return candidates_dict<string_transform>
items = pd.read_csv(items_path)
 cats = pd.read_csv(item_cat_path)
 shops = pd.read_csv(shops_path)
 train = pd.read_csv(train_path)
 test = pd.read_csv(test_path)
def read_candidates(input_pattern):

 input_paths = tf.io.gfile.glob(input_pattern)
 final_dict = {}
for input_path in input_paths:
final_dict.update(read_candidates_from_one_split(input_path))
return final_dict<load_pretrained>
train = train[train.item_price > 0].reset_index(drop = True)
 train.loc[train.item_cnt_day < 1, "item_cnt_day"] = 0
all_results = [bert_utils.RawResult(*x)for x in zip(*result)]
print("Going to candidates file")
 candidates_dict = read_candidates('.. /input/tensorflow2-question-answering/simplified-nq-test.jsonl')
 print("setting up eval features")
 eval_features = list(token_map_ds)
 print("compute_pred_dict")
 tqdm_notebook= tqdm.tqdm_notebook
nq_pred_dict = compute_pred_dict(candidates_dict,
eval_features,
all_results,
tqdm=tqdm_notebook)
 predictions_json = {"predictions": list(nq_pred_dict.values())}
print("writing json")
 with tf.io.gfile.GFile('predictions.json', "w")as f:
json.dump(predictions_json, f, indent=4)
 print('done!' )<load_from_disk>
train.loc[train.shop_id == 0, "shop_id"] = 57
test.loc[test.shop_id == 0 , "shop_id"] = 57
train.loc[train.shop_id == 1, "shop_id"] = 58
test.loc[test.shop_id == 1 , "shop_id"] = 58
train.loc[train.shop_id == 11, "shop_id"] = 10
test.loc[test.shop_id == 11, "shop_id"] = 10
train.loc[train.shop_id == 40, "shop_id"] = 39
test.loc[test.shop_id == 40, "shop_id"] = 39
answers_df = pd.read_json(".. /working/predictions.json")
 answers_df.head()<feature_engineering>
shops.loc[ shops.shop_name == 'Сергиев Посад ТЦ "7Я"',"shop_name" ] = 'СергиевПосад ТЦ "7Я"'
shops["city"] = shops.shop_name.str.split(" " ).map(lambda x: x[0])
 shops["category"] = shops.shop_name.str.split(" " ).map(lambda x: x[1])
 shops.loc[shops.city == "!Якутск", "city"] = "Якутск"
def df_long_index_score(df):
answers = []
cont = 0
for e in df['long_answers']['tokens_and_score']:
if e[2] > 3:
index = {}
index['start'] = e[0]
index['end'] = e[1]
index['score'] = e[2]
answers.append(index)
 cont += 1
if cont == 1:
break
return answers
def df_short_index_score(df):
answers = []
cont = 0
for e in df['short_answers']['tokens_and_score']:
if e[2] > 8:
index = {}
index['start'] = e[0]
index['end'] = e[1]
index['score'] = e[2]
answers.append(index)
 cont += 1
if cont == 1:
break
return answers
def df_example_id(df):
return df['example_id']<feature_engineering>
shops.category = shops.category.apply(lambda x: x if(x in category)else "etc" )
answers_df['example_id'] = answers_df['predictions'].apply(df_example_id)
 answers_df['long_indexes_and_scores'] = answers_df['predictions'].apply(df_long_index_score)
 answers_df['short_indexes_and_scores'] = answers_df['predictions'].apply(df_short_index_score)
 answers_df.head()<drop_column>
for cat in shops.category.unique() :
print(cat, len(shops[shops.category == cat]))
answers_df = answers_df.drop(['predictions'], axis=1)
 answers_df.head()<string_transform>
shops["shop_category"] = LabelEncoder().fit_transform(shops.category)
 shops["shop_city"] = LabelEncoder().fit_transform(shops.city )
def create_answer(entry):
answer = []
for e in entry:
answer.append(str(e['start'])+ ':'+ str(e['end']))
if not answer:
answer = ""
return ", ".join(answer)
<feature_engineering>
shops = shops[["shop_id", "shop_category", "shop_city"]]
answers_df["long_answer"] = answers_df['long_indexes_and_scores'].apply(create_answer)
 answers_df["short_answer"] = answers_df['short_indexes_and_scores'].apply(create_answer)
 answers_df["example_id"] = answers_df['example_id'].apply(lambda q: str(q))
long_answers = dict(zip(answers_df["example_id"], answers_df["long_answer"]))
short_answers = dict(zip(answers_df["example_id"], answers_df["short_answer"]))
answers_df.head()<drop_column>
cats["type_code"] = cats.item_category_name.apply(lambda x: x.split(" ")[0] ).astype(str)
 cats.loc[(cats.type_code == "Игровые")|(cats.type_code == "Аксессуары"), "category" ] = "Игры"
answers_df = answers_df.drop(['long_indexes_and_scores', 'short_indexes_and_scores'], axis=1)
 answers_df.head()<feature_engineering>
cats.type_code = cats.type_code.apply(lambda x: x if(x in category)else "etc" )
sample_submission = pd.read_csv(".. /input/tensorflow2-question-answering/sample_submission.csv")
 long_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_long")].apply(lambda q: long_answers[q["example_id"].replace("_long", "")], axis=1)
 short_prediction_strings = sample_submission[sample_submission["example_id"].str.contains("_short")].apply(lambda q: short_answers[q["example_id"].replace("_short", "")], axis=1)
 sample_submission.loc[sample_submission["example_id"].str.contains("_long"), "PredictionString"] = long_prediction_strings
sample_submission.loc[sample_submission["example_id"].str.contains("_short"), "PredictionString"] = short_prediction_strings 
<save_to_csv>
for cat in cats.type_code.unique() :
print(cat, len(cats[cats.type_code == cat]))
sample_submission.to_csv('submission.csv', index=False )<install_modules>
cats.type_code = LabelEncoder().fit_transform(cats.type_code)
 cats["split"] = cats.item_category_name.apply(lambda x: x.split("-"))
cats["subtype"] = cats.split.apply(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats["subtype_code"] = LabelEncoder().fit_transform(cats["subtype"])
 cats = cats[["item_category_id", "subtype_code", "type_code"]]
!pip install knockknock
!pip install -q efficientnet<import_modules>
def name_correction(x):
x = x.lower()
x = x.partition('[')[0]
x = x.partition('(')[0]
x = re.sub('[^A-Za-z0-9А-Яа-я]+', ' ', x)
 x = x.replace(' ', ' ')
 x = x.strip()
return x
user_secrets = UserSecretsClient()
token = user_secrets.get_secret("token")
 chat_id = user_secrets.get_secret("chat_id")
 warnings.filterwarnings('ignore' )<train_on_grid>
items = pd.read_csv(items_path)
 items["name1"], items["name2"] = items.item_name.str.split("[", 1 ).str
items["name1"], items["name3"] = items.item_name.str.split("(", 1 ).str
items["name2"] = items.name2.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items["name3"] = items.name3.str.replace('[^A-Za-z0-9А-Яа-я]+', " " ).str.lower()
items = items.fillna('0')
 items["item_name"] = items["item_name"].apply(lambda x: name_correction(x))
items.name2 = items.name2.apply(lambda x: x[:-1] if x !="0" else "0" )
AUTO = tf.data.experimental.AUTOTUNE
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
 GCS_DS_PATH = KaggleDatasets().get_gcs_path()<define_search_space>
items["type"] = items.name2.apply(lambda x: x[0:8] if x.split(" ")[0] == "xbox" else x.split(" ")[0])
 items.loc[(items.type == "x360")|(items.type == "xbox360")|(items.type == "xbox 360"),"type"] = "xbox 360"
items.loc[ items.type == "", "type"] = "mac"
items.type = items.type.apply(lambda x: x.replace(" ", ""))
items.loc[(items.type == 'pc')|(items.type == 'pс')|(items.type == "pc"), "type" ] = "pc"
items.loc[ items.type == 'рs3' , "type"] = "ps3"
SEED = 2020
N_FOLDS = 5
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
SIZE = [800,800]
LR = 0.0008
EPOCHS = 35
WARMUP = 15<define_variables>
group_sum = items.groupby(["type"] ).agg({"item_id": "count"})
 print(group_sum.reset_index())
 group_sum = group_sum.reset_index()
def seed_everything(SEED):
np.random.seed(SEED)
 tf.random.set_seed(SEED)
 seed_everything(SEED )<load_from_csv>
items.name2 = items.name2.apply(lambda x: "etc" if(x in drop_cols)else x)
 items = items.drop(["type"], axis = 1 )
DIR_INPUT = '/kaggle/input/plant-pathology-2020-fgvc7'
train_df = pd.read_csv(DIR_INPUT + '/train.csv')
 test_df = pd.read_csv(DIR_INPUT + '/test.csv')
 cols = list(train_df.columns[1:] )<normalization>
items.name2 = LabelEncoder().fit_transform(items.name2)
 items.name3 = LabelEncoder().fit_transform(items.name3)
 items.drop(["item_name", "name1"],axis = 1, inplace= True)
 items.head()
def decode_image(filename, label=None, image_size=(SIZE[0], SIZE[1])) :
bits = tf.io.read_file(filename)
 image = tf.image.decode_jpeg(bits, channels=3)
 image = tf.cast(image, tf.float32)
 image = tf.image.per_image_standardization(image)
 image = tf.image.resize(image, image_size)
 if label is None:
return image
else:
return image, label
def data_augment(image, label=None, seed=SEED):
image = tf.image.rot90(image,k=np.random.randint(4))
image = tf.image.random_flip_left_right(image, seed=seed)
 image = tf.image.random_flip_up_down(image, seed=seed)
 if label is None:
return image
else:
return image, label
def preprocess(df,test=False):
paths = df.image_id.apply(lambda x: GCS_DS_PATH + '/images/' + x + '.jpg' ).values
labels = df.loc[:,'healthy':].values
if test==False:
return paths,labels
else:
return paths<normalization>
ts = time.time()
matrix = []
cols = ["date_block_num", "shop_id", "item_id"]
for i in range(34):
sales = train[train.date_block_num == i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype = np.int16))
matrix = pd.DataFrame(np.vstack(matrix), columns = cols)
 matrix["date_block_num"] = matrix["date_block_num"].astype(np.int8)
 matrix["shop_id"] = matrix["shop_id"].astype(np.int8)
 matrix["item_id"] = matrix["item_id"].astype(np.int16)
 matrix.sort_values(cols, inplace = True)
 time.time() - ts
def get_cosine_schedule_with_warmup(lr,num_warmup_steps, num_training_steps, num_cycles=0.5):

 def lrfn(epoch):
if epoch < num_warmup_steps:
return float(epoch)/ float(max(1, num_warmup_steps)) * lr
progress = float(epoch - num_warmup_steps)/ float(max(1, num_training_steps - num_warmup_steps))
return max(0.0, 0.5 *(1.0 + math.cos(math.pi * float(num_cycles)* 2.0 * progress)))* lr
return tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)
 lr_schedule= get_cosine_schedule_with_warmup(lr=LR,num_warmup_steps=WARMUP,num_training_steps=EPOCHS )<choose_model_class>
train["revenue"] = train["item_cnt_day"] * train["item_price"]
def getmodel() :
with strategy.scope() :
model = tf.keras.Sequential([
efn.EfficientNetB7(input_shape=(SIZE[0], SIZE[1], 3),weights='imagenet',pooling='avg',include_top=False),
Dense(4, activation='softmax')])
 model.compile(
 optimizer='adam',loss = 'categorical_crossentropy',
metrics=['categorical_accuracy',])
 return model<train_model>
ts = time.time()
group = train.groupby(["date_block_num", "shop_id", "item_id"] ).agg({"item_cnt_day": ["sum"]})
 group.columns = ["item_cnt_month"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = cols, how = "left")
 matrix["item_cnt_month"] = matrix["item_cnt_month"].fillna(0 ).clip(0,20 ).astype(np.float16)
 time.time() - ts
@telegram_sender(token=token, chat_id=int(chat_id))
def train_one_fold(model,train_dataset,valid_dataset,steps):
history = model.fit(
 train_dataset,
epochs=EPOCHS,
callbacks=[lr_schedule],
steps_per_epoch=steps,
validation_data=valid_dataset)
 string = 'Train acc:{:.4f} Train loss:{:.4f},Val acc:{:.4f} Val loss:{:.4f}'.format(\
model.history.history['categorical_accuracy'][-1],model.history.history['loss'][-1],\
model.history.history['val_categorical_accuracy'][-1],model.history.history['val_loss'][-1])
 return string<split>
test["date_block_num"] = 34
test["date_block_num"] = test["date_block_num"].astype(np.int8)
 test["shop_id"] = test.shop_id.astype(np.int8)
 test["item_id"] = test.item_id.astype(np.int16)
folds = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)
 train_y = np.argmax(train_df[cols].values,axis=1)
 test_pred = np.zeros(( len(test_df),4))
for i_fold,(train_idx, valid_idx)in enumerate(folds.split(train_df,train_y)) :
print("Fold {}/{}".format(i_fold + 1, N_FOLDS))
tf.tpu.experimental.initialize_tpu_system(tpu)
 train = train_df.iloc[train_idx]
train.reset_index(drop=True, inplace=True)
 valid = train_df.iloc[valid_idx]
valid.reset_index(drop=True, inplace=True)
 train_dataset =(tf.data.Dataset 
.from_tensor_slices(preprocess(train)) 
.map(decode_image, num_parallel_calls=AUTO)
.map(data_augment, num_parallel_calls=AUTO)
.shuffle(SEED)
.batch(BATCH_SIZE,drop_remainder=True)
.repeat() 
.prefetch(AUTO))
valid_dataset =(tf.data.Dataset 
.from_tensor_slices(preprocess(valid)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO))
test_dataset =(tf.data.Dataset 
.from_tensor_slices(preprocess(test_df,test=True)) 
.map(decode_image, num_parallel_calls=AUTO)
.batch(BATCH_SIZE))
model = getmodel()
steps = train.shape[0] // BATCH_SIZE
string = train_one_fold(model,train_dataset,valid_dataset,steps)
 pred = model.predict(test_dataset)
 test_pred += pred
submission_df = pd.read_csv(DIR_INPUT + '/sample_submission.csv')
 submission_df[['healthy', 'multiple_diseases', 'rust', 'scab']] = pred
submission_df.to_csv(f'submission_{(i_fold+1)}.csv', index=False)
 print(pd.Series(np.argmax(submission_df[cols].values,axis=1)).value_counts())
 del model
gc.collect()<save_to_csv>
ts = time.time()
matrix = pd.concat([matrix, test.drop(["ID"],axis = 1)], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace = True)
 time.time() - ts
submission_df = pd.read_csv(DIR_INPUT + '/sample_submission.csv')
 submission_df[['healthy', 'multiple_diseases', 'rust', 'scab']] = test_pred/N_FOLDS
submission_df.to_csv('submission.csv', index=False)
 pd.Series(np.argmax(submission_df[cols].values,axis=1)).value_counts()<import_modules>
ts = time.time()
matrix = pd.merge(matrix, shops, on = ["shop_id"], how = "left")
 matrix = pd.merge(matrix, items, on = ["item_id"], how = "left")
 matrix = pd.merge(matrix, cats, on = ["item_category_id"], how = "left")
 matrix["shop_city"] = matrix["shop_city"].astype(np.int8)
 matrix["shop_category"] = matrix["shop_category"].astype(np.int8)
 matrix["item_category_id"] = matrix["item_category_id"].astype(np.int8)
 matrix["subtype_code"] = matrix["subtype_code"].astype(np.int8)
 matrix["name2"] = matrix["name2"].astype(np.int8)
 matrix["name3"] = matrix["name3"].astype(np.int16)
 matrix["type_code"] = matrix["type_code"].astype(np.int8)
 time.time() - ts
import numpy as np
import pandas as pd
import os<load_from_csv>
def lag_feature(df,lags, cols):
for col in cols:
print(col)
 tmp = df[["date_block_num", "shop_id","item_id",col ]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ["date_block_num", "shop_id", "item_id", col + "_lag_"+str(i)]
shifted.date_block_num = shifted.date_block_num + i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
dsub = [
pd.read_csv('.. /input/plantpathology/fork-of-plant-2020-tpu-915e9c_version1.csv'),
pd.read_csv('.. /input/plantpathology/plant-pathology-pytorch-efficientnet-b4-gpu_version_7.csv'),
pd.read_csv('.. /input/plantpathology/public-first-score-tpu-incepresnetv2-enb7_version8.csv'),
pd.read_csv('.. /input/plantpathology/classification-densenet201-efficientnetb7.csv'),
pd.read_csv('.. /input/plantpathology/tf-zoo-models-on-tpu.csv'),
]
pd.read_csv('.. /input/plantpathology/tf-zoo-models-on-tpu.csv')
 dsub.append(pd.read_csv('.. /input/plantpathology/fork-of-plant-2020-tpu-915e9c_version1.csv'))
dsub.append(pd.read_csv('.. /input/plantpathology/fork-of-plant-2020-tpu-915e9c_version1.csv'))
n = len(dsub )<load_from_csv>
ts = time.time()
group = matrix.groupby(["date_block_num"] ).agg({"item_cnt_month" : ["mean"]})
 group.columns = ["date_avg_item_cnt"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = ["date_block_num"], how = "left")
 matrix.date_avg_item_cnt = matrix["date_avg_item_cnt"].astype(np.float16)
 matrix = lag_feature(matrix, [1], ["date_avg_item_cnt"])
 matrix.drop(["date_avg_item_cnt"], axis = 1, inplace = True)
 time.time() - ts
sub = pd.read_csv('.. /input/plant-pathology-2020-fgvc7/sample_submission.csv')
 sub.head()<save_to_csv>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix.date_item_avg_item_cnt = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3], ['date_item_avg_item_cnt'])
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
sub.healthy = 0
sub.multiple_diseases = 0
sub.rust = 0
sub.scab = 0
for d in dsub:
sub.healthy += d.healthy
sub.multiple_diseases += d.multiple_diseases
sub.rust += d.rust
sub.scab += d.scab
sub.healthy = sub.healthy/n
sub.multiple_diseases = sub.multiple_diseases/n
sub.rust = sub.rust/n
sub.scab = sub.scab/n
sub.to_csv('submission.csv', index=False)
 sub.head()<install_modules>
ts = time.time()
group = matrix.groupby(["date_block_num","shop_id"] ).agg({"item_cnt_month" : ["mean"]})
 group.columns = ["date_shop_avg_item_cnt"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = ["date_block_num","shop_id"], how = "left")
 matrix.date_avg_item_cnt = matrix["date_shop_avg_item_cnt"].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3], ["date_shop_avg_item_cnt"])
 matrix.drop(["date_shop_avg_item_cnt"], axis = 1, inplace = True)
 time.time() - ts
!pip install fastai2 --quiet<import_modules>
ts = time.time()
group = matrix.groupby(["date_block_num","shop_id","item_id"] ).agg({"item_cnt_month" : ["mean"]})
 group.columns = ["date_shop_item_avg_item_cnt"]
group.reset_index(inplace = True)
 matrix = pd.merge(matrix, group, on = ["date_block_num","shop_id","item_id"], how = "left")
 matrix.date_avg_item_cnt = matrix["date_shop_item_avg_item_cnt"].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3], ["date_shop_item_avg_item_cnt"])
 matrix.drop(["date_shop_item_avg_item_cnt"], axis = 1, inplace = True)
 time.time() - ts
from fastai2.vision.all import *<define_variables>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix.date_shop_subtype_avg_item_cnt = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], ['date_shop_subtype_avg_item_cnt'])
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
path = Path('.. /input/plant-pathology-2020-fgvc7' )<load_from_csv>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_city'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_city_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', "shop_city"], how='left')
 matrix.date_city_avg_item_cnt = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], ['date_city_avg_item_cnt'])
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
train_df = pd.read_csv(path/'train.csv')
 test_df = pd.read_csv(path/'test.csv' )<load_pretrained>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'shop_city'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'shop_city'], how='left')
 matrix.date_item_city_avg_item_cnt = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], ['date_item_city_avg_item_cnt'])
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
imgs = get_image_files(path/"images" )<filter>
ts = time.time()
group = train.groupby(["item_id"] ).agg({"item_price": ["mean"]})
 group.columns = ["item_avg_item_price"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group, on = ["item_id"], how = "left")
 matrix["item_avg_item_price"] = matrix.item_avg_item_price.astype(np.float16)
 group = train.groupby(["date_block_num","item_id"] ).agg({"item_price": ["mean"]})
 group.columns = ["date_item_avg_item_price"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group, on = ["date_block_num","item_id"], how = "left")
 matrix["date_item_avg_item_price"] = matrix.date_item_avg_item_price.astype(np.float16)
 lags = [1, 2, 3]
matrix = lag_feature(matrix, lags, ["date_item_avg_item_price"])
 for i in lags:
matrix["delta_price_lag_" + str(i)] =(matrix["date_item_avg_item_price_lag_" + str(i)]- matrix["item_avg_item_price"])/ matrix["item_avg_item_price"]
def select_trends(row):
for i in lags:
if row["delta_price_lag_" + str(i)]:
return row["delta_price_lag_" + str(i)]
return 0
matrix["delta_price_lag"] = matrix.apply(select_trends, axis = 1)
 matrix["delta_price_lag"] = matrix.delta_price_lag.astype(np.float16)
 matrix["delta_price_lag"].fillna(0 ,inplace = True)
 features_to_drop = ["item_avg_item_price", "date_item_avg_item_price"]
for i in lags:
features_to_drop.append("date_item_avg_item_price_lag_" + str(i))
features_to_drop.append("delta_price_lag_" + str(i))
matrix.drop(features_to_drop, axis = 1, inplace = True)
 time.time() - ts
train_df.iloc[0, 1:][train_df.iloc[0, 1:] == 1].index[0]<define_variables>
ts = time.time()
group = train.groupby(["date_block_num","shop_id"] ).agg({"revenue": ["sum"] })
 group.columns = ["date_shop_revenue"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group , on = ["date_block_num", "shop_id"], how = "left")
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(["shop_id"] ).agg({ "date_block_num":["mean"] })
 group.columns = ["shop_avg_revenue"]
group.reset_index(inplace = True)
 matrix = matrix.merge(group, on = ["shop_id"], how = "left")
 matrix["shop_avg_revenue"] = matrix.shop_avg_revenue.astype(np.float32)
 matrix["delta_revenue"] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix["delta_revenue"] = matrix["delta_revenue"].astype(np.float32)
 matrix = lag_feature(matrix, [1], ["delta_revenue"])
 matrix["delta_revenue_lag_1"] = matrix["delta_revenue_lag_1"].astype(np.float32)
 matrix.drop(["date_shop_revenue", "shop_avg_revenue", "delta_revenue"] ,axis = 1, inplace = True)
 time.time() - ts
lbl_cols = ['healthy', 'multiple_diseases', 'rust', 'scab']<prepare_x_and_y>
matrix["days"] = matrix["month"].map(days ).astype(np.int8 )
def get_data(idx, size=224, bs=64):
dblock = DataBlock(blocks =(ImageBlock, CategoryBlock),
get_x=ColReader(0, pref=path/"images", suff=".jpg"),
get_y=Pipeline([lambda o:o.iloc[1:][o.iloc[1:] == 1].index[0]]),
splitter=IndexSplitter(idx),
item_tfms=RandomResizedCrop(size+64),
batch_tfms=[*aug_transforms(size=size, flip_vert=True),
Normalize.from_stats(*imagenet_stats)], 
)
 return dblock.dataloaders(train_df, bs=bs )<compute_train_metric>
ts = time.time()
matrix["item_shop_first_sale"] = matrix["date_block_num"] - matrix.groupby(["item_id","shop_id"])["date_block_num"].transform('min')
 matrix["item_first_sale"] = matrix["date_block_num"] - matrix.groupby(["item_id"])["date_block_num"].transform('min')
 time.time() - ts
def roc_auc(preds, targs, labels=range(4)) :
targs = np.eye(4)[targs]
return np.mean([roc_auc_score(targs[:,i], preds[:,i])for i in labels])
 def healthy_roc_auc(*args):
return roc_auc(*args, labels=[0])
 def multiple_diseases_roc_auc(*args):
return roc_auc(*args, labels=[1])
 def rust_roc_auc(*args):
return roc_auc(*args, labels=[2])
 def scab_roc_auc(*args):
return roc_auc(*args, labels=[3] )<import_modules>
ts = time.time()
matrix = matrix[matrix["date_block_num"] > 3]
time.time() - ts
import gc<prepare_x_and_y>
rcParams['figure.figsize'] = 12, 4
dblock = DataBlock(blocks =(ImageBlock, CategoryBlock),
get_x=ColReader(0, pref=path/"images", suff=".jpg"),
get_y=Pipeline([lambda o:o.iloc[1:][o.iloc[1:] == 1].index[0]]),
splitter=RandomSplitter() ,
item_tfms=RandomResizedCrop(224+64),
batch_tfms=[*aug_transforms(size=224, flip_vert=True),
Normalize.from_stats(*imagenet_stats)], 
)
 dls = dblock.dataloaders(train_df, bs=64 )<set_options>
data = matrix.copy()
del matrix
gc.collect()
gc.collect()<import_modules>
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )
from sklearn.model_selection import StratifiedKFold<train_model>
del data
gc.collect() ;
train_lbls = []
for _, lbl in dls.train.dataset:
train_lbls.append(lbl )<define_variables>
ts = time.time()
model = XGBRegressor(
 max_depth=10,
n_estimators=1000,
min_child_weight=0.5,
colsample_bytree=0.8,
subsample=0.8,
eta=0.1,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 20)
 time.time() - ts
for _, lbl in dls.valid.dataset:
train_lbls.append(lbl )<data_type_conversions>
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('xgb_test.pickle', 'wb'))