Kaggle competition: M5 Forecasting - Uncertainty
<SOS> metric: WeightedRowwisePinballLoss Kaggle data source: m5-forecasting-uncertainty<save_to_csv>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import lightgbm as lgb
from sklearn.metrics import mean_squared_error
from tqdm import tqdm
import gc
test['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
submission = test[['key_id', 'word']]
submission.to_csv('gs_mn_submission_{}.csv'.format(int(map3 * 10**4)) , index=False)
 submission.head()
submission.shape<train_model>
def preprocess_sales(sales, start=1400, upper=1970):
if start is not None:
print("dropping...")
 to_drop = [f"d_{i+1}" for i in range(start-1)]
print(sales.shape)
 sales.drop(to_drop, axis=1, inplace=True)
 print(sales.shape)
 print("adding...")
 new_columns = ['d_%i'%i for i in range(1942, upper, 1)]
for col in new_columns:
sales[col] = np.nan
print("melting...")
 sales = sales.melt(id_vars=["id", "item_id", "dept_id", "cat_id", "store_id", "state_id","scale","start"],
var_name='d', value_name='demand')
 print("generating order")
 if start is not None:
skip = start
else:
skip = 1
sales["nb"] =sales.index // 42840 + skip
return sales
def preprocess_calendar(calendar):
global maps, mods
calendar["event_name"] = calendar["event_name_1"]
calendar["event_type"] = calendar["event_type_1"]
map1 = {mod:i for i,mod in enumerate(calendar['event_name'].unique())}
calendar['event_name'] = calendar['event_name'].map(map1)
 map2 = {mod:i for i,mod in enumerate(calendar['event_type'].unique())}
calendar['event_type'] = calendar['event_type'].map(map2)
 calendar['nday'] = calendar['date'].str[-2:].astype(int)
 maps["event_name"] = map1
maps["event_type"] = map2
mods["event_name"] = len(map1)
 mods["event_type"] = len(map2)
 calendar["wday"] -=1
calendar["month"] -=1
calendar["year"] -= 2011
mods["month"] = 12
mods["year"] = 6
mods["wday"] = 7
mods['snap_CA'] = 2
mods['snap_TX'] = 2
mods['snap_WI'] = 2
calendar.drop(["event_name_1", "event_name_2", "event_type_1", "event_type_2", "date", "weekday"],
axis=1, inplace=True)
 return calendar
def make_dataset(categorize=False ,start=1400, upper= 1970):
global maps, mods
print("loading calendar...")
 calendar = pd.read_csv(".. /input/m5-forecasting-uncertainty/calendar.csv")
 print("loading sales...")
 sales = pd.read_csv(".. /input/walmartadd/sales.csv")
 cols = ["item_id", "dept_id", "cat_id","store_id","state_id"]
if categorize:
for col in cols:
temp_dct = {mod:i for i, mod in enumerate(sales[col].unique())}
mods[col] = len(temp_dct)
 maps[col] = temp_dct
for col in cols:
sales[col] = sales[col].map(maps[col])
 sales =preprocess_sales(sales, start=start, upper= upper)
 calendar = preprocess_calendar(calendar)
 calendar = reduce_mem_usage(calendar)
 print("merge with calendar...")
 sales = sales.merge(calendar, on='d', how='left')
 del calendar
print("reordering...")
 sales.sort_values(by=["id","nb"], inplace=True)
 print("re-indexing.. ")
 sales.reset_index(inplace=True, drop=True)
 gc.collect()
sales['n_week'] =(sales['nb']-1)//7
sales["nday"] -= 1
mods['nday'] = 31
sales = reduce_mem_usage(sales)
 gc.collect()
return sales
end = dt.datetime.now()
print('Latest run {}.
 Total time {}s'.format(end,(end - start ).seconds))<set_options>
%%time
CATEGORIZE = True;
START = 1000; UPPER = 1970;
maps = {}
mods = {}
sales = make_dataset(categorize=CATEGORIZE ,start=START, upper= UPPER )
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
start = dt.datetime.now()<define_variables>
sales["x"] = sales["demand"] / sales["scale"]
DP_DIR = '.. /input/shuffle-csvs/'
INPUT_DIR = '.. /input/quickdraw-doodle-recognition/'
BASE_SIZE = 256
NCSVS = 100
NCATS = 340
np.random.seed(seed=1987)
 tf.set_random_seed(seed=1987)
 def f2cat(filename: str)-> str:
return filename.split('.')[0]
def list_all_categories() :
files = os.listdir(os.path.join(INPUT_DIR, 'train_simplified'))
return sorted([f2cat(f)for f in files], key=str.lower )<compute_test_metric>
nb = sales['nb'].values
MAX_LAG = max(LAGS)
 tr_mask = np.logical_and(nb>START + MAX_LAG, nb<=1941)
 val_mask = np.logical_and(nb>1913, nb<=1941)
 te_mask = np.logical_and(nb>1941, nb<=1969 )
def apk(actual, predicted, k=3):

 if len(predicted)> k:
predicted = predicted[:k]
score = 0.0
num_hits = 0.0
for i, p in enumerate(predicted):
if p in actual and p not in predicted[:i]:
num_hits += 1.0
score += num_hits /(i + 1.0)
 if not actual:
return 0.0
return score / min(len(actual), k)
 def mapk(actual, predicted, k=3):

 return np.mean([apk(a, p, k)for a, p in zip(actual, predicted)])
 def preds2catids(predictions):
return pd.DataFrame(np.argsort(-predictions, axis=1)[:, :3], columns=['a', 'b', 'c'])
 def top_3_accuracy(y_true, y_pred):
return top_k_categorical_accuracy(y_true, y_pred, k=3 )<define_variables>
scale = sales['scale'].values
ids = sales['id'].values
ys = sales['x'].values
Z = sales[FEATS].values
STEPS = 800
EPOCHS = 16
size = 64
batchsize = 680<choose_model_class>
scale
model = MobileNet(input_shape=(size, size, 1), alpha=1., weights=None, classes=NCATS)
 model.compile(optimizer=Adam(lr=0.002), loss='categorical_crossentropy',
metrics=[categorical_crossentropy, categorical_accuracy, top_3_accuracy])
 print(model.summary() )<load_from_csv>
sv = scale[val_mask]
se = scale[te_mask]
ids = ids[te_mask]
ids = ids.reshape(( -1, 28))
valid_df = pd.read_csv(os.path.join(DP_DIR, 'train_k{}.csv.gz'.format(NCSVS - 1)) , nrows=34000)
 x_valid = df_to_image_array_xd(valid_df, size)
 y_valid = keras.utils.to_categorical(valid_df.y, num_classes=NCATS)
 print(x_valid.shape, y_valid.shape)
 print('Validation array memory {:.2f} GB'.format(x_valid.nbytes / 1024.**3))<drop_column>
ca = sales[['snap_CA']].values
tx = sales[['snap_TX']].values
wi = sales[['snap_WI']].values
wday = sales[['wday']].values
month = sales[['month']].values
year = sales[['year']].values
event = sales[['event_name']].values
nday = sales[['nday']].values
lag1=sales[['x_28']].values
train_datagen = image_generator_xd(size=size, batchsize=batchsize, ks=range(NCSVS - 1))<prepare_x_and_y>
item = sales[['item_id']].values
dept = sales[['dept_id']].values
cat = sales[['cat_id']].values
store = sales[['store_id']].values
state = sales[['state_id']].values
%%timeit
x, y = next(train_datagen )<train_on_grid>
def make_data(mask):
x = {"snap_CA":ca[mask], "snap_TX":tx[mask], "snap_WI":wi[mask], "wday":wday[mask],
"month":month[mask], "year":year[mask], "event":event[mask], "nday":nday[mask],
"item":item[mask], "dept":dept[mask], "cat":cat[mask], "store":store[mask],
"state":state[mask],"num":Z[mask]}
t = ys[mask]
return x, t
callbacks = [
ReduceLROnPlateau(monitor='val_top_3_accuracy', factor=0.75, patience=3, min_delta=0.001,
mode='max', min_lr=1e-5, verbose=1),
ModelCheckpoint('model.h5', monitor='val_top_3_accuracy', mode='max', save_best_only=True,
save_weights_only=True),
]
hists = []
hist = model.fit_generator(
 train_datagen, steps_per_epoch=STEPS, epochs=70, verbose=1,
validation_data=(x_valid, y_valid),
callbacks = callbacks 
)
 hists.append(hist )<predict_on_test>
xt, yt = make_data(tr_mask)
 xv, yv = make_data(val_mask)
 xe, ye = make_data(te_mask )
valid_predictions = model.predict(x_valid, batch_size=128, verbose=1)
 map3 = mapk(valid_df[['y']].values, preds2catids(valid_predictions ).values)
 print('Map3: {:.3f}'.format(map3))<load_from_csv>
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
import tensorflow.keras.backend as K
from tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping
import tensorflow as tf
test = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))
test.head()
x_test = df_to_image_array_xd(test, size)
 print(test.shape, x_test.shape)
 print('Test array memory {:.2f} GB'.format(x_test.nbytes / 1024.**3))<predict_on_test>
qs = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
test_predictions = model.predict(x_test, batch_size=128, verbose=1)
 top3 = preds2catids(test_predictions)
 top3.head()
top3.shape
cats = list_all_categories()
id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(cats)}
top3cats = top3.replace(id2cat)
 top3cats.head()
top3cats.shape<save_to_csv>
q = tf.constant(np.array([qs]), dtype=tf.float32)
 q
test['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
submission = test[['key_id', 'word']]
submission.to_csv('gs_mn_submission_{}.csv'.format(int(map3 * 10**4)) , index=False)
 submission.head()
submission.shape<train_model>
ca = L.Input(( 1,), name="snap_CA")
 ca
end = dt.datetime.now()
print('Latest run {}.
 Total time {}s'.format(end,(end - start ).seconds))<import_modules>
def qloss(y_true, y_pred):
qs = [0.005, 0.025, 0.165, 0.250, 0.500, 0.750, 0.835, 0.975, 0.995]
q = tf.constant(np.array([qs]), dtype=tf.float32)
 e = y_true - y_pred
v = tf.maximum(q*e,(q-1)*e)
 return K.mean(v)
 def make_model(n_in):
num = L.Input(( n_in,), name="num")
 ca = L.Input(( 1,), name="snap_CA")
 tx = L.Input(( 1,), name="snap_TX")
 wi = L.Input(( 1,), name="snap_WI")
 wday = L.Input(( 1,), name="wday")
 month = L.Input(( 1,), name="month")
 year = L.Input(( 1,), name="year")
 event = L.Input(( 1,), name="event")
 nday = L.Input(( 1,), name="nday")
 item = L.Input(( 1,), name="item")
 dept = L.Input(( 1,), name="dept")
 cat = L.Input(( 1,), name="cat")
 store = L.Input(( 1,), name="store")
 state = L.Input(( 1,), name="state")
 inp = {"snap_CA":ca, "snap_TX":tx, "snap_WI":wi, "wday":wday,
"month":month, "year":year, "event":event, "nday":nday,
"item":item, "dept":dept, "cat":cat, "store":store,
"state":state, "num":num}
ca_ = L.Embedding(mods["snap_CA"], mods["snap_CA"], name="ca_3d" )(ca)
 tx_ = L.Embedding(mods["snap_TX"], mods["snap_TX"], name="tx_3d" )(tx)
 wi_ = L.Embedding(mods["snap_WI"], mods["snap_WI"], name="wi_3d" )(wi)
 wday_ = L.Embedding(mods["wday"], mods["wday"], name="wday_3d" )(wday)
 month_ = L.Embedding(mods["month"], mods["month"], name="month_3d" )(month)
 year_ = L.Embedding(mods["year"], mods["year"], name="year_3d" )(year)
 event_ = L.Embedding(mods["event_name"], mods["event_name"], name="event_3d" )(event)
 nday_ = L.Embedding(mods["nday"], mods["nday"], name="nday_3d" )(nday)
 item_ = L.Embedding(mods["item_id"], 10, name="item_3d" )(item)
 dept_ = L.Embedding(mods["dept_id"], mods["dept_id"], name="dept_3d" )(dept)
 cat_ = L.Embedding(mods["cat_id"], mods["cat_id"], name="cat_3d" )(cat)
 store_ = L.Embedding(mods["store_id"], mods["store_id"], name="store_3d" )(store)
 state_ = L.Embedding(mods["state_id"], mods["state_id"], name="state_3d" )(state)
 p = [ca_, tx_, wi_, wday_, month_, year_, event_, nday_, item_, dept_, cat_, store_, state_]
emb = L.Concatenate(name="embds" )(p)
 context = L.Flatten(name="context" )(emb)
 x = L.Concatenate(name="x1" )([context, num])
 x = L.Dense(500, activation="relu", name="d1" )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Concatenate(name="m1" )([x, context])
 x = L.Dense(500, activation="relu", name="d2" )(x)
 x = L.Dropout(0.3 )(x)
 x = L.Concatenate(name="m2" )([x, context])
 x = L.Dense(500, activation="relu", name="d3" )(x)
 preds = L.Dense(9, activation="linear", name="preds" )(x)
 model = M.Model(inp, preds, name="M1")
 model.compile(loss=qloss, optimizer="adam")
 return model
import pandas as pd, numpy as np, os, gc
import matplotlib.pyplot as plt
from datetime import datetime
from datetime import timedelta<load_from_csv>
net = make_model(len(FEATS))
ckpt = ModelCheckpoint("w.h5", monitor='val_loss', verbose=1, save_best_only=True,mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
patience=5, min_lr=0.001)
 es = EarlyStopping(monitor='val_loss', patience=3)
 print(net.summary() )
AdjustPublicScore = True
AdjustPrivateScore = 2
dtypes = {}
dtypes['MachineIdentifier'] = 'str'
dtypes['AvSigVersion'] = 'category'
dtypes['HasDetections'] = 'int8'
df_train = pd.read_csv('.. /input/microsoft-malware-prediction/train.csv', usecols=list(dtypes.keys()), dtype=dtypes)
 print('Loaded',len(df_train),'rows of train.CSV!')
 df_test = pd.read_csv('.. /input/microsoft-malware-prediction/test.csv', usecols=list(dtypes.keys())[0:-1], dtype=dtypes)
 print('Loaded',len(df_test),'rows of test.CSV!')
 df_test2 = pd.read_csv('.. /input/kagglebest/super_blend.csv')
 print('Loaded',len(df_test),'rows of super_blend.csv!')
 datedictAS = np.load('.. /input/malware-timestamps/AvSigVersionTimestamps.npy')[() ]
df_test['Date'] = df_test['AvSigVersion'].map(datedictAS)
 df_train['Date'] = df_train['AvSigVersion'].map(datedictAS)
 df_test2 = pd.merge(df_test2, df_test, on='MachineIdentifier', how='left')
 df_test2['AvSigVersion2'] = df_test2['AvSigVersion'].map(lambda x: np.int(x.split('.')[1]))<feature_engineering>
net.fit(xt, yt, batch_size=50_000, epochs=20, validation_data=(xv, yv), callbacks=[ckpt, reduce_lr, es] )
if AdjustPublicScore: df_test2.loc[(df_test2['AvSigVersion2']==275)|(df_test2['AvSigVersion2']==273),'HasDetections'] *= 0.6<feature_engineering>
nett = make_model(len(FEATS))
nett.load_weights("w.h5" )
if AdjustPrivateScore==1:
df_test2.loc[ df_test2['Date']>datetime(2018,11,20,4,0),'HasDetections'] = 0
elif AdjustPrivateScore==2:
df_test2['X'] = df_test2['Date'] - datetime(2018,11,20,4,0)
 df_test2['X'] = df_test2['X'].map(lambda x: x.total_seconds() /86400)
 df_test2['X'].fillna(0,inplace=True)
 s = 5.813888
df_test2['F'] = 1.0
df_test2['F'] = 1 - df_test2['X']/s
df_test2.loc[df_test2['X']<=0,'F'] = 1.0
df_test2.loc[df_test2['X']>s,'F'] = 0
df_test2['HasDetections'] *= df_test2['F']<save_to_csv>
pv = nett.predict(xv, batch_size=50_000, verbose=1)
 pe = nett.predict(xe, batch_size=50_000, verbose=1 )
df_test2[['MachineIdentifier','HasDetections']].to_csv('PrivateLeaderboard.csv', index=False )<import_modules>
nett.evaluate(xv, yv, batch_size=50_000 )
from scipy.stats import rankdata<define_variables>
names = [f"F{i+1}" for i in range(28)]
sub_files = [
".. /input/new-blend/ens_sub_v2.csv",
".. /input/new-blend/sub.csv",
".. /input/new-blend/super_blend.csv",
".. /input/newsub/submission.csv",
]<load_from_csv>
piv = pd.DataFrame(ids[:, 0], columns=["id"] )
df1 = pd.read_csv(sub_files[0])
 df2 = pd.read_csv(sub_files[1])
 df3 = pd.read_csv(sub_files[2])
 df4 = pd.read_csv(sub_files[3] )<load_from_csv>
QUANTILES = ["0.005", "0.025", "0.165", "0.250", "0.500", "0.750", "0.835", "0.975", "0.995"]
VALID = []
EVAL = []
for i, quantile in tqdm(enumerate(QUANTILES)) :
t1 = pd.DataFrame(pv[:,:, i]*sv, columns=names)
 t1 = piv.join(t1)
 t1["id"] = t1["id"] + f"_{quantile}_validation"
t2 = pd.DataFrame(pe[:,:, i]*se, columns=names)
 t2 = piv.join(t2)
 t2["id"] = t2["id"] + f"_{quantile}_evaluation"
VALID.append(t1)
 EVAL.append(t2)
ens_sub = pd.read_csv(".. /input/microsoft-malware-prediction/sample_submission.csv" )<compute_test_metric>
sub = pd.DataFrame()
sub = sub.append(VALID + EVAL)
 del VALID, EVAL, t1, t2
<prepare_output><EOS> 
sub.to_csv("submission.csv", index=False )