Kaggle competition: M5 Forecasting - Uncertainty
<SOS> metric: WeightedRowwisePinballLoss Kaggle data source: m5-forecasting-uncertainty<feature_engineering>
import pandas as pd, numpy as np
from matplotlib import pyplot as plt
import scipy.stats as stats
ens_sub.HasDetections -= ens_sub.HasDetections.min() /2<save_to_csv>
pd.options.display.max_columns = 50
ens_sub.to_csv("ens_sub_v16.csv", index=False )<define_variables>
best = pd.read_csv(".. /input/mym5files/GMmasmodels_v2.csv")
 best.head()
LABELS = ["HasDetections"] 
<load_from_csv>
c=0.04
best.iloc[:,1:]=best.iloc[:,1:].ewm(com=c,axis=1 ).mean().values
predict_list = []
predict_list.append(pd.read_csv(".. /input/ensmbl2/submission.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/lightgbm-baseline-model-using-sparse-matrix/lgb_submission.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/nffm-baseline-0-690-on-lb/nffm_submission.csv")[LABELS].values)
<save_to_csv>
sales = pd.read_csv(".. /input/m5-forecasting-uncertainty/sales_train_validation.csv")
 salesids=sales.id.values
sales = pd.read_csv(".. /input/m5-forecasting-uncertainty/sales_train_evaluation.csv")
 sales.id=salesids
sales.head()
print("Rank averaging on ", len(predict_list), " files")
 predictions = np.zeros_like(predict_list[0])
 for predict in predict_list:
for i in range(1):
predictions[:, i] = np.add(predictions[:, i], rankdata(predict[:, i])/predictions.shape[0])
 predictions /= len(predict_list)
 submission = pd.read_csv('.. /input/microsoft-malware-prediction/sample_submission.csv')
 submission[LABELS] = predictions
submission.to_csv('super_blend.csv', index=False )<install_modules>
sub = best.merge(sales[["id", "item_id", "dept_id", "cat_id", "store_id", "state_id"]], on = "id")
 sub["_all_"] = "Total"
sub.shape
!git clone https://github.com/guoday/ctrNet-tool.git
!cp -r ctrNet-tool/*./
!rm -r ctrNet-tool data.git
!ls -all<import_modules>
qs = np.array([0.005,0.025,0.165,0.25, 0.5, 0.75, 0.835, 0.975, 0.995])
 qs.shape
def get_ratios(coef=0.15):
qs2 = np.log(qs/(1-qs)) *coef
ratios = stats.norm.cdf(qs2)
 ratios /= ratios[4]
ratios = pd.Series(ratios, index=qs)
 return ratios.round(3)
 def get_ratios(coef=0.15):
qs2 = np.log(qs/(1-qs)) *coef
ratios = stats.norm.cdf(qs2)
 ratios /= ratios[4]
ratios = pd.Series(ratios, index=qs)
 return ratios.round(3)
 def get_ratios(coef=0.15):
qs2 = np.log(qs/(1-qs)) *coef
ratios = stats.norm.cdf(qs2)
 ratios /= ratios[4]
ratios[-1] *= 1.03
ratios = pd.Series(ratios, index=qs)
 return ratios.round(3)
 def get_ratios2(coef=0.15, a=1.2):
qs2 = np.log(qs/(1-qs)) *coef
ratios = stats.skewnorm.cdf(qs2, a)
 ratios /= ratios[4]
ratios[-1] *= 1.02
ratios = pd.Series(ratios, index=qs)
 return ratios.round(3)
 def get_ratios3(coef=0.15, c=0.5, s=0.1):
qs2 = qs*coef
ratios = stats.powerlognorm.ppf(qs2, c, s)
 ratios /= ratios[4]
ratios[0] *= 0.25
ratios[-1] *= 1.02
ratios = pd.Series(ratios, index=qs)
 return ratios.round(3)
 def widen(array, pc):
array[array<1]=array[array<1] *(1 - pc/100)
 array[array>1]=array[array>1] *(1 + pc/100)
 return array
widen_coef=0.5
level_coef_dict = {"id": widen(( get_ratios2(coef=0.3)+get_ratios3(coef=.3, c=0.04, s=0.9)) /2, pc=widen_coef), "item_id": widen(get_ratios2(coef=0.18, a=0.4),pc=widen_coef),
"dept_id": widen(get_ratios(coef=0.04),widen_coef), "cat_id": widen(get_ratios(coef=0.03),widen_coef),
"store_id": widen(get_ratios(coef=0.035),widen_coef), "state_id": widen(get_ratios(coef=0.03),widen_coef),
"_all_": widen(get_ratios(coef=0.025),widen_coef), 
("state_id", "item_id"): widen(get_ratios2(coef=0.21, a=0.75),pc=widen_coef),("state_id", "dept_id"): widen(get_ratios(coef=0.05),widen_coef), 
("store_id","dept_id"): widen(get_ratios(coef=0.07),widen_coef),("state_id", "cat_id"): widen(get_ratios(coef=0.04),widen_coef), 
("store_id","cat_id"): widen(get_ratios(coef=0.055),widen_coef)
 }
import ctrNet
import pandas as pd
import numpy as np
import tensorflow as tf
from sklearn.model_selection import train_test_split
from src import misc_utils as utils
import os
import gc
import random<define_variables>
a=(( get_ratios2(coef=0.3)+get_ratios3(coef=.3, c=0.04, s=0.9)) /2 ).values
np.where(a==1)[0][0]
dtypes = {
'MachineIdentifier': 'category',
'ProductName': 'category',
'EngineVersion': 'category',
'AppVersion': 'category',
'AvSigVersion': 'category',
'IsBeta': 'int8',
'RtpStateBitfield': 'float16',
'IsSxsPassiveMode': 'int8',
'DefaultBrowsersIdentifier': 'float16',
'AVProductStatesIdentifier': 'float32',
'AVProductsInstalled': 'float16',
'AVProductsEnabled': 'float16',
'HasTpm': 'int8',
'CountryIdentifier': 'int16',
'CityIdentifier': 'float32',
'OrganizationIdentifier': 'float16',
'GeoNameIdentifier': 'float16',
'LocaleEnglishNameIdentifier': 'int8',
'Platform': 'category',
'Processor': 'category',
'OsVer': 'category',
'OsBuild': 'int16',
'OsSuite': 'int16',
'OsPlatformSubRelease': 'category',
'OsBuildLab': 'category',
'SkuEdition': 'category',
'IsProtected': 'float16',
'AutoSampleOptIn': 'int8',
'PuaMode': 'category',
'SMode': 'float16',
'IeVerIdentifier': 'float16',
'SmartScreen': 'category',
'Firewall': 'float16',
'UacLuaenable': 'float32',
'Census_MDC2FormFactor': 'category',
'Census_DeviceFamily': 'category',
'Census_OEMNameIdentifier': 'float16',
'Census_OEMModelIdentifier': 'float32',
'Census_ProcessorCoreCount': 'float16',
'Census_ProcessorManufacturerIdentifier': 'float16',
'Census_ProcessorModelIdentifier': 'float16',
'Census_ProcessorClass': 'category',
'Census_PrimaryDiskTotalCapacity': 'float32',
'Census_PrimaryDiskTypeName': 'category',
'Census_SystemVolumeTotalCapacity': 'float32',
'Census_HasOpticalDiskDrive': 'int8',
'Census_TotalPhysicalRAM': 'float32',
'Census_ChassisTypeName': 'category',
'Census_InternalPrimaryDiagonalDisplaySizeInInches': 'float16',
'Census_InternalPrimaryDisplayResolutionHorizontal': 'float16',
'Census_InternalPrimaryDisplayResolutionVertical': 'float16',
'Census_PowerPlatformRoleName': 'category',
'Census_InternalBatteryType': 'category',
'Census_InternalBatteryNumberOfCharges': 'float32',
'Census_OSVersion': 'category',
'Census_OSArchitecture': 'category',
'Census_OSBranch': 'category',
'Census_OSBuildNumber': 'int16',
'Census_OSBuildRevision': 'int32',
'Census_OSEdition': 'category',
'Census_OSSkuName': 'category',
'Census_OSInstallTypeName': 'category',
'Census_OSInstallLanguageIdentifier': 'float16',
'Census_OSUILocaleIdentifier': 'int16',
'Census_OSWUAutoUpdateOptionsName': 'category',
'Census_IsPortableOperatingSystem': 'int8',
'Census_GenuineStateName': 'category',
'Census_ActivationChannel': 'category',
'Census_IsFlightingInternal': 'float16',
'Census_IsFlightsDisabled': 'float16',
'Census_FlightRing': 'category',
'Census_ThresholdOptIn': 'float16',
'Census_FirmwareManufacturerIdentifier': 'float16',
'Census_FirmwareVersionIdentifier': 'float32',
'Census_IsSecureBootEnabled': 'int8',
'Census_IsWIMBootEnabled': 'float16',
'Census_IsVirtualDevice': 'float16',
'Census_IsTouchEnabled': 'int8',
'Census_IsPenCapable': 'int8',
'Census_IsAlwaysOnAlwaysConnectedCapable': 'float16',
'Wdft_IsGamer': 'float16',
'Wdft_RegionIdentifier': 'float16',
'HasDetections': 'int8'
}
print('Loading Train and Test Data.
 ')
 train = pd.read_csv('.. /input/train.csv', dtype=dtypes, low_memory=True)
 train['MachineIdentifier'] = train.index.astype('uint32')
 test = pd.read_csv('.. /input/test.csv', dtype=dtypes, low_memory=True)
 test['MachineIdentifier'] = test.index.astype('uint32')
 test['HasDetections']=[0]*len(test )<split>
a[a<1]*0.99,a[a>1]*1.01
def make_bucket(data,num=10):
data.sort()
bins=[]
for i in range(num):
bins.append(data[int(len(data)*(i+1)//num)-1])
 return bins
float_features=['Census_SystemVolumeTotalCapacity','Census_PrimaryDiskTotalCapacity']
for f in float_features:
train[f]=train[f].fillna(1e10)
 test[f]=test[f].fillna(1e10)
 data=list(train[f])+list(test[f])
 bins=make_bucket(data,num=50)
 train[f]=np.digitize(train[f],bins=bins)
 test[f]=np.digitize(test[f],bins=bins)
 train, dev,_,_ = train_test_split(train,train['HasDetections'],test_size=0.02, random_state=2019)
 features=train.columns.tolist() [1:-1]<choose_model_class>
def quantile_coefs(q, level):
ratios = level_coef_dict[level]
return ratios.loc[q].values
def get_group_preds(pred, level):
df = pred.groupby(level)[cols].sum()
q = np.repeat(qs, len(df))
df = pd.concat([df]*9, axis=0, sort=False)
 df.reset_index(inplace = True)
 df[cols] *= quantile_coefs(q, level)[:, None]
if level != "id":
df["id"] = [f"{lev}_X_{q:.3f}_validation" for lev, q in zip(df[level].values, q)]
else:
df["id"] = [f"{lev.replace('_validation', '')}_{q:.3f}_validation" for lev, q in zip(df[level].values, q)]
df = df[["id"]+list(cols)]
return df
def get_couple_group_preds(pred, level1, level2):
df = pred.groupby([level1, level2])[cols].sum()
q = np.repeat(qs, len(df))
df = pd.concat([df]*9, axis=0, sort=False)
 df.reset_index(inplace = True)
 df[cols] *= quantile_coefs(q,(level1, level2)) [:, None]
df["id"] = [f"{lev1}_{lev2}_{q:.3f}_validation" for lev1,lev2, q in
zip(df[level1].values,df[level2].values, q)]
df = df[["id"]+list(cols)]
return df
levels = ["id", "item_id", "dept_id", "cat_id", "store_id", "state_id", "_all_"]
couples = [("state_id", "item_id"),("state_id", "dept_id"),("store_id","dept_id"), 
("state_id", "cat_id"),("store_id","cat_id")]
cols = [f"F{i}" for i in range(1, 29)]
df = []
for level in levels :
df.append(get_group_preds(sub, level))
for level1,level2 in couples:
df.append(get_couple_group_preds(sub, level1, level2))
df = pd.concat(df, axis=0, sort=False)
 df.reset_index(drop=True, inplace=True)
 df = pd.concat([df,df] , axis=0, sort=False)
 df.reset_index(drop=True, inplace=True)
 df.loc[df.index >= len(df.index)//2, "id"] = df.loc[df.index >= len(df.index)//2, "id"].str.replace(
 "_validation$", "_evaluation")
 df.shape
df.head()
hparam=tf.contrib.training.HParams(
 model='xdeepfm',
norm=True,
batch_norm_decay=0.9,
hidden_size=[128,128],
cross_layer_sizes=[128,128,128],
k=8,
hash_ids=int(2e5),
batch_size=1024,
optimizer="adam",
learning_rate=0.001,
num_display_steps=1000,
num_eval_steps=1000,
epoch=1,
metric='auc',
activation=['relu','relu','relu'],
cross_activation='identity',
init_method='uniform',
init_value=0.1,
feature_nums=len(features),
kfold=5)
 utils.print_hparams(hparam )<randomize_order>
salesGitem=sales.groupby('item_id')[['item_id']+[f"d_{i}" for i in range(942, 1942)]].sum()
index=set(range(train.shape[0]))
K_fold=[]
for i in range(hparam.kfold):
if i == hparam.kfold-1:
tmp=index
else:
tmp=random.sample(index,int(1.0/hparam.kfold*train.shape[0]))
index=index-set(tmp)
 print("Number:",len(tmp))
K_fold.append(tmp)
 for i in range(hparam.kfold):
print("Fold",i)
 dev_index=K_fold[i]
dev_index=random.sample(dev_index,int(0.1*len(dev_index)))
 train_index=[]
for j in range(hparam.kfold):
if j!=i:
train_index+=K_fold[j]
model=ctrNet.build_model(hparam)
 model.train(train_data=(train.iloc[train_index][features],train.iloc[train_index]['HasDetections']),\
dev_data=(train.iloc[dev_index][features],train.iloc[dev_index]['HasDetections']))
print("Training Done! Inference...")
 if i==0:
preds=model.infer(dev_data=(test[features],test['HasDetections'])) /hparam.kfold
else:
preds+=model.infer(dev_data=(test[features],test['HasDetections'])) /hparam.kfold<save_to_csv>
sales.sort_values('id',inplace=True)
 sales.reset_index(drop=True, inplace = True )
submission = pd.read_csv('.. /input/sample_submission.csv')
 submission['HasDetections'] = preds
print(submission['HasDetections'].head())
 submission.to_csv('nffm_submission.csv', index=False )<import_modules>
quantsalesW=[]
for i in range(7):
quantsalesW.append(
 np.expand_dims(
 np.average(
 np.stack(
(( sales.iloc[:,np.arange(-28*13+i,0,7)].quantile(np.array(
 [0.005,0.025,0.165,0.25, 0.5, 0.75, 0.835, 0.975, 0.995]), axis=1 ).T ).values.reshape(quantsales.shape[0]*quantsales.shape[1], order='F'), 
(sales.iloc[:,np.arange(-28*3+i,0,7)].quantile(np.array(
 [0.005,0.025,0.165,0.25, 0.5, 0.75, 0.835, 0.975, 0.995]), axis=1 ).T ).values.reshape(quantsales.shape[0]*quantsales.shape[1], order='F'))
,-1),axis=-1), -1)
)
 quantsalesW=np.hstack(quantsalesW )
import os<load_from_csv>
medians=np.where(np.array([float(x.split('_')[-2])for x in df.iloc[:274410,0]])== 0.5)[0]
notmedian=np.array([x for x in np.arange(274410)if x not in medians] )
a = pd.read_csv('.. /input/nffm-baseline-0-690-on-lb/nffm_submission.csv')
 b = pd.read_csv('.. /input/ensemble-lb-score-0-697/sub.csv')
 d = pd.read_csv('.. /input/ms-malware-starter/ms_malware.csv')
 e = pd.read_csv('.. /input/simple-blend-upvote-0-697-accuracy/sub.csv')
 f = pd.read_csv('.. /input/why-no-blend/super_blend.csv' )<save_to_csv>
df.iloc[notmedian,1:] =(0.2*df.iloc[notmedian,1:] + 0.7*np.repeat(np.expand_dims(quantsales.reshape(quantsales.shape[0]*quantsales.shape[1], order='F'),-1),28,1)[notmedian,:]
+ 0.1*quantsalesW[notmedian,:])
 df.iloc[medians,1:] =(0.8*df.iloc[medians,1:] + 0.2*np.repeat(np.expand_dims(quantsales.reshape(quantsales.shape[0]*quantsales.shape[1], order='F'),-1),28,1)[medians,:] 
)
submission = a[['MachineIdentifier']]
submission['HasDetections'] =(a['HasDetections']+b['HasDetections']+d['HasDetections']+e['HasDetections']+f['HasDetections'])/6
submission.to_csv('submission.csv', index=False)
 submission.head()<load_from_disk>
df_kaggle_test = pd.read_hdf(
 '.. /input/save-hdf-full/test.hdf',
key="test"
)<feature_engineering>
def add_timestamps(df):
datedictAS = np.load('.. /input/timestamps/AvSigVersionTimestamps.npy')[() ]
df['DateAS'] = df['AvSigVersion'].map(datedictAS)
 datedictOS = np.load('.. /input/timestamps/OSVersionTimestamps.npy')[() ]
df['DateOS'] = df['Census_OSVersion'].map(datedictOS)
 def convert(x):
try:
d = datetime.strptime(x.split('.')[4],'%y%m%d-%H%M')
 except:
d = np.nan
return d
df['DateBL'] = df['OsBuildLab'].map(convert)
 add_timestamps(df_kaggle_test )<load_from_csv>
quantsalesdf=pd.DataFrame(quantsales)
 quantsalesdf['item_id']=sales['item_id'].values
quantsalesdf['state_id']=sales['state_id'].values
quantsalesdfGB=quantsalesdf.groupby(['state_id','item_id'],as_index=False ).mean().iloc[:,2:]
quantsalesdfGB
df_submission=pd.read_csv(".. /input/nffm-baseline-0-690-on-lb/nffm_submission.csv")
 df_submission.hist(bins=30)
 df_submission.head()<save_to_csv>
salesGitemQ=quantsalesdfGB.values.reshape(quantsalesdfGB.shape[0]*quantsalesdfGB.shape[1], order='F')
 salesGitemQ=np.repeat(np.expand_dims(salesGitemQ,-1),28,1)
 salesGitemQ.shape
df_submission.to_csv("original_submission.csv",index=None )<save_to_csv>
medians=np.where(np.array([float(x.split('_')[-2])for x in df.iloc[302067:302067+3049*3*9,0]])== 0.5)[0]
notmedian=np.array([x for x in np.arange(302067,302067+3049*3*9)if x not in medians+302067])
 df.iloc[notmedian,1:] = 0.91*df.iloc[notmedian,1:] +0.09*salesGitemQ[notmedian-302067,:]
df_private_submission.to_csv("private_lb_submission.csv",index=None )<save_to_csv>
df.iloc[302067:302067+3049*3*9,:]
df_submission.to_csv("public_lb_submission.csv",index=None )<set_options>
%pylab inline 
<define_variables>
qals=np.array([float(x.split('_')[-2])for x in df.id] )
trainPath = r'.. /input/train.csv'
testPath = r'.. /input/test.csv'<load_from_csv>
df.iloc[int(df.shape[0]/2):,1:]=df.iloc[:int(df.shape[0]/2),1:].values
train_df = pd.read_csv(trainPath)
 test_df = pd.read_csv(testPath )<feature_engineering>
df.to_csv("submission01.csv", index = False )
train_df['Year'] = train_df['Original_Quote_Date'].apply(lambda x: int(str(x)[:4]))
train_df['Month'] = train_df['Original_Quote_Date'].apply(lambda x: int(str(x)[5:7]))
train_df['Week'] = train_df['Original_Quote_Date'].apply(lambda x: int(str(x)[8:10]))
test_df['Year'] = test_df['Original_Quote_Date'].apply(lambda x: int(str(x)[:4]))
test_df['Month'] = test_df['Original_Quote_Date'].apply(lambda x: int(str(x)[5:7]))
test_df['Week'] = test_df['Original_Quote_Date'].apply(lambda x: int(str(x)[8:10]))
train_df.drop(['Original_Quote_Date'], axis=1,inplace=True)
 test_df.drop(['Original_Quote_Date'], axis=1,inplace=True )<define_variables>
notNumTypeCol = [col for col in train_df.columns if train_df[col].dtype == dtype('O')]<categorify>
42000*9*2
def charToInt(data_df,convDict):
for colName in convDict.keys() :
valuesCounter = convDict[colName]
valuesCounterSize = len(valuesCounter)
 mapDict = dict(zip(valuesCounter.keys() ,range(valuesCounterSize)))
 for index,record in enumerate(data_df[colName]):
data_df[colName][index] = mapDict[record]<feature_engineering>
df.iloc[:int(771120/2),1:].values-df.iloc[int(771120/2):,1:].values
<prepare_x_and_y><EOS> 
np.sum(df.iloc[:int(771120/2),1:].values-df.iloc[int(771120/2):,1:].values )