Kaggle competition: Predict Future Sales
<SOS> metric: rmse Kaggle data source: predict-future-sales<create_dataframe>
tf.__version__
data=pd.concat(( x_train[genes],x_test[genes]))
g_data=PCA(n_components=0.95,random_state=44 ).fit_transform(data)
 g_train=g_data[:x_train.shape[0]]
g_test=g_data[x_train.shape[0]:]
g_train_df=pd.DataFrame(g_train,columns=[f'pca_g-{i}' for i in range(g_data.shape[1])])
 g_test_df=pd.DataFrame(g_test,columns=[f'pca_g-{i}' for i in range(g_data.shape[1])])
 data=pd.concat(( x_train[cells],x_test[cells]))
c_data=PCA(n_components=0.95,random_state=44 ).fit_transform(data)
 c_train=c_data[:x_train.shape[0]]
c_test=c_data[x_train.shape[0]:]
c_train_df=pd.DataFrame(c_train,columns=[f'pca_c-{i}' for i in range(c_data.shape[1])])
 c_test_df=pd.DataFrame(c_test,columns=[f'pca_c-{i}' for i in range(c_data.shape[1])] )<concatenate>
ic = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/item_categories.csv')
 item = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/items.csv')
 shop = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/shops.csv')
 train = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/sales_train.csv')
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
X_train=pd.concat(( x_train,g_train_df,c_train_df),axis=1)
 X_test=pd.concat(( x_test,g_test_df,c_test_df),axis=1 )<normalization>
train = train[(train['item_price']<10000)&(train['item_cnt_day']<1001)]
varthre = VarianceThreshold(0.8)
 train_varthre=varthre.fit_transform(X_train.iloc[:,2:])
 test_varthre=varthre.transform(X_test.iloc[:,2:] )<concatenate>
negp = train[train['item_price']<=0]
print(negp)
 median = train[(train.shop_id==32)&(train.item_id==2973)& 
(train.date_block_num==4)&(train.item_price>0)]['item_price'].median()
train.loc[train.item_price<0,'item_price'] = median
X_train=pd.concat([X_train.iloc[:,:2],pd.DataFrame(train_varthre,columns=X_train.columns[2:][varthre.get_support() ])],1)
 X_test=pd.concat([X_test.iloc[:,:2],pd.DataFrame(test_varthre,columns=X_test.columns[2:][varthre.get_support() ])],1)
 X_train.shape,X_test.shape,y_train.shape,submission.shape<define_variables>
shop['shop_name'].groupby(shop.shop_id ).value_counts()
genes=[col for col in X_train.columns if col.startswith("g-")]
cells=[col for col in X_train.columns if col.startswith("c-")]<train_model>
train.loc[train.shop_id == 0,'shop_id']=57
test.loc[test.shop_id == 0,'shop_id']=57
train.loc[train.shop_id == 1,'shop_id']=58
test.loc[test.shop_id == 1,'shop_id']=58
train.loc[train.shop_id == 10,'shop_id']=11
test.loc[test.shop_id == 10,'shop_id']=11
kmeans=KMeans(n_clusters=35,random_state=982)
 kmeans.fit(pd.concat([X_train[genes],X_test[genes]]))
X_train["cluster_g"]=kmeans.labels_[:len(X_train)]
X_test["cluster_g"]=kmeans.labels_[len(X_train):]
kmeans=KMeans(n_clusters=5,random_state=982)
 kmeans.fit(pd.concat([X_train[cells],X_test[cells]]))
X_train["cluster_c"]=kmeans.labels_[:len(X_train)]
X_test["cluster_c"]=kmeans.labels_[len(X_train):]
X_train=pd.get_dummies(X_train,columns=['cluster_g','cluster_c'])
 X_test=pd.get_dummies(X_test,columns=['cluster_g','cluster_c'] )<define_variables>
ic['split'] = ic['item_category_name'].str.split('-')
 ic['type'] = ic['split'].map(lambda x:x[0].strip())
 ic['type_code'] = LabelEncoder().fit_transform(ic['type'])
 ic['subtype'] = ic['split'].map(lambda x: x[1].strip() if len(x)>1 else x[0].strip())
 ic['subtype_code'] = LabelEncoder().fit_transform(ic['subtype'])
 ic = ic[['item_category_id','type_code','subtype_code']]
ic.head()
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']<feature_engineering>
item.drop(['item_name'],axis=1,inplace=True)
 item.head()
for df in X_train,X_test:
df['time_dose']=df['cp_time']+df['cp_dose']
df['g_median']=df[genes].median(axis=1)
 df['c_median']=df[cells].median(axis=1)
 df['gc_median']=df[genes+cells].median(axis=1)
 df['g_max']=df[genes].max(axis=1)
 df['c_max']=df[cells].max(axis=1)
 df['gc_max']=df[genes+cells].max(axis=1)
 df['g_min']=df[genes].min(axis=1)
 df['c_min']=df[cells].min(axis=1)
 df['gc_min']=df[genes+cells].min(axis=1)
 df['g_sum']=df[genes].sum(axis=1)
 df['c_sum']=df[cells].sum(axis=1)
 df['gc_sum']=df[genes+cells].sum(axis=1)
 df['g_mean']=df[genes].mean(axis=1)
 df['c_mean']=df[cells].mean(axis=1)
 df['gc_mean']=df[genes+cells].mean(axis=1)
 df['g_std']=df[genes].std(axis=1)
 df['c_std']=df[cells].std(axis=1)
 df['gc_std']=df[genes+cells].std(axis=1)
 df['g_skew']=df[genes].skew(axis=1)
 df['c_skew']=df[cells].skew(axis=1)
 df['gc_skew']=df[genes+cells].skew(axis=1)
 df['g_kurtosis']=df[genes].kurtosis(axis=1)
 df['c_kurtosis']=df[cells].kurtosis(axis=1)
 df['gc_kurtosis']=df[genes+cells].kurtosis(axis=1)
 df['g_q25']=df[genes].quantile(0.25,axis=1)
 df['c_q25']=df[cells].quantile(0.25,axis=1)
 df['gc_q25']=df[genes+cells].quantile(0.25,axis=1)
 df['g_q75']=df[genes].quantile(0.75,axis=1)
 df['c_q75']=df[cells].quantile(0.75,axis=1)
 df['gc_q75']=df[genes+cells].quantile(0.75,axis=1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in cells:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2<prepare_x_and_y>
train[(train['shop_id'] == 5)&(train['item_id']==5320)]
X_train=X_train.loc[trt_cp_index,:]
y_train=y_train.loc[trt_cp_index,:]
y_train_non=y_train_non.loc[trt_cp_index,:]<define_search_space>
train_match = pd.DataFrame(np.vstack(train_match),columns =['date_block_num','shop_id','item_id'])
 train_match.sort_values(by = ['date_block_num','shop_id','item_id'],inplace=True)
 train_match.head()
hps={
30:[
{
"units":[1410,1246,512,512,512],
"activations":['selu', 'swish', 'elu','elu','elu'],
"dropouts":[0.49,0.51,0.0,0.0,0.0]
},
{
"units":[1127,1133, 1209, 512],
"activations":[ 'selu','swish','swish','elu'],
"dropouts":[0.32,0.02,0.26,0.0]
},
{
"units":[796,1995,573],
"activations":['selu','selu','swish'],
"dropouts":[0.22,0.21,0.09]
},
{
"units":[959,1557,1826],
"activations":['selu','selu','elu'],
"dropouts":[0.42,0.61,0.3]
},
{
"units":[1081,2033],
"activations":['elu','elu'],
"dropouts":[0.44,0.12]
}
],
58:[
{
"units":[561,882,2023],
"activations":['elu','elu','elu'],
"dropouts":[0.43,0.17,0.29]
},
{
"units":[687,1085,1365,1427,1325],
"activations":['selu','swish','swish','elu','selu'],
"dropouts":[0.59,0.15,0.31,0.08,0.41]
},
{
"units":[1324,1475,976,512],
"activations":['selu','selu','selu', 'elu'],
"dropouts":[0.39,0.38,0.19,0.0]
},
{
"units":[1410,1246,512,512,512],
"activations":['selu', 'swish', 'elu', 'elu', 'elu'],
"dropouts":[0.49,0.51,0.0,0.0,0.0]
},
{
"units":[750,1734,638,512],
"activations":['selu','swish','selu','elu'],
"dropouts":[0.42,0.5,0.42,0.0]
}
],
3255:[
{
"units":[1444,629,512,512,512],
"activations":['selu', 'swish', 'elu', 'elu', 'elu'],
"dropouts":[0.62,0.4,0.0,0.0,0.0]
},
{
"units":[899,940,893,718],
"activations":['elu', 'elu', 'selu', 'selu'],
"dropouts":[0.33,0.28,0.29,0.17]
},
{
"units":[2030,1422],
"activations":['elu', 'selu'],
"dropouts":[0.53,0.37]
},
{
"units":[1194,1562],
"activations":[ 'swish','swish'],
"dropouts":[0.59,0.0]
},
{
"units":[1130,766,520,512],
"activations":[ 'selu','elu','elu', 'elu'],
"dropouts":[0.29,0.14,0.27,0.0]
}
]
} 
<import_modules>
grouped = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day':['sum']})
 grouped.columns = ['item_cnt_month']
grouped.reset_index(inplace = True)
 train_match = pd.merge(train_match,grouped,on=['date_block_num','shop_id','item_id'],how='left')
 train_match.head()
from pickle import load,dump<split>
train_match['item_cnt_month'] =(train_match['item_cnt_month'] 
.fillna(0 ).clip(0,20 ).astype(np.float16))
train_match.head()
y_train_non=y_train_non.iloc[:,y_train_non.columns!='sig_id']
final_y=pd.concat([y_train,y_train_non],1)
 final_pred=pd.DataFrame(np.zeros(( submission.shape[0], final_y.shape[1])) ,columns=final_y.columns)
 for seed in [30,58,3255]:
for n,(tr,te)in enumerate(MultilabelStratifiedKFold(n_splits=5, random_state=seed, shuffle=True ).split(X_train,y_train)) :
print("SEED:"+str(seed)+" Fold:"+str(n))
units=hps[seed][n]['units']
activations=hps[seed][n]['activations']
dropouts=hps[seed][n]['dropouts']
model=create_model(X_train.shape[1], units, activations, dropouts, 1e-3, final_y.shape[1])
 model.fit(X_train.values[tr],
final_y.values[tr],
batch_size=128,
epochs=150,
validation_data=(X_train.values[te],final_y.values[te]),
verbose=1,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
factor=0.1,
patience=3,
epsilon = 1e-4,
mode = 'min',
verbose=1 
)
 ,
tf.keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0,
patience=10,
mode='auto',
verbose=1,
baseline=None,
restore_best_weights=True 
)
 ] 
)
 model.save(f'pSEED{seed}FOLD{n}.h5')
 model=create_model(X_train.shape[1], units, activations, dropouts, 1e-3, final_y.shape[1])
 model.load_weights(f'pSEED{seed}FOLD{n}.h5')
 final_pred+=model.predict(X_test )<prepare_x_and_y>
test['date_block_num'] = 34
train_match = pd.concat([train_match,test],ignore_index=True,
sort = False,keys = ['date_block_num','shop_id','item_id'])
 train_match.fillna(0,inplace=True)
 train_match.tail()
y_pred=final_pred.loc[:,submission.columns[1:]]/15<prepare_output>
train_match = pd.merge(train_match, item, on=['item_id'], how = 'left')
 train_match = pd.merge(train_match, ic, on=['item_category_id'], how = 'left')
 train_match.head()
final_pred=submission.iloc[:,1:].copy()
final_pred.loc[:,:]=0<train_model>
def create_lag_feature(df,lags,col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for lag in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id',col+'_lag_'+str(lag)]
shifted['date_block_num'] += lag
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'],how = 'left')
 return df
for seed in [30,58,3255]:
for n,(tr,te)in enumerate(MultilabelStratifiedKFold(n_splits=5, random_state=seed, shuffle=True ).split(X_train,y_train)) :
print("SEED:"+str(seed)+" Fold:"+str(n))
units=hps[seed][n]['units']
activations=hps[seed][n]['activations']
dropouts=hps[seed][n]['dropouts']
model2=create_model(X_train.shape[1], units, activations, dropouts, 1e-3,y_train_non.shape[1])
 model2.fit(X_train.values[tr],
y_train_non.values[tr],
batch_size=128,
epochs=150,
validation_data=(X_train.values[te],y_train_non.values[te]),
verbose=1,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
factor=0.1,
patience=3,
epsilon = 1e-4,
mode = 'min',
verbose=1 
)
 ,
tf.keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0,
patience=10,
mode='auto',
verbose=1,
baseline=None,
restore_best_weights=True 
)
 ] 
)
 model=create_model(X_train.shape[1], units, activations, dropouts, 1e-3, y_train.shape[1])
 for i in range(len(model.layers)-1):
model.layers[i].set_weights(model2.layers[i].get_weights())
 model.fit(X_train.values[tr],
y_train.values[tr],
batch_size=128,
epochs=150,
validation_data=(X_train.values[te],y_train.values[te]),
verbose=1,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau(
 monitor='val_loss',
factor=0.1,
patience=3,
epsilon = 1e-4,
mode = 'min',
verbose=1 
)
 ,
tf.keras.callbacks.EarlyStopping(
 monitor='val_loss',
min_delta=0,
patience=10,
mode='auto',
verbose=1,
baseline=None,
restore_best_weights=True 
)
 ] 
)
 model.save(f'tSEED{seed}FOLD{n}.h5')
 model=create_model(X_train.shape[1], units, activations, dropouts, 1e-3, y_train.shape[1])
 model.load_weights(f'tSEED{seed}FOLD{n}.h5')
 final_pred+=model.predict(X_test )<define_variables>
train_match = create_lag_feature(train_match,[1,2,3,6,12],'item_cnt_month' )
y_pred_transfer=final_pred/15 
<import_modules>
grouped2 = train_match.groupby(['date_block_num'] ).agg({'item_cnt_month':['mean']})
 grouped2.columns = ['avg_item_cnt_month']
grouped2.reset_index(inplace = True )
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1-y_true)*np.log(1-logits+1e-15)+ y_true*np.log(logits+1e-15)
 return np.mean(-aux )<choose_model_class>
train_match = pd.merge(train_match, grouped2, on=['date_block_num'], how='left')
 train_match['avg_item_cnt_month'] = train_match['avg_item_cnt_month'].astype('float16')
 train_match = create_lag_feature(train_match, [1],'avg_item_cnt_month')
 train_match.drop(['avg_item_cnt_month'],axis =1, inplace = True )
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = 456,
verbose = 10
)<train_on_grid>
grouped3 = train_match.groupby(['date_block_num','item_id'] ).agg({'item_cnt_month':['mean']})
 grouped3.columns = ['item_avg_item_cnt_month']
grouped3.reset_index(inplace = True)
 train_match = pd.merge(train_match, grouped3, on=['date_block_num','item_id'], how ='left')
 train_match['item_avg_item_cnt_month'] = train_match['item_avg_item_cnt_month'].astype('float16')
 train_match = create_lag_feature(train_match, [1,2,3,6,12], 'item_avg_item_cnt_month')
 train_match.drop(['item_avg_item_cnt_month'], axis =1, inplace =True )
final_pred=submission.iloc[:,1:].copy()
final_pred.loc[:,:]=0
for seed in [30,58,3255]:
for n,(tr,te)in enumerate(MultilabelStratifiedKFold(n_splits=5, random_state=seed, shuffle=True ).split(X_train,y_train)) :
print("SEED:"+str(seed)+" Fold:"+str(n))
model = TabNetRegressor(**tabnet_params)
 model.fit(
 X_train = X_train.values[tr],
y_train = y_train.values[tr],
eval_set = [(X_train.values[te],y_train.values[te])],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs = 200,
patience = 20,
batch_size = 512,
virtual_batch_size = 32,
num_workers = 1,
drop_last = False,
loss_fn = F.binary_cross_entropy_with_logits)
 model.save_model(f"tabSEED{seed}FOLD{n}")
 final_pred+=1 /(1 + np.exp(-model.predict(X_test.values)) )<prepare_x_and_y>
grouped4 = train_match.groupby(['date_block_num','shop_id'] ).agg({'item_cnt_month':['mean']})
 grouped4.columns = ['shop_avg_item_cnt_month']
grouped4.reset_index(inplace = True)
 train_match = pd.merge(train_match, grouped4, on=['date_block_num','shop_id'], how ='left')
 train_match['shop_avg_item_cnt_month'] = train_match['shop_avg_item_cnt_month'].astype('float16')
 train_match = create_lag_feature(train_match, [1,2,3,6,12], 'shop_avg_item_cnt_month')
 train_match.drop(['shop_avg_item_cnt_month'], axis =1, inplace =True )
y_pred_tabnet=final_pred/15<prepare_output>
grouped5 = train_match.groupby(['date_block_num','item_category_id'] ).agg({'item_cnt_month':['mean']})
 grouped5.columns = ['ic_avg_item_cnt_month']
grouped5.reset_index(inplace = True)
 train_match = pd.merge(train_match, grouped5, on=['date_block_num','item_category_id'], how ='left')
 train_match['ic_avg_item_cnt_month'] = train_match['ic_avg_item_cnt_month'].astype('float16')
 train_match = create_lag_feature(train_match, [1], 'ic_avg_item_cnt_month')
 train_match.drop(['ic_avg_item_cnt_month'], axis =1, inplace =True )
y_pred_final=y_pred * 0.25 + y_pred_transfer * 0.25 + y_pred_tabnet * 0.5<feature_engineering>
grouped6 = train_match.groupby(['date_block_num','type_code'] ).agg({'item_cnt_month':['mean']})
 grouped6.columns = ['type_avg_item_cnt_month']
grouped6.reset_index(inplace = True)
 train_match = pd.merge(train_match, grouped6, on=['date_block_num','type_code'], how ='left')
 train_match['type_avg_item_cnt_month'] = train_match['type_avg_item_cnt_month'].astype('float16')
 train_match = create_lag_feature(train_match, [1], 'type_avg_item_cnt_month')
 train_match.drop(['type_avg_item_cnt_month'], axis =1, inplace =True )
submission.iloc[:,1:]=y_pred_final<feature_engineering>
grouped7 = train_match.groupby(['date_block_num','subtype_code'] ).agg({'item_cnt_month':['mean']})
 grouped7.columns = ['subtype_avg_item_cnt_month']
grouped7.reset_index(inplace = True)
 train_match = pd.merge(train_match, grouped7, on=['date_block_num','subtype_code'], how ='left')
 train_match['subtype_avg_item_cnt_month'] = train_match['subtype_avg_item_cnt_month'].astype('float16')
 train_match = create_lag_feature(train_match, [1], 'subtype_avg_item_cnt_month')
 train_match.drop(['subtype_avg_item_cnt_month'], axis =1, inplace =True )
submission.loc[x_test_checkpoint["cp_type"]=='ctl_vehicle','5-alpha_reductase_inhibitor':]=0<save_to_csv>
grouped8 = train_match.groupby(['date_block_num','shop_id','item_category_id'] ).agg({'item_cnt_month':['mean']})
 grouped8.columns = ['shop_ic_avg_item_cnt_month']
grouped8.reset_index(inplace = True)
 train_match = pd.merge(train_match, grouped8, on=['date_block_num','shop_id','item_category_id'], how ='left')
 train_match['shop_ic_avg_item_cnt_month'] = train_match['shop_ic_avg_item_cnt_month'].astype('float16')
 train_match = create_lag_feature(train_match, [1], 'shop_ic_avg_item_cnt_month')
 train_match.drop(['shop_ic_avg_item_cnt_month'], axis =1, inplace =True )
submission.to_csv('submission.csv',index=False )<set_options>
grouped9 = train_match.groupby(['date_block_num','shop_id','type_code'] ).agg({'item_cnt_month':['mean']})
 grouped9.columns = ['shop_type_avg_item_cnt_month']
grouped9.reset_index(inplace = True)
 train_match = pd.merge(train_match, grouped9, on=['date_block_num','shop_id','type_code'], how ='left')
 train_match['shop_type_avg_item_cnt_month'] = train_match['shop_type_avg_item_cnt_month'].astype('float16')
 train_match = create_lag_feature(train_match, [1], 'shop_type_avg_item_cnt_month')
 train_match.drop(['shop_type_avg_item_cnt_month'], axis =1, inplace =True )
warnings.filterwarnings('ignore')
 def seed_everything(seed=1903):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds


class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
seed_everything(seed=1903 )<load_from_csv>
grouped10 = train.groupby(['item_id'] ).agg({'item_price':['mean']})
 grouped10.columns = ['item_avg_item_price']
grouped10.reset_index(inplace = True)
 train_match = pd.merge(train_match, grouped10, on=['item_id'], how='left')
 train_match['item_avg_item_price'] = train_match['item_avg_item_price'].astype('float16')
 grouped11 = train.groupby(['date_block_num','item_id'] ).agg({'item_price':['mean']})
 grouped11.columns = ['item_avg_item_price_month']
grouped11.reset_index(inplace = True)
 train_match = pd.merge(train_match, grouped11, on=['date_block_num','item_id'], how='left')
 train_match['item_avg_item_price_month'] = train_match['item_avg_item_price_month'].astype('float16')
 all_lags = [1,2,3,4,5,6]
train_match = create_lag_feature(train_match,all_lags,'item_avg_item_price_month' )
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<define_variables>
for l in all_lags:
train_match['delta_price_lag_'+str(l)]=(train_match['item_avg_item_price_month_lag_'+str(l)] - train_match['item_avg_item_price'])/train_match['item_avg_item_price']
def valid_trend(row):
for l in all_lags:
if row['delta_price_lag_'+str(l)]:
return row['delta_price_lag_'+str(l)]
return 0
train_match['delta_price_lag'] = train_match.apply(valid_trend, axis=1)
 train_match['delta_price_lag'] = train_match['delta_price_lag'].astype('float16')
 train_match['delta_price_lag'].fillna(0, inplace =True)
 to_drop = ['item_avg_item_price','item_avg_item_price_month']
for l in all_lags:
to_drop += ['item_avg_item_price_month_lag_'+str(l)]
to_drop += ['delta_price_lag_'+str(l)]
train_match.drop(to_drop, axis=1, inplace=True )
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<feature_engineering>
train_match['month'] =(train_match['date_block_num']%12)+1
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<create_dataframe>
sale_record = {}
train_match['last_item_sale'] = -1
train_match['last_item_sale'] = train_match['last_item_sale'].astype('int8')
 for idx,row in train_match.iterrows() :
key = row.item_id
if key not in sale_record.keys() :
if row.item_cnt_month != 0:
sale_record[key] = row.date_block_num
else:
last_date_block_num = sale_record[key]
if row.date_block_num > last_date_block_num:
train_match.at[idx,'last_item_sale'] = row.date_block_num - last_date_block_num
sale_record[key] = row.date_block_num
<create_dataframe>
train_match = train_match[train_match['date_block_num']>11]
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=1903 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=1903 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
<create_dataframe>
def fillna_lag(df):
for col in df.columns:
if '_lag_' in col and df[col].isnull().any() :
if 'item_cnt' in col:
df[col].fillna(0, inplace = True)
 return df
train_match = fillna_lag(train_match )
var_thresh = VarianceThreshold(threshold=0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features<define_search_model>
train_match.to_pickle('training.pkl' )
def fe_cluster(train, test, n_clusters_g = 35, n_clusters_c = 5, SEED = 123):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features ,test_features=fe_cluster(train_features,test_features )<feature_engineering>
del train_match
del sale_record
del item
del shop
del ic
del train
del grouped
del grouped2
del grouped3
del grouped4
del grouped5
del grouped6
del grouped7
del grouped8
del grouped9
del grouped10
del grouped11
gc.collect()
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
df['c11_c63'] = df['c-11'] * df['c-63']
df['c90_c13'] = df['c-90'] * df['c-13']
df['c6_c55'] = df['c-6'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features,test_features=fe_stats(train_features,test_features )<merge>
training = pd.read_pickle('.. /input/training-data-for-item-sale-predict/training.pkl' )
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
 target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()<merge>
train_x = training[training.date_block_num<33].drop(['item_cnt_month'],axis=1)
 train_y = training[training.date_block_num<33]['item_cnt_month']
valid_x = training[training.date_block_num==33].drop(['item_cnt_month'],axis=1)
 valid_y = training[training.date_block_num==33]['item_cnt_month']
test_x = training[training.date_block_num==34].drop(['item_cnt_month'],axis=1 )
<categorify>
max_depth_range = {'max_depth':range(6,10,1)}
cv1 = GridSearchCV(estimator = XGBRegressor(n_estimators = 1000,
eta = 0.3,
subsample=0.8,
colsample_bytree=0.8,
seed = 0),
param_grid = max_depth_range,
scoring = 'neg_root_mean_squared_error')
 cv1.fit(train_x,train_y)
 cv1.cv_results_, cv1.best_score_, cv1.best_params_
<split>
xgb_model = XGBRegressor(n_estimators = 1500,
max_depth = 8,
eta = 0.1,
subsample=0.8,
colsample_bytree=0.8,
min_child_weight = 3,
seed = 42 )
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int )<define_variables>
xgb_param = xgb_model.get_xgb_params()
xg_train = xgb.DMatrix(train_x, train_y)
 cv_res = xgb.cv(xgb_param, xg_train, num_boost_round =1000, nfold =3,
metrics ='rmse', early_stopping_rounds =10, stratified =True,
seed=0)
 cv_res
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )<define_variables>
xgb_model.fit(train_x,train_y,
eval_metric = 'rmse',
eval_set = [(train_x, train_y),(valid_x, valid_y)],
early_stopping_rounds = 10 )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=1500
hidden_2 = 820 
<prepare_x_and_y>
result = xgb_model.predict(test_x ).clip(0,20)
 test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 submission = pd.DataFrame({'ID':test.index, 'item_cnt_month':result})
 submission.to_csv('xgb_submission.csv',index =False )
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.05, div_factor=1.5e3,
max_lr=1e-2, epochs=EPOCHS,
steps_per_epoch=len(trainloader))
loss_fn = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions<categorify>
def rmse(y_pred, y_true):
return K.sqrt(K.mean(K.square(y_pred - y_true)))
 def MLP_model() :
model = Sequential()
model.add(Dense(256, activation ='relu',input_dim =30))
model.add(Dropout(0.2))
model.add(Dense(128, activation ='relu'))
model.add(Dropout(0.2))
model.add(Dense(1))
adam = Adam(lr =5e-5)
 model.compile(loss ='mse', optimizer =adam, metrics =[rmse])
 return model
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<set_options>
base_model = MLP_model()
base_model.fit(train_x, train_y, batch_size =32, epochs =15, validation_data =(valid_x, valid_y))
<normalization>
base_model.save('baseline_model.h5' )
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<categorify>
result = base_model.predict(test_x ).clip(0,20)
 result = [each for r in result for each in r]
test = pd.read_csv('.. /input/competitive-data-science-predict-future-sales/test.csv' ).set_index('ID')
 submission = pd.DataFrame({'ID':test.index, 'item_cnt_month':result})
 submission.to_csv('baseline_submission.csv',index =False )
<merge><EOS> 
valid_result = xgb_model.predict(valid_x ).clip(0,20)
 valid_result = [r for r in valid_result]