Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<load_from_csv>
warnings.filterwarnings("ignore")
 sys.path.append('.. /input/iterative-stratification/iterative-stratification-master')
 np.random.seed(42)
 tf.random.set_seed(42)
 sns.set_style('white')
 sns.set(font_scale=1.2)
sales = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_evaluation.csv')
 sales.name = 'sales'
calendar = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/calendar.csv')
 calendar.name = 'calendar'
prices = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sell_prices.csv')
 prices.name = 'prices'<data_type_conversions>
df_train = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 display(df_train.head(3))
print('train data size', df_train.shape)
 df_target_ns = pd.read_csv('/kaggle/input/lish-moa/train_targets_nonscored.csv')
 display(df_target_ns.head(3))
print('train target nonscored size', df_target_ns.shape)
 df_target_s = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 display(df_target_s.head(3))
print('train target scored size', df_target_s.shape)
 df_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 display(df_test.head(3))
print('test data size', df_test.shape)
 df_sample = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 display(df_sample.head(3))
print('sample submission size', df_sample.shape)
for d in range(1942,1970):
col = 'd_' + str(d)
 sales[col] = 0
sales[col] = sales[col].astype(np.int16 )<data_type_conversions>
g_features = [cols for cols in df_train.columns if cols.startswith('g-')]
def downcast(df):
cols = df.dtypes.index.tolist()
types = df.dtypes.values.tolist()
for i,t in enumerate(types):
if 'int' in str(t):
if df[cols[i]].min() > np.iinfo(np.int8 ).min and df[cols[i]].max() < np.iinfo(np.int8 ).max:
df[cols[i]] = df[cols[i]].astype(np.int8)
 elif df[cols[i]].min() > np.iinfo(np.int16 ).min and df[cols[i]].max() < np.iinfo(np.int16 ).max:
df[cols[i]] = df[cols[i]].astype(np.int16)
 elif df[cols[i]].min() > np.iinfo(np.int32 ).min and df[cols[i]].max() < np.iinfo(np.int32 ).max:
df[cols[i]] = df[cols[i]].astype(np.int32)
 else:
df[cols[i]] = df[cols[i]].astype(np.int64)
 elif 'float' in str(t):
if df[cols[i]].min() > np.finfo(np.float16 ).min and df[cols[i]].max() < np.finfo(np.float16 ).max:
df[cols[i]] = df[cols[i]].astype(np.float16)
 elif df[cols[i]].min() > np.finfo(np.float32 ).min and df[cols[i]].max() < np.finfo(np.float32 ).max:
df[cols[i]] = df[cols[i]].astype(np.float32)
 else:
df[cols[i]] = df[cols[i]].astype(np.float64)
 elif t == np.object:
if cols[i] == 'date':
df[cols[i]] = pd.to_datetime(df[cols[i]], format='%Y-%m-%d')
 else:
df[cols[i]] = df[cols[i]].astype('category')
 return df
sales = downcast(sales)
 prices = downcast(prices)
 calendar = downcast(calendar )<merge>
c_features = [cols for cols in df_train.columns if cols.startswith('c-')]
df = pd.melt(sales, id_vars=['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'], var_name='d', value_name='sold' ).dropna()
df = pd.merge(df, calendar, on='d', how='left')
 df = pd.merge(df, prices, on=['store_id','item_id','wm_yr_wk'], how='left' )<define_variables>
ind_tr = df_train[df_train['cp_type']=='ctl_vehicle'].index
ind_te = df_test[df_test['cp_type']=='ctl_vehicle'].index
d_id = dict(zip(df.id.cat.codes, df.id))
d_item_id = dict(zip(df.item_id.cat.codes, df.item_id))
d_dept_id = dict(zip(df.dept_id.cat.codes, df.dept_id))
d_cat_id = dict(zip(df.cat_id.cat.codes, df.cat_id))
d_store_id = dict(zip(df.store_id.cat.codes, df.store_id))
d_state_id = dict(zip(df.state_id.cat.codes, df.state_id))<data_type_conversions>
transformer = QuantileTransformer(n_quantiles=100,random_state=42, output_distribution="normal")
 def preprocess(df):
df['cp_time'] = df['cp_time'].map({24:1, 48:2, 72:3})
 df['cp_dose'] = df['cp_dose'].map({'D1':0, 'D2':1})
 g_features = [cols for cols in df.columns if cols.startswith('g-')]
c_features = [cols for cols in df.columns if cols.startswith('c-')]
for col in(g_features + c_features):
vec_len = len(df[col].values)
 raw_vec = df[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 df[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
return df
X = preprocess(df_train)
 X_test = preprocess(df_test)
 display(X.head(5))
print('Train data size', X.shape)
 display(X_test.head(3))
print('Test data size', X_test.shape)
 y = df_target_s.drop('sig_id', axis=1)
 display(y.head(3))
print('target size', y.shape)
 y0 = df_target_ns.drop('sig_id', axis=1 )
list1=['event_name_1','event_type_1','event_name_2','event_type_2']
for i in list1:
df[i] = df[i].cat.add_categories("nan" ).fillna("nan")
 df[i]=LabelEncoder().fit_transform(df[i] ).astype(np.int8)
 df[i]=df[i].astype('category')
 df.d = df['d'].apply(lambda x: x.split('_')[1] ).astype(np.int16)
 cols = df.dtypes.index.tolist()
types = df.dtypes.values.tolist()
for i,type in enumerate(types):
if type.name == 'category':
df[cols[i]] = df[cols[i]].cat.codes
df['date'] = df['date'].apply(lambda x: x.strftime('%d')).astype(np.int8 )<data_type_conversions>
g_features = [cols for cols in X.columns if cols.startswith('g-')]
n_comp = 0.95
data = pd.concat([pd.DataFrame(X[g_features]), pd.DataFrame(X_test[g_features])])
 data2 =(PCA(0.95, random_state=42 ).fit_transform(data[g_features]))
train2 = data2[:X.shape[0]]
test2 = data2[-X_test.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_g-{i}' for i in range(data2.shape[1])])
 test2 = pd.DataFrame(test2, columns=[f'pca_g-{i}' for i in range(data2.shape[1])])
 X = pd.concat(( X, train2), axis=1)
 X_test = pd.concat(( X_test, test2), axis=1)
 c_features = [cols for cols in X.columns if cols.startswith('c-')]
n_comp = 0.95
data = pd.concat([pd.DataFrame(X[c_features]), pd.DataFrame(X_test[c_features])])
 data2 =(PCA(0.95, random_state=42 ).fit_transform(data[c_features]))
train2 = data2[:X.shape[0]]
test2 = data2[-X_test.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_c-{i}' for i in range(data2.shape[1])])
 test2 = pd.DataFrame(test2, columns=[f'pca_c-{i}' for i in range(data2.shape[1])])
 X = pd.concat(( X, train2), axis=1)
 X_test = pd.concat(( X_test, test2), axis=1)
 display(X.head(2))
display(X_test.head(2))
lags = [28,35,42,49]
for lag in lags:
df['sold_lag_'+str(lag)] = df.groupby(['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],as_index=False)['sold'].shift(lag ).astype(np.float16)
 lags2 = [1,2]
for lag in lags2:
df['event1_lag_'+str(lag)] = df.groupby(['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],as_index=False)['event_name_1'].shift(lag ).astype(np.float16)
 df['event1_lag_'+str(lag)].fillna(100, inplace=True)
 df['event1_lag_'+str(lag)]=df['event1_lag_'+str(lag)].astype(np.int8)
 df['event1_lag_'+str(lag)]=df['event1_lag_'+str(lag)].astype('category' )<data_type_conversions>
def fe_stats(train, test):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 return train, test
X,X_test=fe_stats(X,X_test)
 display(X.head(2))
print(X.shape)
 display(X_test.head(2))
print(X_test.shape )
df['item_sold_avg'] = df.groupby('item_id')['sold'].transform('mean' ).astype(np.float16)
 df['cat_sold_avg'] = df.groupby('cat_id')['sold'].transform('mean' ).astype(np.float16)
 df['dept_sold_avg'] = df.groupby('dept_id')['sold'].transform('mean' ).astype(np.float16)
 df['cat_dept_sold_avg'] = df.groupby(['cat_id','dept_id'])['sold'].transform('mean' ).astype(np.float16)
 df['store_item_sold_avg'] = df.groupby(['store_id','item_id'])['sold'].transform('mean' ).astype(np.float16)
 df['cat_item_sold_avg'] = df.groupby(['cat_id','item_id'])['sold'].transform('mean' ).astype(np.float16)
 df['dept_item_sold_avg'] = df.groupby(['dept_id','item_id'])['sold'].transform('mean' ).astype(np.float16)
 df['store_cat_dept_sold_avg'] = df.groupby(['store_id','cat_id','dept_id'])['sold'].transform('mean' ).astype(np.float16)
 df['wm_yr_wk_linear']=LabelEncoder().fit_transform(df['wm_yr_wk'].values ).astype(np.int16)
 df.drop(['wm_yr_wk'], axis=1, inplace=True)
 df['price_lag'] = df.groupby(['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],as_index=False)['sell_price'].shift(7 ).astype(np.float16)
 df['price-diff']=df['price_lag']-df['sell_price']
df.drop(['price_lag'], axis=1, inplace=True)
 df['sell_price'].fillna(-1,inplace=True)
 df['decimal']=df['sell_price'].apply(lambda x: 100*(x-int(x)) ).astype(np.int16)
 df['sell_price'].replace(-1,np.nan,inplace=True)
 df['expanding_price_mean'] = df.groupby(['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'])['sell_price'].transform(lambda x: x.expanding(2 ).mean() ).astype(np.float16)
 df['diff_moving_mean']=df['expanding_price_mean']-df['sell_price']
df.drop(['expanding_price_mean'], axis=1, inplace=True)
 df['price-diff']=df['price-diff'].astype(np.float16)
 df.drop(['wday'], axis=1, inplace=True)
 df['decimal']=df['decimal'].astype(np.int8)
 df['year']=LabelEncoder().fit_transform(df['year'] ).astype(np.int8)
 df['daily_avg_sold'] = df.groupby(['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id','d'])['sell_price'].transform('mean' ).astype(np.float16)
 df['avg_sold'] = df.groupby(['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'])['sell_price'].transform('mean' ).astype(np.float16)
 df['selling_trend'] =(df['daily_avg_sold'] - df['avg_sold'] ).astype(np.float16)
 df.drop(['daily_avg_sold','avg_sold'],axis=1,inplace=True)
 df['price_max'] = df.groupby(['store_id','item_id'])['sell_price'].transform('max')
 df['price_norm'] = df['sell_price']/df['price_max']
df['price_momentum_m'] = df['sell_price']/df.groupby(['store_id','item_id','month'])['sell_price'].transform('mean')
 df['price_momentum_y'] = df['sell_price']/df.groupby(['store_id','item_id','year'])['sell_price'].transform('mean')
<drop_column>
def fe_cluster(train, test, n_clusters_g = 35, n_clusters_c = 5, SEED = 239):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
X ,X_test=fe_cluster(X,X_test)
 display(X.head(2))
print(X.shape)
 display(X_test.head(2))
print(X_test.shape )
list3=['cat_id','state_id']
for i in list3:
df.drop([i], axis=1, inplace=True )<load_pretrained>
var_thresh = VarianceThreshold(0.8)
 data = X.append(X_test)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : X.shape[0]]
test_features_transformed = data_transformed[-X_test.shape[0] : ]
X = pd.DataFrame(X[['sig_id','cp_type', 'cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 X = pd.concat([X, pd.DataFrame(train_features_transformed)], axis=1)
 X_test = pd.DataFrame(X_test[['sig_id','cp_type', 'cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 X_test = pd.concat([X_test, pd.DataFrame(test_features_transformed)], axis=1)
 display(X.head(2))
print(X.shape)
 display(X_test.head(2))
print(X_test.shape )
df = df[df['d']>=49]
df.to_pickle('data.pkl')
 del df, sales, prices, calendar
gc.collect()<load_from_csv>
y0 = y0[X['cp_type'] == 'trt_cp'].reset_index(drop = True)
 y = y[X['cp_type'] == 'trt_cp'].reset_index(drop = True)
 X = X[X['cp_type'] == 'trt_cp'].reset_index(drop = True)
 X.drop(['cp_type','sig_id'], axis=1, inplace=True)
 X_test.drop(['cp_type','sig_id'], axis=1, inplace=True)
 print('New data shape', X.shape )
data = pd.read_pickle('data.pkl')
 valid_csv=data[(data['d']>=1914)&(data['d']<1942)][['id','d','sold']]
test = data[data['d']>=1942][['id','d','sold']]
eval_preds = test['sold']
valid_preds_csv=valid_csv['sold']<define_variables>
p_min = 0.001
p_max = 0.999
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -K.mean(y_true*K.log(y_pred)+(1-y_true)*K.log(1-y_pred))
def create_model(num_cols, hid_layers, activations, dropout_rate, lr, num_cols_y):
inp1 = tf.keras.layers.Input(shape =(num_cols,))
x1 = tf.keras.layers.BatchNormalization()(inp1)
 for i, units in enumerate(hid_layers):
x1 = tf.keras.layers.Dense(units, activation=activations[i] )(x1)
 x1 = tf.keras.layers.Dropout(dropout_rate[i] )(x1)
 x1 = tf.keras.layers.BatchNormalization()(x1)
 x1 = tf.keras.layers.Dense(num_cols_y,activation='sigmoid' )(x1)
 model = tf.keras.models.Model(inputs= inp1, outputs= x1)
 model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate=lr),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.001), metrics=logloss)
 return model
cat_column=[]
for i in data.columns:
if(str(data.dtypes[i])=='category'):
cat_column.append(i )<prepare_x_and_y>
hid_layers = [[[512, 768, 896],[384, 640, 1024],[768,768,896],[512, 384, 1024],
[512, 640, 640], [640, 896, 1024], [256,640,896],[512,512,768],
[512, 384, 896],[512,768,768]],
[[512, 768, 896],[384, 640, 1024],[768,768,896],[512, 384, 1024],
[512, 640, 640], [640, 896, 1024], [256,640,896],[512,512,768],
[512, 384, 896],[512,768,768]],
[[1152, 640, 3072],[896, 1024, 3584],[1920,1024,3712],[896, 1024, 3456],
[1408, 896, 3456], [1408, 768, 3456], [2176,640,3840],[1664,1280,2688],
[1792, 768, 2432],[1280,1664,4096]],
[[896, 1792, 3712],[2048, 1024, 1664],[1664,1408,1920],[896, 1408, 3072],
[1152, 1152, 3072], [2816, 3072, 3328], [2304,2432,2176],[3968,4096,2816],
[1920, 1536, 3072],[128,1920,1664]],
[[896, 1152, 1408],[1920, 768, 2176],[2048,2048,1792],[2304, 1664, 512],
[768, 384, 512], [640, 1664, 512], [1664,1920,2688],[2432,1664,1536],
[640, 896, 2432],[1536,2176,2176]]]
dropout_rate = [[[0.65,0.35,0.35],[0.65,0.35,0.45],[0.7,0.4,0.4],[0.65,0.35,0.45],
[0.65,0.35,0.45],[0.7,0.3,0.45],[0.7,0.35,0.4],[0.7,0.4,0.4],
[0.7, 0.3, 0.4],[0.65, 0.3, 0.4]],
[[0.65,0.35,0.35],[0.65,0.35,0.45],[0.7,0.4,0.4],[0.65,0.35,0.45],
[0.65,0.35,0.45],[0.7,0.3,0.45],[0.7,0.35,0.4],[0.7,0.4,0.4],
[0.7, 0.3, 0.4],[0.65, 0.3, 0.4]],
[[0.7,0.55,0.7],[0.7,0.4,0.6],[0.7,0.5,0.55],[0.7,0.55,0.6],
[0.7,0.5,0.65],[0.7,0.5,0.65],[0.7,0.55,0.7],[0.7,0.5,0.65],
[0.7, 0.35, 0.6],[0.7, 0.5, 0.7]],
[[0.7,0.4,0.7],[0.7,0.7,0.7],[0.7,0.7,0.7],[0.7,0.25,0.7],
[0.7,0.4,0.6],[0.7,0.6,0.7],[0.7,0.5,0.7],[0.7,0.7,0.7],
[0.7, 0.45, 0.7],[0.7, 0.3, 0.6]],
[[0.7,0.7,0.45],[0.7,0.7,0.6],[0.7,0.7,0.4],[0.7,0.7,0.55],
[0.65,0.45,0.4],[0.7,0.35,0.5],[0.7,0.7,0.45],[0.7,0.6,0.6],
[0.7, 0.7, 0.25],[0.7, 0.7, 0.25]]]
activations = [[['elu', 'swish', 'selu'], ['selu','swish','selu'], ['selu','swish','selu'],['selu','swish','elu'],
['selu','swish','elu'],['elu','swish','selu'],
['selu','swish','elu'],['selu','elu','selu'],['selu','swish','selu'],
['selu','swish','elu']],
[['elu', 'swish', 'selu'], ['selu','swish','selu'], ['selu','swish','selu'],['selu','swish','elu'],
['selu','swish','elu'],['elu','swish','selu'],
['selu','swish','elu'],['selu','elu','selu'],['selu','swish','selu'],
['selu','swish','elu']],
[['selu', 'relu', 'swish'], ['selu','relu','swish'], ['selu','relu','swish'],['selu','relu','swish'],
['selu','relu','swish'],['selu','relu','swish'],['selu','relu','swish'],['selu','relu','swish'],
['selu','elu','swish'],['selu','relu','swish']],
[['selu', 'elu', 'swish'], ['elu','swish','relu'], ['elu','swish','selu'],['selu','elu','swish'],
['selu','elu','swish'],['selu','elu','swish'],['selu','elu','swish'],['selu','selu','swish'],
['selu','relu','swish'],['elu','elu','swish']],
[['selu', 'swish', 'selu'], ['selu','swish','selu'], ['elu','swish','selu'],['selu','swish','selu'],
['selu','relu','relu'],['selu','relu','relu'],['selu','swish','elu'],['selu','swish','relu'],
['elu','swish','selu'],['elu','swish','swish']]]
lr = 5e-4
feats = np.arange(0,X.shape[1],1)
 inp_size = int(np.ceil(1* len(feats)))
 res = y.copy()
df_sample.loc[:, y.columns] = 0
res.loc[:, y.columns] = 0
for store in d_store_id:
df = data[data['store_id']==store]
X_train, y_train = df[df['d']<1914].drop('sold',axis=1), df[df['d']<1914]['sold']
X_valid_csv, y_valid_csv = df[(df['d']>=1914)&(df['d']<1942)].drop('sold',axis=1), df[(df['d']>=1914)&(df['d']<1942)]['sold']
X_test = df[df['d']>=1942].drop('sold',axis=1)
 model = LGBMRegressor(
 learning_rate= 0.05,
subsample=0.6,
feature_fraction=0.6,
num_iterations = 1200,
max_bin=350,
num_leaves= 100,
lambda_l2=0.003,
max_depth=200,
min_data_in_leaf= 80,
force_row_wise= True, 
)
 print('*****Prediction for Store: {}*****'.format(d_store_id[store]))
model.fit(X_train, y_train, eval_set=[(X_train,y_train),(X_valid_csv,y_valid_csv)],
eval_metric='rmse', verbose=100, early_stopping_rounds=20,categorical_feature=cat_column)
 valid_preds_csv[X_valid_csv.index] = model.predict(X_valid_csv)
 eval_preds[X_test.index] = model.predict(X_test)
 filename = 'model'+str(d_store_id[store])+'.pkl'
joblib.dump(model, filename)
 del model, X_train, y_train, X_valid_csv, y_valid_csv
gc.collect()<prepare_output>
def callbacks() :
rlr = ReduceLROnPlateau(monitor = 'val_logloss', factor = 0.2, patience = 3, verbose = 0,
min_delta = 1e-4, min_lr = 1e-6, mode = 'min')
 ckp = ModelCheckpoint("model.h5", monitor = 'val_logloss', verbose = 0,
save_best_only = True, mode = 'min')
 es = EarlyStopping(monitor = 'val_logloss', min_delta = 1e-5, patience = 10, mode = 'min',
baseline = None, restore_best_weights = True, verbose = 0)
 return rlr, ckp, es
feature_importance_df = pd.DataFrame()
features = [f for f in data.columns if f != 'sold']
for filename in os.listdir('/kaggle/working/'):
if 'model' in filename:
model = joblib.load(filename)
 store_importance_df = pd.DataFrame()
store_importance_df["feature"] = features
store_importance_df["importance"] = model.feature_importances_
store_importance_df["store"] = filename[5:9]
feature_importance_df = pd.concat([feature_importance_df, store_importance_df], axis=0)
 def display_importances(feature_importance_df_):
cols = feature_importance_df_[["feature", "importance"]].groupby("feature" ).mean().sort_values(by="importance", ascending=False)[:].index
best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]
plt.figure(figsize=(8, 10))
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False))
plt.title('LightGBM Features(averaged over store predictions)')
 plt.tight_layout()
display_importances(feature_importance_df )<feature_engineering>
def log_loss_metric(y_true, y_pred):
metrics = []
for _target in y.columns:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels = [0,1]))
return np.mean(metrics )
valid_csv['sold'] = valid_preds_csv
validation = valid_csv[['id','d','sold']]
validation = pd.pivot(validation, index='id', columns='d', values='sold' ).reset_index()
validation.columns=['id'] + ['F' + str(i + 1)for i in range(28)]
validation.id = validation.id.map(d_id ).str.replace('evaluation','validation')
 test['sold'] = eval_preds
evaluation = test[['id','d','sold']]
evaluation = pd.pivot(evaluation, index='id', columns='d', values='sold' ).reset_index()
evaluation.columns=['id'] + ['F' + str(i + 1)for i in range(28)]
evaluation.id = evaluation.id.map(d_id)
 submit = pd.concat([validation,evaluation] ).reset_index(drop=True)
 for i in range(1,29):
submit['F'+str(i)] *= 1.04
submit.to_csv('submission.csv',index=False )<import_modules>
test_preds = []
res_preds = []
np.random.seed(seed=42)
 n_split = 5
n_top = 10
n_round = 1
for seed in range(n_round):
split_cols = np.random.choice(feats, inp_size, replace=False)
 res.loc[:, y.columns] = 0
df_sample.loc[:, y.columns] = 0
for n,(tr, te)in enumerate(MultilabelStratifiedKFold(n_splits = n_split, random_state = seed, shuffle = True ).split(X, y)) :
start_time = time()
x_tr = X.astype('float64' ).values[tr][:, split_cols]
x_val = X.astype('float64' ).values[te][:, split_cols]
y0_tr, y0_val = y0.astype(float ).values[tr], y0.astype(float ).values[te]
y_tr, y_val = y.astype(float ).values[tr], y.astype(float ).values[te]
x_tt = X_test.astype('float64' ).values[:, split_cols]
for num in range(n_top):
model = create_model(inp_size, hid_layers[n][num], activations[n][num], dropout_rate[n][num], lr, y0.shape[1])
 model.fit(x_tr, y0_tr,validation_data=(x_val, y0_val), epochs = 150, batch_size = 128,
callbacks = callbacks() , verbose = 0)
 model.load_weights("model.h5")
 model2 = create_model(inp_size, hid_layers[n][num], activations[n][num], dropout_rate[n][num], lr, y.shape[1])
 for i in range(len(model2.layers)-1):
model2.layers[i].set_weights(model.layers[i].get_weights())
 model2.fit(x_tr, y_tr,validation_data=(x_val, y_val),
epochs = 150, batch_size = 128,
callbacks = callbacks() , verbose = 0)
 model2.load_weights('model.h5')
 df_sample.loc[:, y.columns] += model2.predict(x_tt, batch_size = 128)/(n_split*n_top)
 res.loc[te, y.columns] += model2.predict(x_val, batch_size = 128)/(n_top)
 oof = log_loss_metric(y.loc[te,y.columns], res.loc[te, y.columns])
 print(f'[{str(datetime.timedelta(seconds = time() - start_time)) [2:7]}], Seed {seed}, Fold {n}:', oof)
 K.clear_session()
del model2
x = gc.collect()
df_sample.loc[ind_te, y.columns] = 0
test_preds.append(df_sample.copy())
 res_preds.append(res.copy() )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns<load_from_csv>
aa = [1.0]
res2= res.copy()
res2.loc[:, y.columns] = 0
for i in range(n_round):
res2.loc[:, y.columns] += aa[i] * res_preds[i].loc[:, y.columns]
print(log_loss_metric(y, res2))
df= pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_evaluation.csv")
 sell_prices = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv")
 calendar = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv" )<drop_column>
df_sample.loc[:, y.columns] = 0
for i in range(n_round):
df_sample.loc[:, y.columns] += aa[i] * test_preds[i].loc[:, y.columns]
df_sample.loc[ind_te, y.columns] = 0
del df,store_agg,cat_agg,agg_sales,one,x
gc.collect()<set_options>
display(df_sample.head())
 df_sample.to_csv('submission.csv', index=False )
warnings.filterwarnings('ignore' )<load_from_csv>
y_true = y
y_preds = res2
calendar = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/calendar.csv')
 calendar = reduce_mem_usage(calendar)
 sell_prices = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sell_prices.csv')
 sell_prices = reduce_mem_usage(sell_prices)
 sales_train_validation = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_evaluation.csv')
 submission = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sample_submission.csv')
 submission = reduce_mem_usage(submission )<categorify>
losses = []
for i in range(y.shape[1]):
losses.append(log_loss(y.iloc[:,i], res2.iloc[:,i]))
sales_train_validation = pd.melt(sales_train_validation,
id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'],
var_name = 'day',
value_name = 'demand')
 sales_train_validation = reduce_mem_usage(sales_train_validation )<drop_column>
max_loss_ind= np.argmax(losses)
 max_loss = np.max(losses)
 print("Max loss is", max_loss,'For index', max_loss_ind,'which is',y.iloc[:,max_loss_ind].name )