Kaggle competition: Predict Future Sales
X_test = test.values 
<compute_test_metric>
items = pd.read_csv('.. /input/items.csv')
 shops = pd.read_csv('.. /input/shops.csv')
 cats = pd.read_csv('.. /input/item_categories.csv')
 train = pd.read_csv('.. /input/sales_train.csv')
 test = pd.read_csv('.. /input/test.csv' ).set_index('ID' )
class LogitsLogLoss(Metric):
def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):
logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits + 5e-5)+ y_true * np.log(logits + 5e-5)
 return np.mean(-aux)
<set_options>
train = train[train.item_price<100000]
train = train[train.item_cnt_day<1001]
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )<choose_model_class>
median = train[(train.shop_id==32)&(train.item_id==2973)&(train.date_block_num==4)&(train.item_price>0)].item_price.median()
train.loc[train.item_price<0, 'item_price'] = median
MAX_EPOCH = 200
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = seed,
verbose = 10 
)
<find_best_model_class>
train.loc[train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
train.loc[train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
train.loc[train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11
test_cv_preds = []
NB_SPLITS = 10
mskf = MultilabelStratifiedKFold(n_splits = NB_SPLITS, random_state = 0, shuffle = True)
 SEED = [20,21,22]
for s in SEED:
tabnet_params['seed'] = s
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train, target)) :
model = TabNetRegressor()
model.load_model(f".. /input/moa-tabnet-correct/TabNet_seed_{s}_fold_{fold_nb+1}.zip")
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 test_preds_all = np.stack(test_cv_preds)
<load_from_csv>
shops.loc[shops.shop_name == 'Сергиев Посад ТЦ "7Я"', 'shop_name'] = 'СергиевПосад ТЦ "7Я"'
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
cats['split'] = cats['item_category_name'].str.split('-')
 cats['type'] = cats['split'].map(lambda x: x[0].strip())
 cats['type_code'] = LabelEncoder().fit_transform(cats['type'])
 cats['subtype'] = cats['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 cats['subtype_code'] = LabelEncoder().fit_transform(cats['subtype'])
 cats = cats[['item_category_id','type_code', 'subtype_code']]
items.drop(['item_name'], axis=1, inplace=True )
all_feat = [col for col in df.columns if col not in ["sig_id"]]
test = pd.read_csv(".. /input/lish-moa/test_features.csv")
 sig_id = test[test["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
sub3 = pd.merge(test[["sig_id"]], tmp, on = "sig_id", how = "left")
 sub3.fillna(0, inplace = True)
 sub3.head(5 )<load_from_csv>
len(list(set(test.item_id)- set(test.item_id ).intersection(set(train.item_id)))) , len(list(set(test.item_id))), len(test )
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1)
 json_file_path = '.. /input/t-test-pca-rfe-logistic-regression/main_predictors.json'
with open(json_file_path, 'r')as j:
predictors = json.loads(j.read())
 predictors = predictors['start_predictors'] 
<categorify>
ts = time.time()
matrix = []
cols = ['date_block_num','shop_id','item_id']
for i in range(34):
sales = train[train.date_block_num==i]
matrix.append(np.array(list(product([i], sales.shop_id.unique() , sales.item_id.unique())) , dtype='int16'))
matrix = pd.DataFrame(np.vstack(matrix), columns=cols)
 matrix['date_block_num'] = matrix['date_block_num'].astype(np.int8)
 matrix['shop_id'] = matrix['shop_id'].astype(np.int8)
 matrix['item_id'] = matrix['item_id'].astype(np.int16)
 matrix.sort_values(cols,inplace=True)
 time.time() - ts
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 def preprocessor(train,test):
n_gs = 10
n_cs = 90
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.StandardScaler()
train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test<compute_test_metric>
train['revenue'] = train['item_price'] * train['item_cnt_day']
n_labels = train_targets_scored.shape[1]
n_train = train_features.shape[0]
n_test = test_features.shape[0]
p_min = 0.0005
p_max = 0.9995
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred)) 
<choose_model_class>
ts = time.time()
group = train.groupby(['date_block_num','shop_id','item_id'] ).agg({'item_cnt_day': ['sum']})
 group.columns = ['item_cnt_month']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=cols, how='left')
 matrix['item_cnt_month'] =(matrix['item_cnt_month'] 
.fillna(0)
.clip(0,20)
.astype(np.float16))
time.time() - ts
def build_model(n_features, n_features_2, n_labels, label_smoothing = 0.0005):
input_1 = layers.Input(shape =(n_features,), name = 'Input1')
 input_2 = layers.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.2),
layers.Dense(512, activation="elu"),
layers.BatchNormalization() ,
layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = layers.Concatenate()([input_2, input_3])
 head_2 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.3),
layers.Dense(512, "relu"),
layers.BatchNormalization() ,
layers.Dense(512, "elu"),
layers.BatchNormalization() ,
layers.Dense(256, "relu"),
layers.BatchNormalization() ,
layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = layers.Average()([input_3, input_4])
 head_3 = Sequential([
layers.BatchNormalization() ,
layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = Model(inputs = [input_1, input_2], outputs = output)
 model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(label_smoothing=label_smoothing), metrics=logloss)
 return model 
<prepare_x_and_y>
test['date_block_num'] = 34
test['date_block_num'] = test['date_block_num'].astype(np.int8)
 test['shop_id'] = test['shop_id'].astype(np.int8)
 test['item_id'] = test['item_id'].astype(np.int16 )
n_seeds = 7
np.random.seed(1)
 n_folds = 10
seeds = np.random.randint(0,100,size=n_seeds)
 y_pred = np.zeros(( n_test,n_labels))
""
for seed in seeds:
fold = 0
kf = KFold(n_splits=n_folds,shuffle=True,random_state=seed)
 for train, test in kf.split(train_features):
X_train, X_test = preprocessor(train_features.iloc[train].values,
train_features.iloc[test].values)
 _,data_test = preprocessor(train_features.iloc[train].values,
test_features.drop('cp_type',axis=1 ).values)
 X_train_2 = train_features.iloc[train][predictors].values
X_test_2 = train_features.iloc[test][predictors].values
data_test_2 = test_features[predictors].values
y_train = labels_train[train]
y_test = labels_train[test]
n_features = X_train.shape[1]
n_features_2 = X_train_2.shape[1]
model = build_model(n_features, n_features_2, n_labels)
 model.load_weights('.. /input/resnet-model/TwoHeads_seed_'+str(seed)+'_fold_'+str(fold)+'.h5')
 y_pred += model.predict([data_test,data_test_2])/(n_folds*n_seeds)
 fold += 1 
<load_from_csv>
ts = time.time()
matrix = pd.concat([matrix, test], ignore_index=True, sort=False, keys=cols)
 matrix.fillna(0, inplace=True)
 time.time() - ts
sub4 = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub4.iloc[:,1:] = y_pred
sub4.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0<prepare_output>
ts = time.time()
matrix = pd.merge(matrix, shops, on=['shop_id'], how='left')
 matrix = pd.merge(matrix, items, on=['item_id'], how='left')
 matrix = pd.merge(matrix, cats, on=['item_category_id'], how='left')
 matrix['city_code'] = matrix['city_code'].astype(np.int8)
 matrix['item_category_id'] = matrix['item_category_id'].astype(np.int8)
 matrix['type_code'] = matrix['type_code'].astype(np.int8)
 matrix['subtype_code'] = matrix['subtype_code'].astype(np.int8)
 time.time() - ts
submission = sub.copy()<feature_engineering>
def lag_feature(df, lags, col):
tmp = df[['date_block_num','shop_id','item_id',col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
 return df
submission.iloc[:, 1:] =(sub.iloc[:, 1:] + sub2.iloc[:, 1:]+ sub3.iloc[:, 1:]+ sub4.iloc[:, 1:])/4<feature_engineering>
ts = time.time()
group = matrix.groupby(['date_block_num'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num'], how='left')
 matrix['date_avg_item_cnt'] = matrix['date_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_avg_item_cnt')
 matrix.drop(['date_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
submission.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0<save_to_csv>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_cnt'] = matrix['date_item_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_item_avg_item_cnt')
 matrix.drop(['date_item_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
submission.to_csv("submission.csv",index=False )<load_from_csv>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_shop_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_avg_item_cnt'] = matrix['date_shop_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1,2,3,6,12], 'date_shop_avg_item_cnt')
 matrix.drop(['date_shop_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
pd.set_option('display.max_rows', 50)
 pd.set_option('display.max_columns', 50)
 KAGGLE = True
TRAIN = True
INFERENCE = True
PATH = '.. /input/lish-moa/' if KAGGLE else None
print(PATH)
 test_features = pd.read_csv(PATH + 'test_features.csv')
 train_features = pd.read_csv(PATH + 'train_features.csv')
 train_targets_scored = pd.read_csv(PATH + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(PATH + 'train_targets_nonscored.csv')
 sample_submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 drug_ids = pd.read_csv(PATH + 'train_drug.csv' )<set_options>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_cat_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_category_id'], how='left')
 matrix['date_cat_avg_item_cnt'] = matrix['date_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_cat_avg_item_cnt')
 matrix.drop(['date_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<define_variables>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'item_category_id'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_cat_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'item_category_id'], how='left')
 matrix['date_shop_cat_avg_item_cnt'] = matrix['date_shop_cat_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_cat_avg_item_cnt')
 matrix.drop(['date_shop_cat_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
genes_cols = [col for col in train_features.columns if col.startswith('g-')]
cells_cols = [col for col in train_features.columns if col.startswith('c-')] 
<train_model>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_type_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'type_code'], how='left')
 matrix['date_shop_type_avg_item_cnt'] = matrix['date_shop_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_type_avg_item_cnt')
 matrix.drop(['date_shop_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
train_and_test_genes_features = pd.concat([train_features[genes_cols],test_features[genes_cols]])
 train_and_test_cell_features = pd.concat([train_features[cells_cols],test_features[cells_cols]])
 n_comp_genes = 600
pca_genes = PCA(n_components=n_comp_genes)
 pca_names_genes = ['pca_genes_' + str(i)for i in range(300)]
pca_genes.fit(train_and_test_genes_features)
 print(pca_genes.explained_variance_ratio_[:300].sum())
 n_comp_cells = 60
pca_cells = PCA(n_components=n_comp_cells)
 pca_names_cells = ['pca_cells_' + str(i)for i in range(n_comp_cells)]
pca_cells.fit(train_and_test_cell_features)
 print(pca_cells.explained_variance_ratio_.sum())
 train_pca_features_genes = pca_genes.transform(train_features[genes_cols])[:, :300]
test_pca_features_genes = pca_genes.transform(test_features[genes_cols])[:,:300]
train_features[pca_names_genes] = train_pca_features_genes
test_features[pca_names_genes] = test_pca_features_genes
train_pca_features_cells= pca_cells.transform(train_features[cells_cols])
 test_pca_features_cells = pca_cells.transform(test_features[cells_cols])
 train_features[pca_names_cells] = train_pca_features_cells
test_features[pca_names_cells] = test_pca_features_cells 
<merge>
ts = time.time()
group = matrix.groupby(['date_block_num', 'shop_id', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = ['date_shop_subtype_avg_item_cnt']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'shop_id', 'subtype_code'], how='left')
 matrix['date_shop_subtype_avg_item_cnt'] = matrix['date_shop_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_shop_subtype_avg_item_cnt')
 matrix.drop(['date_shop_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
target_scored_cols = train_targets_scored.columns.tolist() [1:]
target_nonscored_cols = train_targets_nonscored.columns.tolist() [1:]
train_df = train_features.merge(train_targets_scored,on='sig_id',how='left' ).merge(
 train_targets_nonscored,on='sig_id',how='left' ).merge(
 drug_ids,on='sig_id',how='left')
 df = train_df.sample(frac=1.,random_state=2020)
 df['kfold_scored'] = -1
kf = MultilabelStratifiedKFold(n_splits=5)
 y = df[target_scored_cols + ['drug_id']].values
for fold,(t_,v_)in enumerate(kf.split(X=df,y=y)) :
df.loc[v_,'kfold_scored'] = fold
df['kfold_nonscored'] = -1
kf = MultilabelStratifiedKFold(n_splits=5)
 y = df[target_nonscored_cols + target_scored_cols + ['drug_id']].values
for fold,(t_,v_)in enumerate(kf.split(X=df,y=y)) :
df.loc[v_,'kfold_nonscored'] = fold 
<define_variables>
ts = time.time()
group = matrix.groupby(['date_block_num', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'city_code'], how='left')
 matrix['date_city_avg_item_cnt'] = matrix['date_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_city_avg_item_cnt')
 matrix.drop(['date_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
test_sig_ids = test_features[test_features['cp_type'] == 'ctl_vehicle']['sig_id'].values
cat_names = ['cp_type', 'cp_time', 'cp_dose']
cont_names = [c for c in train_features.columns if c not in cat_names
and c != 'sig_id'
and c != 'drug_id']<categorify>
ts = time.time()
group = matrix.groupby(['date_block_num', 'item_id', 'city_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_item_city_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'item_id', 'city_code'], how='left')
 matrix['date_item_city_avg_item_cnt'] = matrix['date_item_city_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_item_city_avg_item_cnt')
 matrix.drop(['date_item_city_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
class LabelSmoothingCrossEntropyEgm(nn.Module):
y1 = None
lam = None
def __init__(self, smoothing:float=0.005, reduction='mean'):
super().__init__()
self.smoothing,self.reduction = smoothing,reduction
self.f = BCELoss(reduction = 'none')
 def forward(self, output, target):
c = output.size() [-1]
target=target.float()
with torch.no_grad() :
target = target *(1.0 - self.smoothing)+ 0.08 * self.smoothing
bce_loss = self.f(output, target)
 if(self.y1 == None):
loss = bce_loss.mean()
return loss
else:
target1=self.y1.float()
with torch.no_grad() :
target1 = target1 *(1.0 - self.smoothing)+ 0.08 * self.smoothing
bce_loss1 = self.f(output, target1)
 all_loss = torch.lerp(bce_loss, bce_loss1, self.lam)
 return all_loss.mean()
class ChangeLoss(Callback):
_order = 90
valid_loss = BCELossFlat()
train_loss = LabelSmoothingCrossEntropyEgm()
def before_batch(self, **kwargs):
val_condition =(self.learn.dls[1] == self.learn.dl)
 if(val_condition):
self.learn.loss_func = self.valid_loss
else:
self.learn.loss_func = self.train_loss
class NormalizeCallback(Callback):
def before_batch(self, **kwargs):
a_cat, a_cont = self.learn.xb
x_cont = a_cont.cpu()
x_cont =(a_cont.cpu() - mean)/std
x_cont = x_cont.float()
x_cont = x_cont.to(a_cont.device)
 self.learn.xb =(a_cat, x_cont)
 a, b = None,None
class CatMixUp(Callback):
run_before = [Normalize]
def __init__(self, alpha=0.4): self.distrib = Beta(tensor(alpha), tensor(alpha))
def before_batch(self, **kwargs):
ret_condition =(self.learn.dls[0] != self.learn.dl)
 if(ret_condition):
return
a=self.learn.xb
b, =self.learn.yb
lam = self.distrib.sample(( self.y.size(0),)).unsqueeze(-1)
 a_cat, a_cont = a
x_cat = a_cat.detach().clone()
x_cont = a_cont.detach().clone()
y=b.detach().clone()
codes = x_cat[:,0]*16 + x_cat[:,1]*4 +x_cat[:,2]
uniq_codes = torch.unique(codes)
 for code in uniq_codes:
indexes =(codes==code ).nonzero().view(-1)
 ind_perm = torch.randperm(len(indexes))
x_cont[indexes] = x_cont[indexes[ind_perm]]
y[indexes] = y[indexes[ind_perm]]
x_cont = x_cont.to(a_cont.device)
 y=y.to(a_cont.device)
 lam=lam.to(a_cont.device)
 out_cont = torch.lerp(a_cont, x_cont, lam)
 out_cont[:,772+100:772+100+300]=torch.tensor(pca_genes.transform(out_cont[:,:772].tolist())[:,:300])
 out_cont[:,-60:]=torch.tensor(pca_cells.transform(out_cont[:, 772:772+100].tolist()))
out_y = torch.lerp(b.float() , y.float() , lam)
 out_cont = out_cont.to(a_cont.device)
 out_y = out_y.to(b.device)
 self.learn.loss_func.lam = lam
self.learn.loss_func.y1 = y
self.learn.xb =(a_cat, out_cont)
 self.learn.yb =(out_y,)
 seeds = [42, 7, 9, 13, 37, 11, 5, 29, 31, 37, 41, 53]
BCE_LOSS = BCELoss(reduction='mean' )<load_pretrained>
ts = time.time()
group = matrix.groupby(['date_block_num', 'type_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_type_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'type_code'], how='left')
 matrix['date_type_avg_item_cnt'] = matrix['date_type_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_type_avg_item_cnt')
 matrix.drop(['date_type_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
def get_data(fold, target_names,
procs = [Categorify, FillMissing],
cat_names = cat_names,
cont_names = cont_names):
val_idx = df[df.kfold_nonscored==fold].index
dls = TabularDataLoaders.from_df(df, path=PATH,
y_names=target_names,
cat_names = cat_names,
cont_names = cont_names,
procs = procs,
valid_idx=val_idx,
bs=64)
 return dls
def get_cbs(do_mixup):
ncb = NormalizeCallback()
ch_loss_cb = ChangeLoss()
if do_mixup:
return [ch_loss_cb,CatMixUp() ,ncb]
return [ncb, ch_loss_cb]
test_scores = []
results = []
def do_train_and_inf(num_iters=1, do_train=True, do_inference=False, cbs=get_cbs(False),
lr=9e-3, epochs=5, target_names=target_scored_cols,
use_pretr=False, pretr_model=False, file_name = 'something'):
global test_scores
global results
model = None
for ind in tqdm(range(num_iters)) :
seed_everything(seeds[ind])
 i = ind % 5
dls = get_data(i, target_names = target_names)
 model_dir = '/kaggle/working/' if TRAIN else '/kaggle/input/fastai-egm'
config = tabular_config(ps=0.2)
 learn = tabular_learner(dls , y_range=(0,1),
layers = [1024, 512, 512, 256],
loss_func = LabelSmoothingCrossEntropyEgm() ,
config=config,
model_dir=model_dir,
cbs=cbs 
)
 model = learn.model
if(use_pretr):
print("Will change model")
 remember = learn.model.layers[-2]
learn.model = copy.deepcopy(pretr_model)
 learn.model.layers[-2] = remember
name = file_name + str(ind)
 cb = SaveModelCallback(monitor='valid_loss',fname=name ,mode='min')
 if(do_train):
learn.fit_one_cycle(epochs, lr=slice(lr/(2.6**4),lr), cbs=cb)
 results = results + [learn.recorder.loss.value.item() ]
if(do_inference):
learn.load(name)
 test_dl = learn.dls.test_dl(test_features)
 sub = learn.get_preds(dl=test_dl)
 test_scores.append(sub[0].numpy())
 return model 
<train_model>
ts = time.time()
group = matrix.groupby(['date_block_num', 'subtype_code'] ).agg({'item_cnt_month': ['mean']})
 group.columns = [ 'date_subtype_avg_item_cnt' ]
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num', 'subtype_code'], how='left')
 matrix['date_subtype_avg_item_cnt'] = matrix['date_subtype_avg_item_cnt'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'date_subtype_avg_item_cnt')
 matrix.drop(['date_subtype_avg_item_cnt'], axis=1, inplace=True)
 time.time() - ts
model = do_train_and_inf(num_iters=1, do_train = True, do_inference=False,
cbs = get_cbs(False), lr = 9e-3, epochs = 5,
target_names = target_nonscored_cols + target_scored_cols,
use_pretr = False, pretr_model = None,
file_name = 'pretrain_')
<train_model>
ts = time.time()
group = train.groupby(['item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['item_id'], how='left')
 matrix['item_avg_item_price'] = matrix['item_avg_item_price'].astype(np.float16)
 group = train.groupby(['date_block_num','item_id'] ).agg({'item_price': ['mean']})
 group.columns = ['date_item_avg_item_price']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','item_id'], how='left')
 matrix['date_item_avg_item_price'] = matrix['date_item_avg_item_price'].astype(np.float16)
 lags = [1,2,3,4,5,6]
matrix = lag_feature(matrix, lags, 'date_item_avg_item_price')
 for i in lags:
matrix['delta_price_lag_'+str(i)] = \ 
(matrix['date_item_avg_item_price_lag_'+str(i)] - matrix['item_avg_item_price'])/ matrix['item_avg_item_price']
def select_trend(row):
for i in lags:
if row['delta_price_lag_'+str(i)]:
return row['delta_price_lag_'+str(i)]
return 0
matrix['delta_price_lag'] = matrix.apply(select_trend, axis=1)
 matrix['delta_price_lag'] = matrix['delta_price_lag'].astype(np.float16)
 matrix['delta_price_lag'].fillna(0, inplace=True)
 fetures_to_drop = ['item_avg_item_price', 'date_item_avg_item_price']
for i in lags:
fetures_to_drop += ['date_item_avg_item_price_lag_'+str(i)]
fetures_to_drop += ['delta_price_lag_'+str(i)]
matrix.drop(fetures_to_drop, axis=1, inplace=True)
 time.time() - ts
do_train_and_inf(num_iters=10, do_train = True, do_inference=True,
cbs = get_cbs(True), lr = 9e-3, epochs = 10,
target_names = target_scored_cols,
use_pretr = True, pretr_model = model,
file_name = 'pretrain_')
 test_sc = np.array(test_scores )<define_variables>
ts = time.time()
group = train.groupby(['date_block_num','shop_id'] ).agg({'revenue': ['sum']})
 group.columns = ['date_shop_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['date_block_num','shop_id'], how='left')
 matrix['date_shop_revenue'] = matrix['date_shop_revenue'].astype(np.float32)
 group = group.groupby(['shop_id'] ).agg({'date_shop_revenue': ['mean']})
 group.columns = ['shop_avg_revenue']
group.reset_index(inplace=True)
 matrix = pd.merge(matrix, group, on=['shop_id'], how='left')
 matrix['shop_avg_revenue'] = matrix['shop_avg_revenue'].astype(np.float32)
 matrix['delta_revenue'] =(matrix['date_shop_revenue'] - matrix['shop_avg_revenue'])/ matrix['shop_avg_revenue']
matrix['delta_revenue'] = matrix['delta_revenue'].astype(np.float16)
 matrix = lag_feature(matrix, [1], 'delta_revenue')
 matrix.drop(['date_shop_revenue','shop_avg_revenue','delta_revenue'], axis=1, inplace=True)
 time.time() - ts
for r in results:
print(str(r ).replace('.',','))<feature_engineering>
matrix['month'] = matrix['date_block_num'] % 12
avg_prds = test_sc.mean(axis=0)
 submission = sample_submission.copy()
submission[target_scored_cols] = avg_prds
submission.loc[submission['sig_id'].isin(test_features.loc[test_features['cp_type'] =='ctl_vehicle', 'sig_id']), train_targets_scored.columns[1:]] = 0
submission.to_csv('submission.csv', index=False )<install_modules>
days = pd.Series([31,28,31,30,31,30,31,31,30,31,30,31])
 matrix['days'] = matrix['month'].map(days ).astype(np.int8 )
!pip install -r /kaggle/input/python-library/wheelhouse/requirements.txt --no-index --find-links /kaggle/input/python-library/wheelhouse<import_modules>
ts = time.time()
cache = {}
matrix['item_shop_last_sale'] = -1
matrix['item_shop_last_sale'] = matrix['item_shop_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = str(row.item_id)+' '+str(row.shop_id)
 if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
matrix.at[idx, 'item_shop_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts
print(np.__version__)
 print(pd.__version__)
 print(torch.__version__)
 print(torch_optimizer.__version__)
 print(iterstrat.__version__ )<define_variables>
ts = time.time()
cache = {}
matrix['item_last_sale'] = -1
matrix['item_last_sale'] = matrix['item_last_sale'].astype(np.int8)
 for idx, row in matrix.iterrows() :
key = row.item_id
if key not in cache:
if row.item_cnt_month!=0:
cache[key] = row.date_block_num
else:
last_date_block_num = cache[key]
if row.date_block_num>last_date_block_num:
matrix.at[idx, 'item_last_sale'] = row.date_block_num - last_date_block_num
cache[key] = row.date_block_num
time.time() - ts
SEED = 42
SEEDS = [4, SEED, 6, 89]
def set_seed(seed):
os.environ['PYTHONHASHSEED']=str(seed)
 random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 set_seed(SEED)
 N_EPOCHS = 250
N_FOLDS = 5
BATCH_SIZE = 128
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 device<load_from_csv>
ts = time.time()
matrix['item_shop_first_sale'] = matrix['date_block_num'] - matrix.groupby(['item_id','shop_id'])['date_block_num'].transform('min')
 matrix['item_first_sale'] = matrix['date_block_num'] - matrix.groupby('item_id')['date_block_num'].transform('min')
 time.time() - ts
X = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 y = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 X_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv' )<create_dataframe>
ts = time.time()
matrix = matrix[matrix.date_block_num > 11]
time.time() - ts
df_submission = pd.DataFrame(columns=y.columns)
 df_submission['sig_id'] = X_test['sig_id']<drop_column>
ts = time.time()
def fill_na(df):
for col in df.columns:
if('_lag_' in col)&(df[col].isnull().any()):
if('item_cnt' in col):
df[col].fillna(0, inplace=True)
 return df
matrix = fill_na(matrix)
 time.time() - ts
del X['sig_id']
del y['sig_id']
del X_test['sig_id']<feature_engineering>
matrix.to_pickle('data.pkl')
 del matrix
del cache
del group
del items
del shops
del cats
del train
gc.collect() ;
cat_feature = ['cp_type', 'cp_dose']
for cat in cat_feature:
val_list = X[cat].unique()
for val in val_list:
X[f'{cat}_{val}'] = X[cat].apply(lambda i:i == val)
 X_test[f'{cat}_{val}'] = X_test[cat].apply(lambda i:i == val)
 del X[cat]
del X_test[cat]
print(X.columns[1:773])
 print(X.columns[773:873] )<data_type_conversions>
data = pd.read_pickle('data.pkl' )
X = torch.tensor(X.astype(np.float32 ).to_numpy())
 y = torch.tensor(y.astype(np.float32 ).to_numpy())
 X_test = torch.tensor(X_test.astype(np.float32 ).to_numpy() )<prepare_x_and_y>
data = data[[
'date_block_num',
'shop_id',
'item_id',
'item_cnt_month',
'city_code',
'item_category_id',
'type_code',
'subtype_code',
'item_cnt_month_lag_1',
'item_cnt_month_lag_2',
'item_cnt_month_lag_3',
'item_cnt_month_lag_6',
'item_cnt_month_lag_12',
'date_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_1',
'date_item_avg_item_cnt_lag_2',
'date_item_avg_item_cnt_lag_3',
'date_item_avg_item_cnt_lag_6',
'date_item_avg_item_cnt_lag_12',
'date_shop_avg_item_cnt_lag_1',
'date_shop_avg_item_cnt_lag_2',
'date_shop_avg_item_cnt_lag_3',
'date_shop_avg_item_cnt_lag_6',
'date_shop_avg_item_cnt_lag_12',
'date_cat_avg_item_cnt_lag_1',
'date_shop_cat_avg_item_cnt_lag_1',
'date_city_avg_item_cnt_lag_1',
'date_item_city_avg_item_cnt_lag_1',
'delta_price_lag',
'month',
'days',
'item_shop_last_sale',
'item_last_sale',
'item_shop_first_sale',
'item_first_sale',
]]
class MoADataset(torch.utils.data.Dataset):
def __init__(self, X, y=None):
if y is not None:
assert X.shape[0] == y.shape[0]
self.X = X
self.y = y
def __len__(self):
return self.X.shape[0]
def __getitem__(self, idx):
if self.y is not None:
return self.X[idx], self.y[idx]
else:
return self.X[idx]<create_dataframe>
X_train = data[data.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = data[data.date_block_num < 33]['item_cnt_month']
X_valid = data[data.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = data[data.date_block_num == 33]['item_cnt_month']
X_test = data[data.date_block_num == 34].drop(['item_cnt_month'], axis=1 )
train_ds = MoADataset(X, y)
 test_ds = MoADataset(X_test )<choose_model_class>
del data
gc.collect() ;
def train_ae(mode, whole_dl):
train_loss = 0
lowest_loss = np.Inf
last_epoch_with_lowest_loss = 0
if mode == 'AE_Gene':
print('Train autoencoder gene!')
 net = AE_Gene_Net().float().to(device)
 criterion = nn.MSELoss()
optimizer = torch_optimizer.RAdam(net.parameters() , lr=0.002, weight_decay=0.0000125)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
 optimizer=optimizer, factor=0.5, patience=8, min_lr = 0.00005, verbose=True 
)
 else:
print('Train autoencoder cell!')
 net = AE_Cell_Net().float().to(device)
 criterion = nn.MSELoss()
optimizer = torch_optimizer.RAdam(net.parameters() , lr=0.002, weight_decay=0.00001)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
 optimizer=optimizer, factor=0.25, patience=11, min_lr = 0.000025, verbose=True 
)
 net.train()
for epoch in range(N_EPOCHS):
for X_local in whole_dl:
X_local = X_local.to(device)
 optimizer.zero_grad()
output = net(X_local)
 loss = criterion(output, X_local)
 loss.backward()
optimizer.step()
train_loss += loss.item()
train_loss /= len(whole_dl)
 scheduler.step(train_loss)
 print(f'Epoch: {epoch+1:02d}/{N_EPOCHS:02d} | Train loss: {train_loss:.08f}{"(Saving model state!)" if train_loss < lowest_loss else ""}')
 if train_loss < lowest_loss:
torch.save(net.state_dict() , f'./model_{mode}.state_dict')
 lowest_loss = train_loss
last_epoch_with_lowest_loss = epoch
if last_epoch_with_lowest_loss + 50 < epoch:
print('Early stopping!')
 break
train_loss = 0
if mode == 'AE_Gene':
net = AE_Gene_Net().float().to(device)
 else:
net = AE_Cell_Net().float().to(device)
 net.load_state_dict(torch.load(f'./model_{mode}.state_dict'))
return net, lowest_loss<import_modules>
ts = time.time()
model = XGBRegressor(
 max_depth=8,
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 10)
 time.time() - ts
class AE_Gene_Net(nn.Module):
def __init__(self):
super().__init__()
self.encoder = nn.Sequential(
 nn.BatchNorm1d(772),
nn.utils.weight_norm(nn.Linear(772, 541)) ,
nn.ReLU6() ,
nn.BatchNorm1d(541),
nn.utils.weight_norm(nn.Linear(541, 309)) ,
nn.ReLU6() , 
)
 self.decoder = nn.Sequential(
 nn.BatchNorm1d(309),
nn.utils.weight_norm(nn.Linear(309, 541)) ,
nn.ReLU6() ,
nn.BatchNorm1d(541),
nn.utils.weight_norm(nn.Linear(541, 772)) , 
)
 def forward(self, x, mode='reconstruct'):
x = self.encoder(x)
 if mode == 'reconstruct':
x = self.decoder(x)
 return x
ae_gene_net = AE_Gene_Net().float().to(device)
 print(ae_gene_net)
 summary(ae_gene_net,(772,))<categorify>
Y_pred = model.predict(X_valid ).clip(0, 20)
 Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False)
 pickle.dump(Y_pred, open('xgb_train.pickle', 'wb'))
pickle.dump(Y_test, open('xgb_test.pickle', 'wb'))