Kaggle competition: Predict Future Sales
scores_auc_all = []
test_cv_preds = []
oof_preds = []
oof_targets = []
scores = []
scores_auc = []
oof_all = np.zeros(( train_df.shape[0], targets[target_cols].shape[1]))
for s in SEEDS:
oof = np.zeros(( train_df.shape[0], targets[target_cols].shape[1]))
tabnet_params['seed'] = s
for fold_id in range(NFOLDS):
print(b_,"FOLDS: ", r_, fold_id + 1, y_, 'seed:', tabnet_params['seed'])
 print(g_, '*' * 60, c_)
 kfold_col = f'kfold_{s}'
trn_idx = train_df[train_df[kfold_col] != fold_id].index
val_idx = train_df[train_df[kfold_col] == fold_id].index
X_train, y_train = train_df[feature_cols].values[trn_idx, :], targets[target_cols].values[trn_idx, :]
X_val, y_val = train_df[feature_cols].values[val_idx, :], targets[target_cols].values[val_idx, :]
saved_path_name = '.. /input/tabnetfe7seeds424810folds/TabNet_seed_'\
+str(s)+'_fold_'+str(fold_id+1)
 saved_file = '/kaggle/working/model'
shutil.make_archive(saved_file,'zip',saved_path_name)
 print(f'Successfully saved model at {saved_file}.zip')
 with zipfile.ZipFile('model.zip')as z:
with z.open('model_params.json')as f:
loaed_params = json.load(f)
 with z.open('network.pt')as f:
try:
saved_state_dict = torch.load(f, map_location=DEVICE)
 except io.UnsupportedOperation:
saved_state_dict = torch.load(io.bytesIO(f.read()),map_location = DEVICE)
 loaded_model = TabNetRegressor(**loaed_params)
 loaded_model._set_network()
loaded_model.network.load_state_dict(saved_state_dict)
 loaded_model.network.eval()
preds_val = loaded_model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
oof[val_idx] = preds
oof_preds.append(preds_val)
 oof_targets.append(y_val)
 preds_test = loaded_model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_all += oof
oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds)
 oof_all = oof_all/len(SEEDS)
 saved_path_name = '.. /input/tabnetfe7seeds424810folds/'
oof_TabNet_all = np.load(saved_path_name + 'oof_TabNet_all.npy')
 print(oof_TabNet_all.shape )<compute_test_metric>
sub_path_to_data = '/kaggle/input/competitive-data-science-predict-future-sales/'
categories = pd.read_csv(sub_path_to_data + 'item_categories.csv')
 items = pd.read_csv(sub_path_to_data + 'items.csv')
 sales_train = pd.read_csv(sub_path_to_data + 'sales_train.csv')
 shops = pd.read_csv(sub_path_to_data + 'shops.csv')
 test = pd.read_csv(sub_path_to_data + 'test.csv' )
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true = oof_targets_all[:, task_id],
y_score = oof_preds_all[:, task_id] 
))
print(f"{b_}Overall AUC: {r_}{np.mean(aucs)}")
<save_to_csv>
sales_train.isnull().sum()
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
test = pd.read_csv(data_path + "test_features.csv")
 sig_id = test[test["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission = pd.merge(test[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission.fillna(0, inplace = True)
 submission.to_csv("submission.csv", index = None)
 submission.head()<save_model>
duplicate_sales = sales_train.loc[sales_train.duplicated(keep=False)]
np.save('oof_TabNet_no_ctrl.npy', oof_all )<load_from_csv>
sales_train.loc[sales_train.duplicated(subset= ['date', 'shop_id', 'item_id'], keep=False)&\
~sales_train.duplicated(keep=False)]
train_targets_scored = pd.read_csv(data_path + "train_targets_scored.csv")
 target_cols = [col for col in train_targets_scored.columns if col != 'sig_id']<load_from_csv>
sales_train.loc[sales_train.duplicated(subset= ['date', 'shop_id', 'item_id', 'item_price'], keep=False)& ~sales_train.duplicated(keep=False)]
train = pd.read_csv(data_path + 'train_features.csv')
 print(train.shape)
 train_no_ctrl = train[train["cp_type"] != "ctl_vehicle"].reset_index(drop=True)
 print(train_no_ctrl.shape)
 train_pred = pd.DataFrame(oof_all, columns = target_cols)
 print(train_pred.shape)
 train_pred = pd.concat([train_no_ctrl[['sig_id']], train_pred], axis=1 ).reset_index(drop=True)
 print(train_pred.shape)
 display(train_pred.head() )<merge>
sales_train.drop_duplicates(keep="first", inplace=True )
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train_pred, on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
np.save('oof_TabNet_all.npy', y_pred )<compute_test_metric>
sales_train.loc[sales_train.shop_id == 0, 'shop_id'] = 57
test.loc[test.shop_id == 0, 'shop_id'] = 57
sales_train.loc[sales_train.shop_id == 1, 'shop_id'] = 58
test.loc[test.shop_id == 1, 'shop_id'] = 58
sales_train.loc[sales_train.shop_id == 10, 'shop_id'] = 11
test.loc[test.shop_id == 10, 'shop_id'] = 11
def log_loss_numpy(y_pred, y_true):
y_true_ravel = np.asarray(y_true ).ravel()
y_pred = np.asarray(y_pred ).ravel()
y_pred = np.clip(y_pred, 1e-15, 1 - 1e-15)
 loss = np.where(y_true_ravel == 1, - np.log(y_pred), - np.log(1 - y_pred))
return loss.mean()<compute_test_metric>
sales_train.loc[~sales_train['date'].str.match('^[0-3]\d\.[0-1]\d\.20\d\d$')]
y_true = train_targets_scored[target_cols].values
log_loss_numpy(y_pred, y_true )<set_options>
sales_train['date'] = pd.to_datetime(sales_train['date'], format='%d.%m.%Y' )
warnings.filterwarnings('ignore' )<install_modules>
sales_sum = sales_train.groupby(['date'])['item_cnt_day'].sum().reset_index().set_index("date" )
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/ 
<set_options>
sales_sum.loc[sales_sum.item_cnt_day > 7000]
seed = 42
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(seed )<load_from_csv>
sales_train.loc[sales_train.item_price > 50000]
def preprocessor_nn_transfer() :
variance_threshould = 0.8
ncompo_genes = 600
ncompo_cells = 50
data_dir = '.. /input/lish-moa/'
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 train_features = train_features[train_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test_features = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 train_features = train_features.drop('cp_type', axis=1)
 test_features = test_features.drop('cp_type', axis=1)
 GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
pca_genes = PCA(n_components = ncompo_genes, random_state = seed)
 pca_genes_train = pca_genes.fit_transform(train_features[GENES])
 pca_cells = PCA(n_components = ncompo_cells, random_state = seed)
 pca_cells_train = pca_cells.fit_transform(train_features[CELLS])
 pca_genes_train = pd.DataFrame(pca_genes_train, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_train = pd.DataFrame(pca_cells_train, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 train_features = pd.concat([train_features, pca_genes_train, pca_cells_train], axis = 1)
 pca_genes_test = pca_genes.transform(test_features[GENES])
 pca_cells_test = pca_cells.transform(test_features[CELLS])
 pca_genes_test = pd.DataFrame(pca_genes_test, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_test = pd.DataFrame(pca_cells_test, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 test_features = pd.concat([test_features, pca_genes_test, pca_cells_test], axis = 1)
 PCA')
 cols_numeric = [feat for feat in list(train_features.columns)if feat not in ["sig_id", "cp_time", "cp_dose"]]
mask =(train_features[cols_numeric].var() >= variance_threshould ).values
tmp = train_features[cols_numeric].loc[:, mask]
train_features = pd.concat([train_features[["sig_id", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(train_features.columns)if feat not in ["sig_id", "cp_time", "cp_dose"]]
test_features = pd.concat([test_features[["sig_id", "cp_time", "cp_dose"]], test_features.loc[:,cols_numeric]], axis = 1)
 train_features = pd.get_dummies(train_features, columns = ['cp_time', 'cp_dose'])
 test_features = pd.get_dummies(test_features, columns = ['cp_time', 'cp_dose'])
 Feature selection')
 train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 test = test_features
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577',
'g-153','g-389','g-60','g-370','g-248','g-167',
'g-203','g-177','g-301','g-332','g-517','g-6',
'g-744','g-224','g-162','g-3','g-736','g-486',
'g-283','g-22','g-359','g-361','g-440','g-335',
'g-106','g-307','g-745','g-146','g-416','g-298',
'g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
for df in [train, test]:
df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in gsquarecols:
if feature in GENES:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in CELLS:
df[f'{feature}_squared'] = df[feature] ** 2
Join')
 target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 num_targets: {}'.format(num_targets))
')
 return train, test, num_targets, num_aux_targets, num_all_targets, target_cols, aux_target_cols, all_target_cols<split>
sales_train = sales_train.loc[sales_train.item_price < 50000]
train, test, num_targets, num_aux_targets, num_all_targets, target_cols, aux_target_cols, all_target_cols = preprocessor_nn_transfer()<categorify>
train = sales_train.groupby(['date_block_num', 'shop_id', 'item_id'])['item_cnt_day'].sum().reset_index()
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct<train_model>
train.rename(columns={"item_cnt_day": "item_cnt_month"}, inplace=True )
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds<normalization>
train['item_cnt_month'] = train['item_cnt_month'].fillna(0 ).clip(0,20 )
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss<choose_model_class>
shop_ids_test = test.shop_id.unique()
for shop_id in [0,1,11,20,8]:
print(shop_id in shop_ids_test )
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x<choose_model_class>
test['date_block_num'] = 34
train = pd.concat([train, test.drop('ID', axis=1)], ignore_index=True, sort=False, keys=['date_block_num', 'shop_id', 'item_id'])
 train.fillna(0, inplace=True )
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]<define_variables>
index_cols = ['shop_id', 'item_id', 'date_block_num']
grid = []
for block_num in train.date_block_num.unique() :
cur_shops = train[train['date_block_num']==block_num]['shop_id'].unique()
cur_items = train[train['date_block_num']==block_num]['item_id'].unique()
grid.append(np.array(list(product(*[cur_shops, cur_items, [block_num]])) ,dtype='int32'))
grid = pd.DataFrame(np.vstack(grid), columns = index_cols,dtype=np.int32)
 train = pd.merge(grid,train,how='left',on=index_cols ).fillna(0)
 train.sort_values(['date_block_num','shop_id','item_id'],inplace=True )
feature_cols = [c for c in train.columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 num_features<define_variables>
cumsum = train.groupby('item_id')['item_cnt_month'].cumsum() - train['item_cnt_month']
cumc = train.groupby('item_id' ).cumcount() + 1
train['item_target_enc'] = cumsum/cumc
train['item_target_enc'].fillna(0.3343, inplace=True)
 encoded_feature = train['item_target_enc'].values
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1<choose_model_class>
train['month'] = train['date_block_num'].map(lambda month: month-12*(month//12))
model = Model(num_features, num_all_targets)
 model<define_search_model>
def number_of_weekens_in_month(first_day):
ndays = first_day.daysinmonth
weekends = 0
for i in range(ndays):
if(pd.to_datetime(first_day + timedelta(days=(np.long(i)))).dayofweek in [5, 6]):
weekends = weekends + 1
return weekends
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in SEEDS:
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = list(range(42,49))
NFOLDS = 10
DRUG_THRESH = 18
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 train.head()<prepare_x_and_y>
first_days_map = sales_train.groupby('date_block_num')['date'].min().map(lambda date: date.replace(day=1))
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 train_ = train
test_ = test
kfold_col = f'kfold_{seed_id}'
trn_idx = train_[train_[kfold_col] != fold_id].index
val_idx = train_[train_[kfold_col] == fold_id].index
train_df = train_[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 def train_model(model, tag_name, target_cols_now, fine_tune_scheduler=None):
x_train, y_train = train_df[feature_cols].values, train_df[target_cols_now].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols_now].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3, weight_decay=WEIGHT_DECAY[tag_name])
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
steps_per_epoch=len(trainloader),
pct_start=PCT_START,
div_factor=DIV_FACTOR[tag_name],
max_lr=MAX_LR[tag_name],
epochs=EPOCHS)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing=0.001)
 oof = np.zeros(( len(train), len(target_cols_now)))
 best_loss = np.inf
for epoch in range(EPOCHS):
if fine_tune_scheduler is not None:
fine_tune_scheduler.step(epoch, model)
 train_loss = train_fn(model, optimizer, scheduler, loss_tr, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed_id}, FOLD: {fold_id}, {tag_name}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}")
 if np.isnan(valid_loss):
break
if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"{tag_name}_SEED_{seed_id}_FOLD{fold_id}_.pth")
 return oof
model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f".. /input/nn-transferlearning-7seeds4248-10folds/SCORED_ONLY_SEED_{seed_id}_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 print(f"Pretrained_Model_Loaded_SCORED_ONLYSEED: {seed_id}, FOLD: {fold_id} ")
 return 0.02, predictions<categorify>
train['number_of_weekends'] = train['date_block_num'].map(first_days_map.map(number_of_weekens_in_month))
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<categorify>
sales_train['month'] = sales_train['date_block_num'].map(lambda month: month-12*(month//12))
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_begin = time()
for seed_id in SEEDS:
oof_, predictions_ = run_k_fold(NFOLDS, seed_id)
 oof += oof_ / len(SEEDS)
 predictions += predictions_ / len(SEEDS)
 time_diff = time() - time_begin
train[target_cols] = oof
test[target_cols] = predictions<feature_engineering>
train['days_in_month'] = train['date_block_num'].map(first_days_map.map(lambda day: day.daysinmonth))
str(timedelta(seconds=time_diff))<load_from_csv>
def specify_the_accuracy_group(row):
num_of_block = row.date_block_num - 1
if(num_of_block != -1):
values = train.loc[(train['shop_id']==row.shop_id)&(train['item_id']==row.item_id)&\ 
(train['date_block_num']==num_of_block),['item_cnt_month']].values
return 0 if(values.size==0)else values[0][0]
else:
return 0
data_dir = '.. /input/lish-moa/'
train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_scored.head()<compute_test_metric>
def previous_months_value(df, collumn, offset):
previous_month_values = df.copy()
previous_month_values['date_block_num'] = previous_month_values['date_block_num'] + offset
previous_month_values.rename(columns={collumn: 'prev_month_' + str(offset)+ '_' + collumn}, inplace=True)
 return pd.merge(df, previous_month_values[['date_block_num','shop_id','item_id','prev_month_' + str(offset)+ '_' + collumn]], on=['date_block_num','shop_id','item_id'], how='left' )
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
y_pred = np.clip(y_pred, 0.0005, 0.9995)
 score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1] )<save_to_csv>
offsets = [1,2,3,9]
sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<load_pretrained>
column_to_offset = 'item_cnt_month'
saved_path_name = '.. /input/nn-transferlearning-7seeds4248-10folds/'
oof_TabNet_all = np.load(saved_path_name + 'oof_nn_transfer_all.npy')
 print(oof_TabNet_all.shape)
<compute_test_metric>
for offset in offsets:
train = previous_months_value(train, column_to_offset, offset)
 values = {'prev_month_' + str(offset)+ '_' + column_to_offset: 0}
train = train.fillna(value=values )
def log_loss_numpy(y_pred, y_true):
y_true_ravel = np.asarray(y_true ).ravel()
y_pred = np.asarray(y_pred ).ravel()
y_pred = np.clip(y_pred, 1e-15, 1 - 1e-15)
 loss = np.where(y_true_ravel == 1, - np.log(y_pred), - np.log(1 - y_pred))
return loss.mean()<compute_test_metric>
item_price_map = sales_train.loc[sales_train.item_price > 0].groupby(['item_id'])['item_price'].mean()
log_loss_numpy(y_pred, y_true )<load_from_csv>
train['price'] = train['item_id'].map(item_price_map )
train=pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 train<count_missing_values>
first_month_product_appeared = sales_train.groupby(['shop_id', 'item_id'])['date_block_num'].min().reset_index()
first_month_product_appeared.rename(columns={'date_block_num': 'first_appeared'}, inplace=True)
 train = pd.merge(train,first_month_product_appeared, on=['shop_id','item_id'], how='left')
 train['first_appeared'] = train['date_block_num'] - train['first_appeared'] + 1
train['first_appeared'] = train['first_appeared'].map(lambda n: 0 if(n < 0)else n)
 values = {'first_appeared': 0}
train = train.fillna(value=values )
train.isnull().sum(axis=0 ).sum() , train.isnull().sum(axis=1 ).sum()<count_unique_values>
new_shop = train.groupby('shop_id')['date_block_num'].min().map(lambda month: 1 if(month > 10)else 0 )
train.nunique(dropna=False ).sort_values()<load_from_csv>
train['new_shop'] = train['shop_id'].map(new_shop )
train_target = pd.read_csv(".. /input/lish-moa/train_targets_scored.csv")
 limit = 0
for col in tqdm_notebook(train_target.columns):
if col != "sig_id":
print(train_target[col].value_counts())
 limit+=1
if limit >= 15:
break<drop_column>
sales_incompleate_data = train.groupby('shop_id')['date_block_num'].max().map(lambda month: 1 if(month < 33)else 0 )
ctlVehicle_idx = train["cp_type"] != "ctl_vehicle"
train = train.loc[ctlVehicle_idx].reset_index(drop=True)
 train = train.drop("cp_type", axis=1)
 train_target = train_target.loc[ctlVehicle_idx].reset_index(drop=True )<define_variables>
train['incompleate_data'] = train['shop_id'].map(sales_incompleate_data )
gcols = [g for g in train.columns if "g-" in g]
ccols = [c for c in train.columns if "c-" in c]
cpcols = [cp for cp in train.columns if "cp_" in cp]<load_from_csv>
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0] )
test = pd.read_csv("/kaggle/input/lish-moa/test_features.csv")
 ctlVehicle_test = test["cp_type"] == "ctl_vehicle"
test = test.drop("cp_type", axis=1)
 enc = LabelEncoder()
for col in train[cpcols]:
train[col] = enc.fit_transform(train[col])
 enc = LabelEncoder()
for col in test[cpcols]:
test[col] = enc.fit_transform(test[col] )<train_model>
shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
g_pca = PCA(n_components=0.95)
 c_pca = PCA(n_components=0.95)
 train_test_g_concat = pd.concat([train[gcols], test[gcols]], axis=0)
 train_test_c_concat = pd.concat([train[ccols], test[ccols]], axis=0)
 g_pca.fit(train_test_g_concat)
 c_pca.fit(train_test_c_concat)
 train_gtrans = pd.DataFrame(g_pca.transform(train[gcols]), columns=["g_PCA" + str(i)for i in range(g_pca.n_components_)], index=train.index)
 test_gtrans = pd.DataFrame(g_pca.transform(test[gcols]), columns=["g_PCA" + str(i)for i in range(g_pca.n_components_)], index=test.index)
 train_ctrans = pd.DataFrame(c_pca.transform(train[ccols]), columns=["c_PCA" + str(i)for i in range(c_pca.n_components_)], index=train.index)
 test_ctrans = pd.DataFrame(c_pca.transform(test[ccols]), columns=["c_PCA" + str(i)for i in range(c_pca.n_components_)], index=test.index)
 g_pca.n_components_, c_pca.n_components_<concatenate>
shops['city'] = shops['shop_name'].str.split(' ' ).map(lambda x: x[0])
 shops.loc[shops.city == '!Якутск', 'city'] = 'Якутск'
shops['city_code'] = LabelEncoder().fit_transform(shops['city'])
 shops = shops[['shop_id','city_code']]
train = pd.concat([train_gtrans, train_ctrans, train[cpcols]], axis=1)
 test = pd.concat([test_gtrans, test_ctrans, test[cpcols]], axis=1)
 train<drop_column>
categories['split'] = categories['item_category_name'].str.split('-')
 categories['type'] = categories['split'].map(lambda x: x[0].strip())
 categories['type_code'] = LabelEncoder().fit_transform(categories['type'])
 categories['subtype'] = categories['split'].map(lambda x: x[1].strip() if len(x)> 1 else x[0].strip())
 categories['subtype_code'] = LabelEncoder().fit_transform(categories['subtype'])
 categories = categories[['item_category_id','type_code', 'subtype_code']]
train_target = train_target.drop("sig_id", axis=1)
 train_target<choose_model_class>
train = pd.merge(train, shops, on=['shop_id'], how='left' )
def create_model() :
model = tf.keras.Sequential([
tfa.layers.WeightNormalization(L.Dense(train.shape[1], input_shape=(train.shape[1],))),
L.BatchNormalization() ,
tfa.layers.WeightNormalization(L.Dense(1024, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.3),
tfa.layers.WeightNormalization(L.Dense(1024, activation="relu")) ,
L.BatchNormalization() ,
L.Dropout(0.2),
tfa.layers.WeightNormalization(L.Dense(train_target.shape[1], activation="sigmoid"))
])
 sgd = tf.keras.optimizers.SGD()
adamw = tfa.optimizers.AdamW(weight_decay = 1e-5)
 adam = tf.keras.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead_radam = tfa.optimizers.Lookahead(radam)
 model.compile(loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-15), optimizer=lookahead_radam)
 return model<train_model>
train.drop('month', axis=1, inplace=True )
predictions = []
kf = KFold(shuffle=True, random_state=42)
 for fold_id,(train_idx, valid_idx)in enumerate(kf.split(train)) :
model = create_model()
history = model.fit(train.iloc[train_idx], train_target.iloc[train_idx],
validation_data=(train.iloc[valid_idx], train_target.iloc[valid_idx]),
epochs=50,
verbose=2,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True),
tf.keras.callbacks.ModelCheckpoint("model_fold" + str(fold_id)+ ".h5", save_best_only=True, save_weights_only=True)
 ])
 print("Fold ID: {}, train loss: {}, valid loss: {}".format(fold_id, min(history.history["loss"]), min(history.history["val_loss"])))
 model.load_weights("model_fold" + str(fold_id)+ ".h5")
 predictions.append(model.predict(test))<normalization>
train = pd.merge(train, items, on=['item_id'], how='left')
 train = pd.merge(train, categories, on=['item_category_id'], how='left')
 train['city_code'] = train['city_code'].astype(np.int8)
 train['item_category_id'] = train['item_category_id'].astype(np.int8)
 train['type_code'] = train['type_code'].astype(np.int8)
 train['subtype_code'] = train['subtype_code'].astype(np.int8 )
pred = np.average(predictions, axis=0)
 pred = np.clip(pred, 0.001, 0.999)
 pred.shape<load_from_csv>
train.drop(['item_name'], axis=1, inplace=True )
sub = pd.read_csv(".. /input/lish-moa/sample_submission.csv")
 sub.loc[:, 1:] = pred
sub.loc[ctlVehicle_test, sub.columns != "sig_id"] = 0
sub<save_to_csv>
X_train = train[train.date_block_num < 33].drop(['item_cnt_month'], axis=1)
 Y_train = train[train.date_block_num < 33]['item_cnt_month']
X_valid = train[train.date_block_num == 33].drop(['item_cnt_month'], axis=1)
 Y_valid = train[train.date_block_num == 33]['item_cnt_month']
X_test = train[train.date_block_num == 34].drop(['item_cnt_month'], axis=1 )
sub.to_csv("submission.csv", index=False )<install_modules>
baseline_preds = np.full(( len(X_valid)) , train.loc[train.date_block_num < 33, 'item_cnt_month'].mean())
 rmse_b = np.sqrt(mean_squared_error(Y_valid.values, baseline_preds))
print("RMSE: %f" %(rmse_b))
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/<set_options>
parameters = {'tree_method': 'exact'}
model = XGBRegressor(
 max_depth=8,
tree_method='exact',
n_estimators=1000,
min_child_weight=300,
colsample_bytree=0.8,
subsample=0.8,
eta=0.3,
seed=42)
 model.fit(
 X_train,
Y_train,
eval_metric="rmse",
eval_set=[(X_train, Y_train),(X_valid, Y_valid)],
verbose=True,
early_stopping_rounds = 10 )
warnings.filterwarnings("ignore")
 sys.path.append(".. /input/rank-gauss")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN<set_options>
X_test.days_in_month = 30
X_test.number_of_weekends = 9
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )<define_variables>
X_test = pd.merge(test.drop('ID', axis=1), X_test, on=['shop_id', 'item_id', 'date_block_num'], how='left' )
<load_from_csv><EOS> 
Y_test = model.predict(X_test ).clip(0, 20)
 submission = pd.DataFrame({
"ID": test.index,
"item_cnt_month": Y_test
})
 submission.to_csv('xgb_submission.csv', index=False )