Kaggle competition: Predict Future Sales
LEARNING_RATE = config.lr
WEIGHT_DECAY = config.weight_decay
BATCH_SIZE = config.batch_size
EPOCHS = config.epochs
SEED_AVG = config.seed_avg
NFOLDS = config.nfold
VERBOSE = config.verbose
if config.is_debug:
EPOCHS = 3
SEED_AVG = [0, 101]
NFOLDS = 3
VERBOSE = 2<prepare_x_and_y>
def import_data() :

 sales = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/sales_train.csv")
 item_cat = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/item_categories.csv")
 items = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/items.csv")
 shops = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/shops.csv")
 test = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/test.csv")
 return sales, item_cat, items, shops, test
def downcast_dtypes(df):

 float_cols = [c for c in df if df[c].dtype == "float64"]
int_cols = [c for c in df if df[c].dtype in ["int64", "int32"]]
df[float_cols] = df[float_cols].astype(np.float32)
 df[int_cols] = df[int_cols].astype(np.int16)
 return df
def run_training(tr_idx, va_idx, fold, seed):
seed_everything(seed)
 oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
predictions = np.zeros(( len(test), target.iloc[:, 1:].shape[1]))
train_df = train.iloc[tr_idx]
valid_df = train.iloc[va_idx]
x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
K.clear_session()
model = create_model_3l(len(feature_cols), len(target_cols))
early_stopping = tf.keras.callbacks.EarlyStopping(
 monitor = 'val_binary_crossentropy',
mode = 'min',
patience = 10,
restore_best_weights = True,
verbose = 2 
)
 reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(
 monitor = 'val_binary_crossentropy',
mode = 'min',
factor = 0.3,
patience = 3,
verbose = 2 
)
 model.fit(
 x_train, y_train,
validation_data =(x_valid, y_valid),
epochs = EPOCHS,
batch_size = BATCH_SIZE,
callbacks = [early_stopping, reduce_lr],
verbose = 2 
)
 oof[va_idx] = model.predict(x_valid)
 predictions = model.predict(test[feature_cols].values)
 return oof, predictions<train_model>
SALES, ITEM_CAT, ITEMS, SHOPS, TEST = import_data()
%%time
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED_AVG:
print(f'============== Training SEED is {seed} ==============')
 start = time.time()
oof_fold = np.zeros(( len(train), len(target_cols)))
 predictions_fold = np.zeros(( len(test), len(target_cols)))
 mskf = MultilabelStratifiedKFold(n_splits=NFOLDS, random_state=seed, shuffle=True)
 for i_fold,(tr_idx, va_idx)in enumerate(tqdm(mskf.split(X=train, y=target))):
oof_, pred_ = run_training(tr_idx, va_idx, i_fold, seed)
 oof_fold += oof_
predictions_fold += pred_ / NFOLDS
oof_score = mean_log_loss(target.drop('sig_id', axis=1 ).values, oof_fold)
 print(f'seed: {seed} fold mean log loss score is {oof_score}')
 oof += oof_fold / len(SEED_AVG)
 predictions += predictions_fold / len(SEED_AVG)
 elapsed_time = time.time() - start
print(f'SEED: {seed} Elapsed_time:{elapsed_time:.4f} sec')
 seed_log_loss = mean_log_loss(target.drop('sig_id', axis=1 ).values, oof)
 print(f'Our out of folds log loss for our seed blend model is {seed_log_loss}')
 print(f'==================== Training END ====================' )<merge>
def pre_process_item_data(df):

 df['name_pre'] = df.item_name.str.replace(r'[\*\!\./,]', '')
 df['name_pre'] = df.name_pre.str.lower()
df['name_pre'] = df.name_pre.str.replace(r'd$', '' ).str.strip()
df['item_lab'] = df.name_pre.factorize(sort=True)[0]
train[target_cols] = oof
test[target_cols] = predictions
print(len(target_cols))
valid_results = df_Train_target_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 print(df_Train_target_scored.shape, valid_results.shape )<compute_test_metric>
pre_process_item_data(ITEMS)
 SALES = SALES.merge(ITEMS[['item_id', 'item_category_id', 'item_lab']], how='left', on='item_id')
y_true = df_Train_target_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )<set_options>
SALES.drop(['item_id', 'date', 'item_category_id'], axis=1, inplace=True )
config.is_kaggle<save_to_csv>
clip_fillna_prices(SALES)
 PRICE = SALES[['item_lab', 'item_price']][SALES.item_price > 0].groupby('item_lab' ).agg('mean')
 MATRIX = aggregate_item_count(SALES)
 MATRIX = add_missing_item_shop_pairs(MATRIX, PRICE)
 MATRIX = MATRIX.merge(ITEMS[['item_category_id', 'item_lab']].drop_duplicates() , how='left', on='item_lab' )
if config.is_kaggle:
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )<install_modules>
TEST['date_block_num'] = 34
TEST.drop('ID', axis=1, inplace=True)
 TEST['item_cnt_month'] = np.nan
TEST = TEST.merge(ITEMS[['item_id', 'item_lab', 'item_category_id']], how='left', on='item_id')
 TEST = pd.merge(TEST.drop('item_id', axis=1), PRICE, how='left', on='item_lab')
 MATRIX = MATRIX.append(TEST, ignore_index=True)
 MATRIX['item_category_id'] = MATRIX.item_category_id.astype('int16')
!pip uninstall -y typing
!pip install.. /input/pytorchtabnetpretraining/pytorch_tabnet-2.0.1-py3-none-any.whl
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/<feature_engineering>
def fillna_means(df, col):

 df[col] = df.groupby('item_lab')[col].transform(lambda x: x.fillna(x.mean()))
df[col] = df.groupby('item_category_id')[col].transform(lambda x: x.fillna(x.mean()))
os.environ["CUDA_LAUNCH_BLOCKING"] = "1" 
<set_options>
def pre_process_item_cat_data(df):

 cat = df.item_category_name.str.split('-', n=1, expand=True)
 df['cat1'] = cat[0].str.strip().str.lower()
df['cat2'] = cat[1].str.strip().str.lower()
df.cat1.fillna('', inplace=True)
 df.cat2.fillna('', inplace=True)
 df['cat1_lab'] = df.cat1.factorize(sort=True)[0]
df['cat2_lab'] = df.cat2.factorize(sort=True)[0]
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )<load_from_csv>
def get_string_correlation(array):
def dist(x, y):
lev = Levenshtein.distance(x[0],y[0])
 m = np.mean([len(x[0]), len(y[0])])
 return lev / m
cor = pdist(array.reshape(-1, 1), dist)
 return squareform(cor)
 def agglomerative_clustering(distance_matrix, num_clusters=None, threshold=None):
model = AgglomerativeClustering(n_clusters=num_clusters, affinity='precomputed', linkage='average', distance_threshold=threshold)
 model.fit(distance_matrix)
 return model.labels_
def add_levenshtein_feature(df, cols):
for col in cols:
categories = df[col].unique()
corr = get_string_correlation(categories)
 labels = agglomerative_clustering(corr, threshold=0.5)
 features = pd.DataFrame(np.array([categories, labels] ).T, columns=[col, col + '_lev'])
 df = df.merge(features, how='left', on=col)
 df[col + '_lev'] = df[col + '_lev'].astype('int32')
 return df
data_path = ".. /input/lish-moa/"
train = pd.read_csv(data_path+'train_features.csv')
 train.drop(columns=["sig_id"], inplace=True)
 drug = pd.read_csv(data_path +'train_drug.csv')
 train_targets_scored = pd.read_csv(data_path+'train_targets_scored.csv')
 targets = train_targets_scored.columns[1:]
test = pd.read_csv(data_path+'test_features.csv')
 test.drop(columns=["sig_id"], inplace=True)
 submission = pd.read_csv(data_path+'sample_submission.csv')
 remove_vehicle = True
if remove_vehicle:
kept_index = train['cp_type']=='trt_cp'
train = train.loc[kept_index].reset_index(drop=True)
 train_targets_scored = train_targets_scored.loc[kept_index].reset_index(drop=True)
 train["cp_type"] =(train["cp_type"]=="trt_cp")+ 0
train["cp_dose"] =(train["cp_dose"]=="D1")+ 0
test["cp_type"] =(test["cp_type"]=="trt_cp")+ 0
test["cp_dose"] =(test["cp_dose"]=="D1")+ 0
X_test = test.values
SEED = 42
NB_SPLITS = 5
scored = train_targets_scored.merge(drug, on='sig_id', how='left')
 vc = scored.drug_id.value_counts()
vc1 = vc.loc[vc<=18].index.sort_values()
vc2 = vc.loc[vc>18].index.sort_values()
dct1 = {}; dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NB_SPLITS, shuffle=True,
random_state=SEED)
 tmp = scored.groupby('drug_id')[targets].mean().loc[vc1]
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NB_SPLITS, shuffle=True,
random_state=SEED)
 tmp = scored.loc[scored.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 scored['fold'] = scored.drug_id.map(dct1)
 scored.loc[scored.fold.isna() ,'fold'] =\
scored.loc[scored.fold.isna() ,'sig_id'].map(dct2)
 scored.fold = scored.fold.astype('int8')
<import_modules>
pre_process_item_cat_data(ITEM_CAT)
 ITEM_CAT = add_levenshtein_feature(ITEM_CAT, ['cat1', 'cat2'])
 MATRIX = MATRIX.merge(ITEM_CAT[['item_category_id', 'cat1_lab', 'cat2_lab', 'cat1_lev', 'cat2_lev']], how='left',
on='item_category_id')
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1-y_true)*np.log(1-logits+1e-15)+ y_true*np.log(logits+1e-15)
 return np.mean(-aux )<init_hyperparams>
def pre_process_shop_data(df):
split = df.shop_name.str.split(n=1, expand=True)
 df['city'] = split[0].str.lower().str.strip()
df['city_id'] = df.city.factorize(sort=True)[0]
df['shopping_center'] = split[1].str.lower().str.contains('ТЦ'.lower())
BS=1024
MAX_EPOCH=101
N_D = 128
N_A = 32
N_INDEP = 1
N_SHARED = 1
N_STEPS = 3
tabnet_params = dict(n_d=N_D, n_a=N_A, n_steps=N_STEPS,
n_independent=N_INDEP, n_shared=N_SHARED,
lambda_sparse=0., optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2),
mask_type="entmax",
scheduler_params=dict(mode="min",
patience=5,
min_lr=1e-5,
factor=0.9,),
scheduler_fn=torch.optim.lr_scheduler.ReduceLROnPlateau,
verbose=10, 
)
 pretrainer = TabNetPretrainer(**tabnet_params)
 pretrainer.fit(X_train=test.values[:,1:],
eval_set=[train.values[:,1:]],
max_epochs=MAX_EPOCH,
patience=20, batch_size=BS, virtual_batch_size=128,
num_workers=0, drop_last=True,
pretraining_ratio=0.8 )<prepare_x_and_y>
pre_process_shop_data(SHOPS)
 MATRIX = MATRIX.merge(SHOPS[['shop_id', 'city_id', 'shopping_center']], how='left', on='shop_id' )
scores_auc_all= []
test_cv_preds = []
oof_preds = []
oof_targets = []
scores = []
scores_auc = []
NB_FOLD = 5
for fold_nb in range(NB_FOLD):
train_idx = scored[scored.fold!=fold_nb].index
val_idx = scored[scored.fold==fold_nb].index
print("FOLDS : ", fold_nb)
 if fold_nb >= NB_FOLD:
break
X_train, y_train = train.values[train_idx, 1:], train_targets_scored.values[train_idx, 1:].astype(float)
 X_val, y_val = train.values[val_idx, 1:], train_targets_scored.values[val_idx, 1:].astype(float)
 MAX_EPOCH=51
BS=1024
tabnet_params = dict(n_d=N_D, n_a=N_A, n_steps=N_STEPS,
n_independent=N_INDEP, n_shared=N_SHARED,
gamma=1.0,
lambda_sparse=0., optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2,
weight_decay=1e-5 
),
mask_type="entmax",
scheduler_params=dict(max_lr=0.05,
steps_per_epoch=int(X_train.shape[0] / BS),
epochs=MAX_EPOCH,
is_batch_level=True),
scheduler_fn=torch.optim.lr_scheduler.OneCycleLR,
verbose=10, 
)
 model = TabNetRegressor(**tabnet_params)
 model.fit(X_train=X_train,
y_train=y_train,
eval_set=[(X_val, y_val)],
eval_name = ["val"],
eval_metric = ["logits_ll"],
max_epochs=MAX_EPOCH,
patience=20, batch_size=BS, virtual_batch_size=128,
num_workers=1, drop_last=True,
from_unsupervised=pretrainer,
loss_fn=torch.nn.functional.binary_cross_entropy_with_logits)
 preds_val = model.predict(X_val)
 preds = 1 /(1 + np.exp(-preds_val))
score = np.min(model.history["val_logits_ll"])
 scores.append(score)
 oof_preds.append(preds)
 oof_targets.append(y_val)
 preds_test = model.predict(X_test[:,1:])
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 oof_preds_all = np.concatenate(oof_preds)
 oof_targets_all = np.concatenate(oof_targets)
 test_preds_all = np.stack(test_cv_preds )<compute_test_metric>
def add_shop_revenue(df):
df['revenue'] = df.item_price * df.item_cnt_month
df['shop_revenue_month'] = df.groupby(['shop_id', 'date_block_num'])['revenue'].transform('sum')
 del df['revenue']
aucs = []
for task_id in range(oof_preds_all.shape[1]):
aucs.append(roc_auc_score(y_true=oof_targets_all[:, task_id],
y_score=oof_preds_all[:, task_id]))
print(f"Overall AUC : {np.mean(aucs)}")
 print(f"Average CV : {np.mean(scores)}" )<save_to_csv>
add_shop_revenue(MATRIX)
all_feat = [col for col in submission.columns if col not in ["sig_id"]]
submission[all_feat] = test_preds_all.mean(axis=0)
 submission.loc[test['cp_type']==0, submission.columns[1:]] = 0
submission.to_csv('submission.csv', index=None )<set_options>
def lag_feature(df, lags, col):

 tmp = df[['date_block_num', 'shop_id', 'item_lab', col]]
for i in lags:
shifted = tmp.copy()
shifted.columns = ['date_block_num', 'shop_id', 'item_lab', col+'_lag_'+str(i)]
shifted['date_block_num'] += i
df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_lab'], how='left')
 df[col+'_lag_'+str(i)].fillna(0, inplace=True)
 return df
warnings.filterwarnings('ignore' )<import_modules>
MATRIX = lag_feature(MATRIX,(1, 2, 12), 'item_cnt_month')
 MATRIX = lag_feature(MATRIX,(1, 2, 12), 'shop_revenue_month')
 del MATRIX['shop_revenue_month']
from sklearn.preprocessing import QuantileTransformer<load_from_csv>
MATRIX['month'] = MATRIX.date_block_num.mod(12)
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 df = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )<create_dataframe>
MATRIX['release'] = MATRIX.item_lab.map(
 MATRIX[MATRIX.item_cnt_month > 0].groupby(['item_lab'] ).agg(release=('date_block_num', 'min')).loc[:,'release'] 
)
 MATRIX['since_release'] = MATRIX.date_block_num - MATRIX.release
MATRIX['since_release'] = MATRIX.since_release.fillna(0)
 MATRIX = MATRIX[MATRIX.since_release >= 0]
del MATRIX['release']
train_features2=train_features.copy()
test_features2=test_features.copy()<define_variables>
MATRIX['mean_item_cnt_month'] = MATRIX.groupby('date_block_num')['item_cnt_month'].transform('mean')
 MATRIX[MATRIX.item_cnt_month == 34].mean_item_cnt_month = 0.28394
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]<categorify>
IDX_TRAIN =(MATRIX.date_block_num > 11)&(MATRIX.date_block_num < 33)
 IDX_VAL = MATRIX.date_block_num == 33
IDX_TEST = MATRIX.date_block_num == 34
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]<set_options>
def add_max_cnt(df):
max_cnt = df.loc[IDX_TRAIN, ['item_lab', 'shop_id', 'item_cnt_month']].groupby(['item_lab', 'shop_id'] ).agg(
 max_cnt=('item_cnt_month', 'max')).reset_index()
df = df.merge(max_cnt, how='left', on=['item_lab', 'shop_id'])
 return df
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )<create_dataframe>
MATRIX = add_max_cnt(MATRIX)
 fillna_means(MATRIX, 'max_cnt')
 IDX_TRAIN =(MATRIX.date_block_num > 11)&(MATRIX.date_block_num < 33)
 IDX_VAL = MATRIX.date_block_num == 33
IDX_TEST = MATRIX.date_block_num == 34
n_comp = 600
pca_g = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 gpca=(pca_g.fit(data[GENES]))
train2=(gpca.transform(train_features[GENES]))
test2 =(gpca.transform(test_features[GENES]))
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1)
 dump(gpca, open('gpca.pkl', 'wb'))<create_dataframe>
def get_mean_encodings_kfold(df, idx_train, idx_val, idx_test, feature_names, target_name, folds=5):
skf = StratifiedKFold(n_splits=folds, shuffle=True)
 global_mean = df.loc[idx_train, target_name].mean()
for feature_name in feature_names:
print(feature_name)
 df.loc[:, feature_name + '_mean'] = np.nan
data = df.loc[idx_train, [feature_name, feature_name + '_mean', target_name]]
for idx_1, idx_2 in skf.split(data[[feature_name]], data[feature_name]):
x_1 = data.iloc[idx_1].loc[:, [feature_name, feature_name + '_mean', target_name]]
x_2 = data.iloc[idx_2].loc[:, [feature_name, feature_name + '_mean', target_name]]
means = x_1.groupby(feature_name ).agg(mean_target=(target_name, 'mean')).loc[:, 'mean_target']
x_2[feature_name + '_mean'] = x_2[feature_name].map(means)
 data.update(x_2)
 data[feature_name + '_mean'].fillna(global_mean, inplace=True)
 df.update(data)
 for feature_name in feature_names:
x_1 = df.loc[idx_train, [feature_name, target_name]]
x_2 = df.loc[np.logical_or(idx_val, idx_test), [feature_name, target_name]]
means = x_1.groupby(feature_name ).agg(mean_target=(target_name, 'mean')).loc[:, 'mean_target']
x_2[feature_name + '_mean'] = x_2[feature_name].map(means)
 df.update(x_2)
 return df
n_comp = 50
pca_c = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 cpca=(pca_c.fit(data[CELLS]))
train2=(cpca.transform(train_features[CELLS]))
test2 =(cpca.transform(test_features[CELLS]))
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1)
 dump(cpca, open('cpca.pkl', 'wb'))<concatenate>
MATRIX = get_mean_encodings_kfold(
 MATRIX, IDX_TRAIN, IDX_VAL, IDX_TEST,
['shop_id', 'city_id', 'item_lab', 'cat1_lab', 'cat2_lab', 'cat1_lev', 'cat2_lev', 'month'],
'item_cnt_month')
c_n = [f for f in list(train_features.columns)if f not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train_features[c_n].var() >= 0.85 ).values
tmp = train_features[c_n].loc[:, mask]
train_features = pd.concat([train_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 tmp = test_features[c_n].loc[:, mask]
test_features = pd.concat([test_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1 )<define_search_model>
TRAIN_X = MATRIX[IDX_TRAIN].drop('item_cnt_month', axis=1)
 TRAIN_Y = MATRIX[IDX_TRAIN].item_cnt_month
VAL_X = MATRIX[IDX_VAL].drop('item_cnt_month', axis=1)
 VAL_Y = MATRIX[IDX_VAL].item_cnt_month
TEST_X = MATRIX[IDX_TEST].drop('item_cnt_month', axis=1 )
def fe_cluster_genes(train, test, n_clusters_g = 22, SEED = 42):
features_g = GENES
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans_genes = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 dump(kmeans_genes, open('kmeans_genes.pkl', 'wb'))
train[f'clusters_{kind}'] = kmeans_genes.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_genes.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 return train, test
train_features2 ,test_features2=fe_cluster_genes(train_features2,test_features2 )<find_best_model_class>
del SALES, ITEM_CAT, ITEMS, SHOPS, TEST
def fe_cluster_cells(train, test, n_clusters_c = 4, SEED = 42):
features_c = CELLS
def create_cluster(train, test, features, kind = 'c', n_clusters = n_clusters_c):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans_cells = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 dump(kmeans_cells, open('kmeans_cells.pkl', 'wb'))
train[f'clusters_{kind}'] = kmeans_cells.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_cells.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster_cells(train_features2,test_features2 )<concatenate>
gc.collect()
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )<categorify>
regressor = xgb.XGBRegressor(n_estimators = 100,
learning_rate = 0.08,
max_depth = 5,
subsample = 0.8,
colsample_bytree = 0.8,
n_jobs = 8 
)
 reg = regressor.fit(
 X=TRAIN_X,
y=TRAIN_Y,
eval_metric='rmse',
eval_set=[(TRAIN_X, TRAIN_Y),(VAL_X, VAL_Y)],
verbose=True,
early_stopping_rounds=8
)
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
data=pd.concat([train,test],axis=0)
 kmeans_pca = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 dump(kmeans_pca, open('kmeans_pca.pkl', 'wb'))
train[f'clusters_pca'] = kmeans_pca.predict(train.values)
 test[f'clusters_pca'] = kmeans_pca.predict(test.values)
 train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )<split>
pickle.dump(reg, open("xgb.pickle.dat", "wb"))
train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]<split>
importance.sort_values(ascending=False )
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]<define_variables>
TEST = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/test.csv")
 PRED = TEST_X[['item_lab', 'shop_id']]
PRED['item_cnt_month'] = reg.predict(TEST_X)
 PRED.drop_duplicates(inplace=True)
 print(TEST.shape, PRED.shape )
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']<feature_engineering>
ITEMS = pd.read_csv(".. /input/competitive-data-science-predict-future-sales/items.csv")
 pre_process_item_data(ITEMS)
 TEST = TEST.merge(ITEMS[['item_id', 'item_lab']], how='left', on='item_id')
 print(TEST.shape )
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-26'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
df['c26_c38'] = df['c-26'] * df['c-38']
df['c90_c13'] = df['c-90'] * df['c-13']
df['c85_c31'] = df['c-85'] * df['c-31']
df['c63_c42'] = df['c-63'] * df['c-42']
df['c94_c11'] = df['c-94'] * df['c-11']
df['c94_c60'] = df['c-94'] * df['c-60']
df['c55_c42'] = df['c-55'] * df['c-42']
df['g37_c50'] = df['g-37'] * df['g-50']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )<split>
TEST = TEST.merge(PRED, how='left', on=['item_lab', 'shop_id'])
 TEST.shape
<concatenate><EOS> 
TEST[['ID', 'item_cnt_month']].sort_values('ID' ).to_csv('submission.csv', index=False, header=True )